1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
|
// Tests for feed-identity token and scoring-text construction.
package core
import (
"strings"
"testing"
)
func TestFeedSlug(t *testing.T) {
cases := []struct{ in, want string }{
{"Marginal Revolution", "marginal-revolution"},
{"ACM / IEEE", "acm-ieee"},
{" Already-Hyphenated ", "already-hyphenated"},
{"---", ""},
{"", ""},
{"MIT Tech Review #2", "mit-tech-review-2"},
{"Über Wörter", "ber-w-rter"}, // non-ascii stripped to hyphens
}
for _, c := range cases {
if got := FeedSlug(c.in); got != c.want {
t.Errorf("FeedSlug(%q) = %q, want %q", c.in, got, c.want)
}
}
}
func TestFeedSlugIdempotent(t *testing.T) {
for _, in := range []string{"Marginal Revolution", "ACM / IEEE", "X---Y", "a b c"} {
once := FeedSlug(in)
twice := FeedSlug(once)
if once != twice {
t.Errorf("slug not idempotent: %q → %q → %q", in, once, twice)
}
}
}
func TestScoringTextIncludesToken(t *testing.T) {
got := ScoringText("A Title", "Marginal Revolution")
if !strings.HasSuffix(got, " feed-marginal-revolution") {
t.Errorf("expected feed-identity token suffix, got %q", got)
}
}
func TestScoringTextEmptyFeedOmitsToken(t *testing.T) {
if got := ScoringText("A Title", ""); got != "A Title" {
t.Errorf("empty feed should yield title only, got %q", got)
}
if got := ScoringText("A Title", " --- "); got != "A Title" {
t.Errorf("whitespace-only feed should yield title only, got %q", got)
}
}
// The feed-identity token must round-trip through Tokenize as a SINGLE token
// (hyphens survive; underscores would fragment). Same feed → shared token;
// different feeds → different tokens.
func TestFeedTokenRoundTripsAsSingleToken(t *testing.T) {
doc := ScoringText("some title here", "Marginal Revolution")
toks := Tokenize(doc)
found := false
for _, tk := range toks {
if tk == "feed-marginal-revolution" {
found = true
break
}
}
if !found {
t.Fatalf("feed token not present as a single token in %v", toks)
}
// And it must not have fragmented into feed/marginal/revolution.
for _, bad := range []string{"feed", "marginal", "revolution"} {
for _, tk := range toks {
if tk == bad {
t.Errorf("token fragmented: %q appeared standalone", bad)
}
}
}
// Same feed → shared token; different feed → different token.
a := ScoringText("x", "Marginal Revolution")
b := ScoringText("y", "Marginal Revolution")
c := ScoringText("z", "Other Blog")
tokA, tokC := feedToken(t, a), feedToken(t, c)
if feedToken(t, b) != tokA {
t.Error("same feed should share the token")
}
if tokA == tokC {
t.Error("different feeds should differ")
}
}
func feedToken(t *testing.T, doc string) string {
t.Helper()
for _, tk := range Tokenize(doc) {
if strings.HasPrefix(tk, "feed-") {
return tk
}
}
t.Fatalf("no feed token in %q", doc)
return ""
}
|
