// Tests for feed-identity token and scoring-text construction. package core import ( "strings" "testing" ) func TestFeedSlug(t *testing.T) { cases := []struct{ in, want string }{ {"Marginal Revolution", "marginal-revolution"}, {"ACM / IEEE", "acm-ieee"}, {" Already-Hyphenated ", "already-hyphenated"}, {"---", ""}, {"", ""}, {"MIT Tech Review #2", "mit-tech-review-2"}, {"Über Wörter", "ber-w-rter"}, // non-ascii stripped to hyphens } for _, c := range cases { if got := FeedSlug(c.in); got != c.want { t.Errorf("FeedSlug(%q) = %q, want %q", c.in, got, c.want) } } } func TestFeedSlugIdempotent(t *testing.T) { for _, in := range []string{"Marginal Revolution", "ACM / IEEE", "X---Y", "a b c"} { once := FeedSlug(in) twice := FeedSlug(once) if once != twice { t.Errorf("slug not idempotent: %q → %q → %q", in, once, twice) } } } func TestScoringTextIncludesToken(t *testing.T) { got := ScoringText("A Title", "Marginal Revolution") if !strings.HasSuffix(got, " feed-marginal-revolution") { t.Errorf("expected feed-identity token suffix, got %q", got) } } func TestScoringTextEmptyFeedOmitsToken(t *testing.T) { if got := ScoringText("A Title", ""); got != "A Title" { t.Errorf("empty feed should yield title only, got %q", got) } if got := ScoringText("A Title", " --- "); got != "A Title" { t.Errorf("whitespace-only feed should yield title only, got %q", got) } } // The feed-identity token must round-trip through Tokenize as a SINGLE token // (hyphens survive; underscores would fragment). Same feed → shared token; // different feeds → different tokens. func TestFeedTokenRoundTripsAsSingleToken(t *testing.T) { doc := ScoringText("some title here", "Marginal Revolution") toks := Tokenize(doc) found := false for _, tk := range toks { if tk == "feed-marginal-revolution" { found = true break } } if !found { t.Fatalf("feed token not present as a single token in %v", toks) } // And it must not have fragmented into feed/marginal/revolution. for _, bad := range []string{"feed", "marginal", "revolution"} { for _, tk := range toks { if tk == bad { t.Errorf("token fragmented: %q appeared standalone", bad) } } } // Same feed → shared token; different feed → different token. a := ScoringText("x", "Marginal Revolution") b := ScoringText("y", "Marginal Revolution") c := ScoringText("z", "Other Blog") tokA, tokC := feedToken(t, a), feedToken(t, c) if feedToken(t, b) != tokA { t.Error("same feed should share the token") } if tokA == tokC { t.Error("different feeds should differ") } } func feedToken(t *testing.T, doc string) string { t.Helper() for _, tk := range Tokenize(doc) { if strings.HasPrefix(tk, "feed-") { return tk } } t.Fatalf("no feed token in %q", doc) return "" }