1 files changed, 99 insertions, 0 deletions
diff --git a/core/scoring_text_test.go b/core/scoring_text_test.go
new file mode 100644
index 0000000..87b3948
--- /dev/null
+++ b/core/scoring_text_test.go
@@ -0,0 +1,99 @@
+// Tests for feed-identity token and scoring-text construction.
+package core
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestFeedSlug(t *testing.T) {
+	cases := []struct{ in, want string }{
+		{"Marginal Revolution", "marginal-revolution"},
+		{"ACM / IEEE", "acm-ieee"},
+		{"  Already-Hyphenated  ", "already-hyphenated"},
+		{"---", ""},
+		{"", ""},
+		{"MIT Tech Review #2", "mit-tech-review-2"},
+		{"Über Wörter", "ber-w-rter"}, // non-ascii stripped to hyphens
+	}
+	for _, c := range cases {
+		if got := FeedSlug(c.in); got != c.want {
+			t.Errorf("FeedSlug(%q) = %q, want %q", c.in, got, c.want)
+		}
+	}
+}
+
+func TestFeedSlugIdempotent(t *testing.T) {
+	for _, in := range []string{"Marginal Revolution", "ACM / IEEE", "X---Y", "a b c"} {
+		once := FeedSlug(in)
+		twice := FeedSlug(once)
+		if once != twice {
+			t.Errorf("slug not idempotent: %q → %q → %q", in, once, twice)
+		}
+	}
+}
+
+func TestScoringTextIncludesToken(t *testing.T) {
+	got := ScoringText("A Title", "Marginal Revolution")
+	if !strings.HasSuffix(got, " feed-marginal-revolution") {
+		t.Errorf("expected feed-identity token suffix, got %q", got)
+	}
+}
+
+func TestScoringTextEmptyFeedOmitsToken(t *testing.T) {
+	if got := ScoringText("A Title", ""); got != "A Title" {
+		t.Errorf("empty feed should yield title only, got %q", got)
+	}
+	if got := ScoringText("A Title", "   ---   "); got != "A Title" {
+		t.Errorf("whitespace-only feed should yield title only, got %q", got)
+	}
+}
+
+// The feed-identity token must round-trip through Tokenize as a SINGLE token
+// (hyphens survive; underscores would fragment). Same feed → shared token;
+// different feeds → different tokens.
+func TestFeedTokenRoundTripsAsSingleToken(t *testing.T) {
+	doc := ScoringText("some title here", "Marginal Revolution")
+	toks := Tokenize(doc)
+	found := false
+	for _, tk := range toks {
+		if tk == "feed-marginal-revolution" {
+			found = true
+			break
+		}
+	}
+	if !found {
+		t.Fatalf("feed token not present as a single token in %v", toks)
+	}
+	// And it must not have fragmented into feed/marginal/revolution.
+	for _, bad := range []string{"feed", "marginal", "revolution"} {
+		for _, tk := range toks {
+			if tk == bad {
+				t.Errorf("token fragmented: %q appeared standalone", bad)
+			}
+		}
+	}
+
+	// Same feed → shared token; different feed → different token.
+	a := ScoringText("x", "Marginal Revolution")
+	b := ScoringText("y", "Marginal Revolution")
+	c := ScoringText("z", "Other Blog")
+	tokA, tokC := feedToken(t, a), feedToken(t, c)
+	if feedToken(t, b) != tokA {
+		t.Error("same feed should share the token")
+	}
+	if tokA == tokC {
+		t.Error("different feeds should differ")
+	}
+}
+
+func feedToken(t *testing.T, doc string) string {
+	t.Helper()
+	for _, tk := range Tokenize(doc) {
+		if strings.HasPrefix(tk, "feed-") {
+			return tk
+		}
+	}
+	t.Fatalf("no feed token in %q", doc)
+	return ""
+}