// Scoring-text construction: the document fed to the vectorizer. // // fluxrec is title-only by design (no body/excerpt ever enters the scoring // text). On top of the raw title we inject one synthetic feed-identity token // `feed-` built from the entry's feed_title, so LR can learn a per-feed // prior inside the existing TF-IDF + LR model. MinDF shrinks low-data feeds // toward zero for free (a feed with <2 labeled entries never enters the // vocabulary), so no separate shrinkage-prior machinery. // // Hyphens, not underscores: the tokenizer keeps [a-zA-Z0-9-] and splits on // everything else, so `feed-marginal-revolution` survives as ONE token while // `feed_marginal_revolution` would fragment into three. Identical slug // construction at train (from labels.jsonl feed_title) and serve (from the // entry's feed.title) keeps the feature consistent across both sides. package core import ( "regexp" "strings" ) var slugSep = regexp.MustCompile(`[^a-z0-9]+`) // FeedSlug normalizes a feed title into a stable slug of lowercase // letters/digits joined by single hyphens, trimmed. Empty/whitespace input // returns "". Idempotent: slug(slug(x)) == slug(x). func FeedSlug(feedTitle string) string { s := strings.ToLower(strings.TrimSpace(feedTitle)) s = slugSep.ReplaceAllString(s, "-") return strings.Trim(s, "-") } // ScoringText builds the document fed to the vectorizer: the entry title plus // one synthetic token "feed-". If the feed title is empty (no slug), the // token is omitted and the title alone is returned. No body/excerpt is ever // added — title-only is the design. func ScoringText(title, feedTitle string) string { slug := FeedSlug(feedTitle) if slug == "" { return title } return title + " feed-" + slug }