aboutsummaryrefslogtreecommitdiff
path: root/core/scoring_text.go
blob: 84cd75896571429d6c10441070c8e3d52b73942c (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
pre { line-height: 125%; }
td.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
span.linenos { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
td.linenos .special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
span.linenos.special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
.highlight .hll { background-color: #ffffcc }
.highlight .c { color: #888888 } /* Comment */
.highlight .err { color: #a61717; background-color: #e3d2d2 } /* Error */
.highlight .k { color: #008800; font-weight: bold } /* Keyword */
.highlight .ch { color: #888888 } /* Comment.Hashbang */
.highlight .cm { color: #888888 } /* Comment.Multiline */
.highlight .cp { color: #cc0000; font-weight: bold } /* Comment.Preproc */
.highlight
// Scoring-text construction: the document fed to the vectorizer.
//
// fluxrec is title-only by design (no body/excerpt ever enters the scoring
// text). On top of the raw title we inject one synthetic feed-identity token
// `feed-<slug>` built from the entry's feed_title, so LR can learn a per-feed
// prior inside the existing TF-IDF + LR model. MinDF shrinks low-data feeds
// toward zero for free (a feed with <2 labeled entries never enters the
// vocabulary), so no separate shrinkage-prior machinery.
//
// Hyphens, not underscores: the tokenizer keeps [a-zA-Z0-9-] and splits on
// everything else, so `feed-marginal-revolution` survives as ONE token while
// `feed_marginal_revolution` would fragment into three. Identical slug
// construction at train (from labels.jsonl feed_title) and serve (from the
// entry's feed.title) keeps the feature consistent across both sides.
package core

import (
	"regexp"
	"strings"
)

var slugSep = regexp.MustCompile(`[^a-z0-9]+`)

// FeedSlug normalizes a feed title into a stable slug of lowercase
// letters/digits joined by single hyphens, trimmed. Empty/whitespace input
// returns "". Idempotent: slug(slug(x)) == slug(x).
func FeedSlug(feedTitle string) string {
	s := strings.ToLower(strings.TrimSpace(feedTitle))
	s = slugSep.ReplaceAllString(s, "-")
	return strings.Trim(s, "-")
}

// ScoringText builds the document fed to the vectorizer: the entry title plus
// one synthetic token "feed-<slug>". If the feed title is empty (no slug), the
// token is omitted and the title alone is returned. No body/excerpt is ever
// added — title-only is the design.
func ScoringText(title, feedTitle string) string {
	slug := FeedSlug(feedTitle)
	if slug == "" {
		return title
	}
	return title + " feed-" + slug
}