blob: 84cd75896571429d6c10441070c8e3d52b73942c (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
|
// Scoring-text construction: the document fed to the vectorizer.
//
// fluxrec is title-only by design (no body/excerpt ever enters the scoring
// text). On top of the raw title we inject one synthetic feed-identity token
// `feed-<slug>` built from the entry's feed_title, so LR can learn a per-feed
// prior inside the existing TF-IDF + LR model. MinDF shrinks low-data feeds
// toward zero for free (a feed with <2 labeled entries never enters the
// vocabulary), so no separate shrinkage-prior machinery.
//
// Hyphens, not underscores: the tokenizer keeps [a-zA-Z0-9-] and splits on
// everything else, so `feed-marginal-revolution` survives as ONE token while
// `feed_marginal_revolution` would fragment into three. Identical slug
// construction at train (from labels.jsonl feed_title) and serve (from the
// entry's feed.title) keeps the feature consistent across both sides.
package core
import (
"regexp"
"strings"
)
var slugSep = regexp.MustCompile(`[^a-z0-9]+`)
// FeedSlug normalizes a feed title into a stable slug of lowercase
// letters/digits joined by single hyphens, trimmed. Empty/whitespace input
// returns "". Idempotent: slug(slug(x)) == slug(x).
func FeedSlug(feedTitle string) string {
s := strings.ToLower(strings.TrimSpace(feedTitle))
s = slugSep.ReplaceAllString(s, "-")
return strings.Trim(s, "-")
}
// ScoringText builds the document fed to the vectorizer: the entry title plus
// one synthetic token "feed-<slug>". If the feed title is empty (no slug), the
// token is omitted and the title alone is returned. No body/excerpt is ever
// added — title-only is the design.
func ScoringText(title, feedTitle string) string {
slug := FeedSlug(feedTitle)
if slug == "" {
return title
}
return title + " feed-" + slug
}
|
