aboutsummaryrefslogtreecommitdiff
path: root/core
diff options
context:
space:
mode:
Diffstat (limited to 'core')
-rw-r--r--core/constants.go23
-rw-r--r--core/labels.go127
-rw-r--r--core/labels_test.go192
-rw-r--r--core/ml.go435
-rw-r--r--core/ml_test.go76
-rw-r--r--core/model.go22
-rw-r--r--core/runs.go167
-rw-r--r--core/scoring.go16
-rw-r--r--core/scoring_text.go43
-rw-r--r--core/scoring_text_test.go99
-rw-r--r--core/text.go38
-rw-r--r--core/types.go59
12 files changed, 1297 insertions, 0 deletions
diff --git a/core/constants.go b/core/constants.go
new file mode 100644
index 0000000..baba926
--- /dev/null
+++ b/core/constants.go
@@ -0,0 +1,23 @@
+// Vendored from scholscan/core/constants.go (rev 7c1a5ef, 2026-08-28).
+// Owned here; no upstream sync.
+// Default configuration constants.
+//
+// Timeouts are defensive: 30s for HTTP requests, 5s for graceful shutdown.
+// Score threshold 0.5 is neutral; models should learn their own.
+// MinTitleLength filters junk/broken titles (<15 chars rarely meaningful).
+// ChunkSize 50 balances memory usage vs batch efficiency.
+package core
+
+import "time"
+
+const (
+ DefaultHTTPTimeout = 30 * time.Second
+ DefaultContextTimeout = 10 * time.Second
+ DefaultReadTimeout = 30 * time.Second
+ DefaultWriteTimeout = 30 * time.Second
+ DefaultIdleTimeout = 120 * time.Second
+ DefaultShutdownTimeout = 5 * time.Second
+ DefaultScoreThreshold = 0.5
+ MinTitleLength = 15
+ DefaultChunkSize = 50
+)
diff --git a/core/labels.go b/core/labels.go
new file mode 100644
index 0000000..bdff515
--- /dev/null
+++ b/core/labels.go
@@ -0,0 +1,127 @@
+// labels.jsonl read/write and URL normalization.
+//
+// labels.jsonl is the canonical label store: one JSON LabelRow per line,
+// append-only. ReadLabels is strict (per-line errors with line numbers);
+// AppendLabels is the only writer — rows are never rewritten in place.
+package core
+
+import (
+ "bufio"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "net/url"
+ "os"
+ "sort"
+ "strings"
+)
+
+// NormalizeURL collapses trivially-equivalent article links into one dedup
+// key: scheme/host lowercased, leading "www." and trailing "/" dropped.
+// Unparseable input degrades to a plain lowercase-trim.
+func NormalizeURL(u string) string {
+ u = strings.TrimSpace(u)
+ pr, err := url.Parse(u)
+ if err != nil || pr.Host == "" {
+ return strings.TrimSuffix(strings.ToLower(u), "/")
+ }
+ pr.Scheme = strings.ToLower(pr.Scheme)
+ pr.Host = strings.TrimPrefix(strings.ToLower(pr.Host), "www.")
+ return strings.TrimSuffix(pr.String(), "/")
+}
+
+// ReadLabels parses path strictly. A missing file is not an error — it just
+// means no labels yet (first export run).
+func ReadLabels(path string) ([]LabelRow, error) {
+ f, err := os.Open(path)
+ if err != nil {
+ if errors.Is(err, os.ErrNotExist) {
+ return nil, nil
+ }
+ return nil, fmt.Errorf("opening labels: %w", err)
+ }
+ defer func() { _ = f.Close() }()
+
+ var rows []LabelRow
+ scanner := bufio.NewScanner(f)
+ scanner.Buffer(make([]byte, 64*1024), 4*1024*1024)
+ line := 0
+ for scanner.Scan() {
+ line++
+ text := strings.TrimSpace(scanner.Text())
+ if text == "" {
+ continue
+ }
+ var row LabelRow
+ if err := json.Unmarshal([]byte(text), &row); err != nil {
+ return nil, fmt.Errorf("%s:%d: invalid JSON: %w", path, line, err)
+ }
+ if row.EntryID <= 0 {
+ return nil, fmt.Errorf("%s:%d: entry_id must be positive", path, line)
+ }
+ if row.URL == "" {
+ return nil, fmt.Errorf("%s:%d: url is required", path, line)
+ }
+ if row.Label != LabelPos && row.Label != LabelNeg {
+ return nil, fmt.Errorf("%s:%d: label must be %q or %q, got %q", path, line, LabelPos, LabelNeg, row.Label)
+ }
+ if row.CapturedAt.IsZero() {
+ return nil, fmt.Errorf("%s:%d: captured_at is required", path, line)
+ }
+ rows = append(rows, row)
+ }
+ if err := scanner.Err(); err != nil {
+ return nil, fmt.Errorf("reading %s: %w", path, err)
+ }
+ return rows, nil
+}
+
+// SplitByTime partitions rows into train (older) and val (newest valFrac)
+// by CapturedAt, the time-split validation strategy: train on older labels,
+// evaluate on the most recent window, which is what serve actually faces.
+// Sort is stable; ties on CapturedAt break by EntryID for determinism.
+// valFrac is clamped: <=0 → all train, >=1 → all val. nVal is computed by
+// truncation, so a tiny set may yield an empty val side (callers guard).
+func SplitByTime(rows []LabelRow, valFrac float64) (train, val []LabelRow) {
+ if len(rows) == 0 {
+ return nil, nil
+ }
+ if valFrac <= 0 {
+ return rows, nil
+ }
+ if valFrac >= 1 {
+ return nil, rows
+ }
+ sorted := make([]LabelRow, len(rows))
+ copy(sorted, rows)
+ sort.SliceStable(sorted, func(i, j int) bool {
+ if sorted[i].CapturedAt.Equal(sorted[j].CapturedAt) {
+ return sorted[i].EntryID < sorted[j].EntryID
+ }
+ return sorted[i].CapturedAt.Before(sorted[j].CapturedAt)
+ })
+ nVal := int(float64(len(sorted)) * valFrac)
+ split := len(sorted) - nVal
+ return sorted[:split], sorted[split:]
+}
+
+// AppendLabels appends rows to path (created if needed). Rows already in
+// the file are never touched.
+func AppendLabels(path string, rows []LabelRow) error {
+ if len(rows) == 0 {
+ return nil
+ }
+ f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
+ if err != nil {
+ return fmt.Errorf("opening labels for append: %w", err)
+ }
+ defer func() { _ = f.Close() }()
+
+ enc := json.NewEncoder(f)
+ for _, row := range rows {
+ if err := enc.Encode(row); err != nil {
+ return fmt.Errorf("writing label row: %w", err)
+ }
+ }
+ return nil
+}
diff --git a/core/labels_test.go b/core/labels_test.go
new file mode 100644
index 0000000..d8abb43
--- /dev/null
+++ b/core/labels_test.go
@@ -0,0 +1,192 @@
+// Tests for labels.jsonl handling and URL normalization.
+package core
+
+import (
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+)
+
+func TestNormalizeURL(t *testing.T) {
+ cases := []struct{ in, want string }{
+ {"HTTPS://WWW.Example.com/Path/", "https://example.com/Path"},
+ {"https://example.com/a?x=1&y=2", "https://example.com/a?x=1&y=2"},
+ {"https://example.com", "https://example.com"},
+ {" https://EXAMPLE.com/ ", "https://example.com"},
+ {"https://www.www.example.com/x", "https://www.example.com/x"}, // only one leading www.
+ {"not a url", "not a url"},
+ }
+ for _, c := range cases {
+ if got := NormalizeURL(c.in); got != c.want {
+ t.Errorf("NormalizeURL(%q) = %q, want %q", c.in, got, c.want)
+ }
+ }
+}
+
+func sampleRow() LabelRow {
+ return LabelRow{
+ EntryID: 42, URL: "https://example.com/post", Title: "A title",
+ FeedID: 7, FeedTitle: "Feed", Label: LabelPos,
+ CapturedAt: time.Date(2024, 3, 1, 12, 0, 0, 0, time.UTC),
+ }
+}
+
+func TestReadLabelsMissingFile(t *testing.T) {
+ rows, err := ReadLabels(filepath.Join(t.TempDir(), "nope.jsonl"))
+ if err != nil {
+ t.Fatalf("missing file should not error: %v", err)
+ }
+ if len(rows) != 0 {
+ t.Fatalf("expected no rows, got %d", len(rows))
+ }
+}
+
+func TestAppendThenReadRoundTrip(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "labels.jsonl")
+ r1, r2 := sampleRow(), sampleRow()
+ r2.EntryID, r2.Label = 43, LabelNeg
+
+ if err := AppendLabels(path, []LabelRow{r1}); err != nil {
+ t.Fatalf("append 1: %v", err)
+ }
+ if err := AppendLabels(path, []LabelRow{r2}); err != nil {
+ t.Fatalf("append 2: %v", err)
+ }
+
+ rows, err := ReadLabels(path)
+ if err != nil {
+ t.Fatalf("read: %v", err)
+ }
+ if len(rows) != 2 || rows[0].EntryID != 42 || rows[1].Label != LabelNeg {
+ t.Fatalf("round trip mismatch: %+v", rows)
+ }
+}
+
+func TestAppendLabelsNoopOnEmpty(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "labels.jsonl")
+ if err := AppendLabels(path, nil); err != nil {
+ t.Fatalf("noop append: %v", err)
+ }
+ if _, err := os.Stat(path); !os.IsNotExist(err) {
+ t.Fatal("empty append should not create the file")
+ }
+}
+
+func TestSplitByTime(t *testing.T) {
+ mk := func(id int64, day int, label string) LabelRow {
+ return LabelRow{
+ EntryID: id, URL: "https://example.com/" + fmt.Sprint(id),
+ Title: "T", FeedID: 1, FeedTitle: "F", Label: label,
+ CapturedAt: time.Date(2024, 1, day, 0, 0, 0, 0, time.UTC),
+ }
+ }
+ // 10 rows, days 1..10. val-frac 0.3 → newest 3 (days 8,9,10) are val.
+ rows := []LabelRow{
+ mk(1, 1, LabelPos), mk(2, 2, LabelNeg), mk(3, 3, LabelPos),
+ mk(4, 4, LabelNeg), mk(5, 5, LabelPos), mk(6, 6, LabelNeg),
+ mk(7, 7, LabelPos), mk(8, 8, LabelNeg), mk(9, 9, LabelPos), mk(10, 10, LabelNeg),
+ }
+ train, val := SplitByTime(rows, 0.3)
+ if len(val) != 3 {
+ t.Fatalf("val size = %d, want 3", len(val))
+ }
+ if len(train) != 7 {
+ t.Fatalf("train size = %d, want 7", len(train))
+ }
+ // Val must be the newest (days 8,9,10).
+ wantValDays := map[int]bool{8: true, 9: true, 10: true}
+ for _, r := range val {
+ d := r.CapturedAt.Day()
+ if !wantValDays[d] {
+ t.Errorf("val contains day %d, expected 8/9/10", d)
+ }
+ }
+ // Train must be the older days 1..7.
+ for _, r := range train {
+ if r.CapturedAt.Day() > 7 {
+ t.Errorf("train contains day %d, expected <=7", r.CapturedAt.Day())
+ }
+ }
+}
+
+func TestSplitByTimeEdgeCases(t *testing.T) {
+ rows := []LabelRow{
+ {EntryID: 1, CapturedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)},
+ {EntryID: 2, CapturedAt: time.Date(2024, 1, 2, 0, 0, 0, 0, time.UTC)},
+ }
+ if train, val := SplitByTime(rows, 0); len(train) != 2 || len(val) != 0 {
+ t.Errorf("valFrac 0: train=%d val=%d, want 2/0", len(train), len(val))
+ }
+ if train, val := SplitByTime(rows, 1); len(train) != 0 || len(val) != 2 {
+ t.Errorf("valFrac 1: train=%d val=%d, want 0/2", len(train), len(val))
+ }
+ if train, val := SplitByTime(nil, 0.3); len(train) != 0 || len(val) != 0 {
+ t.Errorf("empty: train=%d val=%d, want 0/0", len(train), len(val))
+ }
+}
+
+func TestSplitByTimeTieBreakDeterministic(t *testing.T) {
+ // Same CapturedAt, different EntryIDs: tie broken by EntryID ascending.
+ day := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)
+ rows := []LabelRow{
+ {EntryID: 30, CapturedAt: day}, {EntryID: 10, CapturedAt: day}, {EntryID: 20, CapturedAt: day},
+ }
+ // Two runs must produce identical splits.
+ train1, val1 := SplitByTime(rows, 0.34)
+ _, val2 := SplitByTime(rows, 0.34)
+ if len(val1) != len(val2) {
+ t.Fatalf("non-deterministic val size: %d vs %d", len(val1), len(val2))
+ }
+ for i := range val1 {
+ if val1[i].EntryID != val2[i].EntryID {
+ t.Fatalf("non-deterministic tie break at %d: %d vs %d", i, val1[i].EntryID, val2[i].EntryID)
+ }
+ }
+ // Val (newest) should carry the highest EntryIDs.
+ if len(val1) > 0 {
+ maxTrain := int64(0)
+ for _, r := range train1 {
+ if r.EntryID > maxTrain {
+ maxTrain = r.EntryID
+ }
+ }
+ for _, r := range val1 {
+ if r.EntryID <= maxTrain {
+ t.Errorf("val id %d not greater than max train id %d (tie break ascending)", r.EntryID, maxTrain)
+ }
+ }
+ }
+}
+
+func TestReadLabelsStrictErrors(t *testing.T) {
+ cases := []struct {
+ name, body, wantSub string
+ }{
+ {"bad json", `{not json`, "invalid JSON"},
+ {"bad id", `{"entry_id":0,"url":"https://x","label":"pos","captured_at":"2024-01-01T00:00:00Z"}`, "entry_id"},
+ {"no url", `{"entry_id":1,"label":"pos","captured_at":"2024-01-01T00:00:00Z"}`, "url"},
+ {"bad label", `{"entry_id":1,"url":"https://x","label":"maybe","captured_at":"2024-01-01T00:00:00Z"}`, "label"},
+ {"no captured_at", `{"entry_id":1,"url":"https://x","label":"pos"}`, "captured_at"},
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "labels.jsonl")
+ if err := os.WriteFile(path, []byte(c.body+"\n"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ _, err := ReadLabels(path)
+ if err == nil {
+ t.Fatal("expected error")
+ }
+ if !strings.Contains(err.Error(), c.wantSub) {
+ t.Errorf("error %q should mention %q", err, c.wantSub)
+ }
+ if !strings.Contains(err.Error(), ":1") {
+ t.Errorf("error %q should carry the line number", err)
+ }
+ })
+ }
+}
diff --git a/core/ml.go b/core/ml.go
new file mode 100644
index 0000000..8a29f9f
--- /dev/null
+++ b/core/ml.go
@@ -0,0 +1,435 @@
+// Vendored from scholscan/core/ml.go (rev 7c1a5ef, 2026-08-28).
+// Owned here; no upstream sync.
+// ML implementation: TF-IDF + Logistic Regression for article filtering.
+//
+// Why title-only: Avoids content scraping overhead, titles are already informative.
+// MinDF=2: Removes typos and rare terms that don't generalize.
+// MaxDF=0.8: Removes common words that appear in >80% of documents.
+// λ=0.001: Light L2 regularization to prevent overfitting on small datasets.
+//
+// Public API:
+// - TFIDFVectorizer.Fit(): Learn vocabulary from documents
+// - TFIDFVectorizer.Transform(): Convert documents to TF-IDF vectors
+// - LogisticRegression.Fit(): Train classifier on vectors
+// - CreateVectorizerFromModel(): Reconstruct vectorizer from saved model
+// - PredictScore(): Score article using trained weights
+package core
+
+import (
+ "fmt"
+ "math"
+ "regexp"
+ "sort"
+ "strings"
+)
+
+// ============================================================================
+// ╻ ╻┏━╸┏━╸╺┳╸┏━┓┏━┓╻┏━┓┏━╸┏━┓
+// ┃┏┛┣╸ ┃ ┃ ┃ ┃┣┳┛┃┗━┓┣╸ ┣┳┛
+// ┗┛ ┗━╸┗━╸ ╹ ┗━┛╹┗╸╹┗━┛┗━╸╹┗╸
+// ============================================================================
+
+var wordHyphenRegex = regexp.MustCompile("[^a-zA-Z0-9-]+")
+
+// StopWords: Common words that don't help distinguish articles.
+// Why: Reduces noise and improves model generalization.
+var stopWords = map[string]struct{}{
+ // Single letters and symbols
+ "s": {}, "-": {}, "0": {}, "1": {}, "2": {}, "3": {}, "4": {}, "5": {}, "6": {}, "7": {}, "8": {}, "9": {},
+
+ // Common English stop words
+ "a": {}, "about": {}, "above": {}, "after": {}, "again": {}, "against": {}, "al": {}, "all": {}, "am": {}, "an": {}, "and": {}, "any": {}, "are": {}, "aren't": {}, "as": {}, "at": {}, "be": {}, "because": {}, "been": {}, "before": {}, "being": {}, "below": {}, "between": {}, "both": {}, "but": {}, "by": {}, "can't": {}, "cannot": {}, "could": {}, "couldn't": {}, "did": {}, "didn't": {}, "do": {}, "does": {}, "doesn't": {}, "doing": {}, "don't": {}, "down": {}, "during": {}, "each": {}, "et": {}, "few": {}, "for": {}, "from": {}, "further": {}, "had": {}, "hadn't": {}, "has": {}, "hasn't": {}, "have": {}, "haven't": {}, "having": {}, "he": {}, "he'd": {}, "he'll": {}, "he's": {}, "her": {}, "here": {}, "here's": {}, "hers": {}, "herself": {}, "him": {}, "himself": {}, "his": {}, "how": {}, "how's": {}, "i": {}, "i'd": {}, "i'll": {}, "i'm": {}, "i've": {}, "if": {}, "in": {}, "into": {}, "is": {}, "isn't": {}, "it": {}, "it's": {}, "its": {}, "itself": {}, "let's": {}, "me": {}, "more": {}, "most": {}, "mustn't": {}, "my": {}, "myself": {}, "no": {}, "nor": {}, "not": {}, "of": {}, "off": {}, "on": {}, "once": {}, "only": {}, "or": {}, "other": {}, "ought": {}, "our": {}, "ours": {}, "ourselves": {}, "out": {}, "over": {}, "own": {}, "same": {}, "shan't": {}, "she": {}, "she'd": {}, "she'll": {}, "she's": {}, "should": {}, "shouldn't": {}, "so": {}, "some": {}, "such": {}, "than": {}, "that": {}, "that's": {}, "the": {}, "their": {}, "theirs": {}, "them": {}, "themselves": {}, "then": {}, "there": {}, "there's": {}, "these": {}, "they": {}, "they'd": {}, "they'll": {}, "they're": {}, "they've": {}, "this": {}, "those": {}, "through": {}, "to": {}, "too": {}, "under": {}, "until": {}, "up": {}, "very": {}, "was": {}, "wasn't": {}, "we": {}, "we'd": {}, "we'll": {}, "we're": {}, "we've": {}, "were": {}, "weren't": {}, "what": {}, "what's": {}, "when": {}, "when's": {}, "where": {}, "where's": {}, "which": {}, "while": {}, "who": {}, "who's": {}, "whom": {}, "why": {}, "why's": {}, "with": {}, "won't": {}, "would": {}, "wouldn't": {}, "you": {}, "you'd": {}, "you'll": {}, "you're": {}, "you've": {}, "your": {}, "yours": {}, "yourself": {}, "yourselves": {},
+}
+
+type TFIDFVectorizer struct {
+ Vocabulary map[string]float64
+ OrderedVocab []string
+ NgramMin int
+ NgramMax int
+ MinDF int // Minimum document frequency (absolute)
+ MaxDF float64 // Maximum document frequency (ratio)
+ VocabCap int
+}
+
+func CreateVectorizerFromModel(model *ModelEnvelope) *TFIDFVectorizer {
+ v := &TFIDFVectorizer{
+ Vocabulary: model.Vectorizer,
+ OrderedVocab: model.OrderedVocab,
+ }
+ // Restore the n-gram range recorded at fit time so scoring text is
+ // vectorized exactly as training/validation saw it (zero values would
+ // silently degrade bigram models to unigram scoring). Absent meta —
+ // older or hand-built models — leaves zeros, i.e. unigrams.
+ if params, ok := model.Meta["vectorizer_params"].(map[string]any); ok {
+ if n, ok := params["ngram_min"].(float64); ok {
+ v.NgramMin = int(n)
+ }
+ if n, ok := params["ngram_max"].(float64); ok {
+ v.NgramMax = int(n)
+ }
+ }
+ return v
+}
+
+// Learns vocabulary and IDF from documents
+func (v *TFIDFVectorizer) Fit(documents []string) {
+ numDocs := len(documents)
+ docFreqs := make(map[string]int)
+
+ for _, doc := range documents {
+ unigrams := Tokenize(doc)
+ ngrams := generateNgrams(unigrams, v.NgramMin, v.NgramMax)
+ seenInDoc := make(map[string]struct{})
+ for _, ngram := range ngrams {
+ if _, seen := seenInDoc[ngram]; !seen {
+ docFreqs[ngram]++
+ seenInDoc[ngram] = struct{}{}
+ }
+ }
+ }
+
+ maxDocs := int(v.MaxDF * float64(numDocs))
+ filteredVocab := make(map[string]int)
+ for term, freq := range docFreqs {
+ if freq >= v.MinDF && freq <= maxDocs {
+ filteredVocab[term] = freq
+ }
+ }
+
+ if v.VocabCap > 0 && len(filteredVocab) > v.VocabCap {
+ type termFreq struct {
+ term string
+ freq int
+ }
+ terms := make([]termFreq, 0, len(filteredVocab))
+ for term, freq := range filteredVocab {
+ terms = append(terms, termFreq{term, freq})
+ }
+ sort.Slice(terms, func(i, j int) bool {
+ return terms[i].freq > terms[j].freq
+ })
+
+ cappedTerms := terms[:v.VocabCap]
+ filteredVocab = make(map[string]int, v.VocabCap)
+ for _, tf := range cappedTerms {
+ filteredVocab[tf.term] = tf.freq
+ }
+ }
+
+ v.OrderedVocab = make([]string, 0, len(filteredVocab))
+ for term := range filteredVocab {
+ v.OrderedVocab = append(v.OrderedVocab, term)
+ }
+ sort.Strings(v.OrderedVocab) // deterministic order
+
+ v.Vocabulary = make(map[string]float64, len(v.OrderedVocab))
+ for _, term := range v.OrderedVocab {
+ // IDF = log(total num of docs / num of docs with term)
+ idf := math.Log(float64(numDocs) / float64(filteredVocab[term]))
+ v.Vocabulary[term] = idf
+ }
+}
+
+// Converts documents to TF-IDF vectors using learned vocabulary
+func (v *TFIDFVectorizer) Transform(documents []string) [][]float64 {
+ vectors := make([][]float64, len(documents))
+
+ for i, doc := range documents {
+ unigrams := Tokenize(doc)
+ ngrams := generateNgrams(unigrams, v.NgramMin, v.NgramMax)
+ vector := make([]float64, len(v.OrderedVocab))
+
+ if len(ngrams) > 0 {
+ // tf: term frequency (normalized count of each n-gram in document)
+ tf := make(map[string]float64)
+ for _, ngram := range ngrams {
+ tf[ngram]++
+ }
+ numNgrams := float64(len(ngrams))
+ for ngram, count := range tf {
+ tf[ngram] = count / numNgrams
+ }
+
+ for j, term := range v.OrderedVocab {
+ if tfValue, ok := tf[term]; ok {
+ // only score terms that were in our training vocabulary
+ if idfValue, inVocab := v.Vocabulary[term]; inVocab {
+ vector[j] = tfValue * idfValue
+ }
+ }
+ }
+ }
+ vectors[i] = vector
+ }
+
+ return vectors
+}
+
+func Tokenize(text string) []string {
+ text = strings.ToLower(text)
+ words := wordHyphenRegex.Split(text, -1)
+ tokens := make([]string, 0, len(words))
+ for _, word := range words {
+ if word == "" {
+ continue
+ }
+ if _, isStopWord := stopWords[word]; isStopWord {
+ continue
+ }
+ tokens = append(tokens, word)
+ }
+ return tokens
+}
+
+func generateNgrams(tokens []string, minN, maxN int) []string {
+ if minN <= 0 {
+ minN = 1
+ }
+ if maxN < minN {
+ maxN = minN
+ }
+
+ numTokens := len(tokens)
+
+ estimatedCap := 0
+ for n := minN; n <= maxN; n++ {
+ if numTokens >= n {
+ estimatedCap += numTokens - n + 1
+ }
+ }
+ ngrams := make([]string, 0, estimatedCap)
+
+ for n := minN; n <= maxN; n++ {
+ if numTokens < n {
+ continue
+ }
+ for i := 0; i <= numTokens-n; i++ {
+ ngrams = append(ngrams, strings.Join(tokens[i:i+n], " "))
+ }
+ }
+ return ngrams
+}
+
+// ============================================================================
+// ┏━╸╻ ┏━┓┏━┓┏━┓╻┏━╸╻┏━╸┏━┓
+// ┃ ┃ ┣━┫┗━┓┗━┓┃┣╸ ┃┣╸ ┣┳┛
+// ┗━╸┗━╸╹ ╹┗━┛┗━┛╹╹ ╹┗━╸╹┗╸
+// ============================================================================
+
+// Binary logistic regression with L2 regularization
+// Bias term stored separately (not regularized)
+type LogisticRegression struct {
+ LearningRate float64
+ Lambda float64 // L2 regularization parameter
+ Iterations int
+ Tolerance float64 // Convergence tolerance on loss improvement
+}
+
+// validate checks and clamps hyperparams to reasonable bounds.
+func (lr *LogisticRegression) Validate() *LogisticRegression {
+ const (
+ defaultLearningRate = 0.5
+ defaultIterations = 500
+ defaultTolerance = 0.000001
+ )
+
+ if lr.LearningRate <= 0 {
+ lr.LearningRate = defaultLearningRate
+ }
+ if lr.LearningRate > 10 {
+ lr.LearningRate = 10.0
+ }
+ if lr.Lambda < 0 {
+ lr.Lambda = 0.0
+ }
+ if lr.Iterations <= 0 {
+ lr.Iterations = defaultIterations
+ }
+ if lr.Tolerance <= 0 {
+ lr.Tolerance = defaultTolerance
+ }
+ return lr
+}
+
+// Fit trains via SGD with L2 regularization on feature weights (not bias).
+// Class weights reweight samples; unused in our pipeline (we downsample instead).
+// Returns weights with bias as last element.
+func (lr *LogisticRegression) Fit(vectors [][]float64, labels []float64, classWeights map[float64]float64) ([]float64, error) {
+ if len(vectors) == 0 {
+ return nil, fmt.Errorf("cannot train on empty dataset")
+ }
+ if len(vectors) != len(labels) {
+ return nil, fmt.Errorf(
+ "mismatch between number of vectors (%d) and labels (%d)",
+ len(vectors),
+ len(labels),
+ )
+ }
+
+ for i, y := range labels {
+ if y != 0 && y != 1 {
+ return nil, fmt.Errorf("invalid label at %d: %v (expected 0 or 1)", i, y)
+ }
+ }
+
+ numFeatures := len(vectors[0])
+ if numFeatures == 0 {
+ return nil, fmt.Errorf("cannot train with zero-length feature vectors")
+ }
+ for i := 1; i < len(vectors); i++ {
+ if len(vectors[i]) != numFeatures {
+ return nil, fmt.Errorf(
+ "inconsistent feature vector length at index %d: got %d, expected %d",
+ i,
+ len(vectors[i]),
+ numFeatures,
+ )
+ }
+ }
+ useUniformWeights := classWeights == nil
+ if useUniformWeights {
+ classWeights = map[float64]float64{0.0: 1.0, 1.0: 1.0}
+ }
+
+ numSamples := float64(len(vectors))
+ var totalWeight float64
+ if useUniformWeights {
+ totalWeight = numSamples
+ } else {
+ for _, y := range labels {
+ totalWeight += classWeights[y]
+ }
+ }
+ if totalWeight == 0 {
+ totalWeight = numSamples // Fallback
+ }
+
+ weights := make([]float64, numFeatures)
+ var bias float64
+
+ prevLoss := math.MaxFloat64
+
+ for i := 0; i < lr.Iterations; i++ {
+ gradWeights := make([]float64, numFeatures)
+ var gradBias float64
+ var currentLoss float64
+
+ for j, x := range vectors {
+ y := labels[j]
+ sampleWeight := classWeights[y]
+
+ z, err := dot(weights, x)
+ if err != nil {
+ return nil, fmt.Errorf("error calculating dot product for vector %d: %w", j, err)
+ }
+ p := Sigmoid(z + bias)
+
+ // Compute prediction error. This term gets multiplied by each feature value
+ // to accumulate gradients (higher error pushes weights harder).
+ errTerm := p - y
+ for k := 0; k < numFeatures; k++ {
+ gradWeights[k] += sampleWeight * errTerm * x[k]
+ }
+ gradBias += sampleWeight * errTerm
+
+ cp := clamp(p)
+ currentLoss += sampleWeight * (-(y*math.Log(cp) + (1-y)*math.Log(1-cp)))
+ }
+
+ // Update weights with L2 regularization (only on feature weights, not bias).
+ // This pulls weights toward zero, preventing overfitting on small datasets.
+ for k := 0; k < numFeatures; k++ {
+ regularizedGrad := (gradWeights[k] / totalWeight) + (lr.Lambda * weights[k])
+ weights[k] -= lr.LearningRate * regularizedGrad
+ }
+ gradBias /= totalWeight
+ bias -= lr.LearningRate * gradBias
+
+ // Check convergence: if loss change is below tolerance, we're done.
+ // We include the L2 penalty in total loss to assess true convergence.
+ avgLoss := currentLoss / totalWeight
+ var l2Penalty float64
+ for _, w := range weights {
+ l2Penalty += w * w
+ }
+ totalLoss := avgLoss + 0.5*lr.Lambda*l2Penalty
+ if math.Abs(prevLoss-totalLoss) < lr.Tolerance {
+ break
+ }
+ prevLoss = totalLoss
+ }
+
+ // bias is stored as the last element
+ return append(weights, bias), nil
+}
+
+// PredictScore computes the probability for a single vec given weights.
+// the last element of weights is the bias.
+func PredictScore(vector []float64, weights []float64) (float64, error) {
+ if len(weights) == 0 {
+ return 0, fmt.Errorf("weights cannot be empty")
+ }
+ if len(vector) != len(weights)-1 {
+ return 0, fmt.Errorf(
+ "vector length mismatch: expected %d features, got %d",
+ len(weights)-1,
+ len(vector),
+ )
+ }
+
+ for i, v := range vector {
+ if math.IsNaN(v) || math.IsInf(v, 0) {
+ return 0, fmt.Errorf("invalid value at vector[%d]: %v", i, v)
+ }
+ }
+ for i, w := range weights {
+ if math.IsNaN(w) || math.IsInf(w, 0) {
+ return 0, fmt.Errorf("invalid value at weights[%d]: %v", i, w)
+ }
+ }
+
+ featureWeights := weights[:len(weights)-1]
+ bias := weights[len(weights)-1]
+
+ z, err := dot(featureWeights, vector)
+ if err != nil {
+ return 0, fmt.Errorf("failed to compute dot product: %w", err)
+ }
+ return Sigmoid(z + bias), nil
+}
+
+// ============================================================================
+// ┏┳┓┏━┓╺┳╸╻ ╻┏━┓
+// ┃┃┃┣━┫ ┃ ┣━┫┗━┓
+// ╹ ╹╹ ╹ ╹ ╹ ╹┗━┛
+// ============================================================================
+
+func Sigmoid(z float64) float64 {
+ if z >= 0 {
+ return 1.0 / (1.0 + math.Exp(-z))
+ }
+ ez := math.Exp(z)
+ return ez / (1.0 + ez)
+}
+
+func dot(a, b []float64) (float64, error) {
+ if len(a) != len(b) {
+ return 0, fmt.Errorf("vector length mismatch: %d != %d", len(a), len(b))
+ }
+ var sum float64
+ for i := range a {
+ sum += a[i] * b[i]
+ }
+ return sum, nil
+}
+
+func clamp(p float64) float64 {
+ const probabilityClamp = 1e-15
+ if p < probabilityClamp {
+ return probabilityClamp
+ }
+ if p > 1.0-probabilityClamp {
+ return 1.0 - probabilityClamp
+ }
+ return p
+}
diff --git a/core/ml_test.go b/core/ml_test.go
new file mode 100644
index 0000000..3623b39
--- /dev/null
+++ b/core/ml_test.go
@@ -0,0 +1,76 @@
+// Roundtrip test for the vendored TF-IDF + LR pipeline: fit a tiny model,
+// confirm scores separate the two synthetic topics. Guards against a broken
+// copy more than against algorithm regressions.
+package core
+
+import (
+ "testing"
+)
+
+func TestTFIDFLRRoundtrip(t *testing.T) {
+ // Two separable synthetic "topics": optics vs cooking.
+ docs := []string{
+ "quantum laser optics photon",
+ "laser photon quantum measurement",
+ "photon optics quantum laser",
+ "optics quantum laser photon",
+ "recipe cooking knife onion",
+ "onion recipe cooking garlic",
+ "cooking garlic onion skillet",
+ "garlic recipe onion cooking",
+ }
+ labels := []float64{1, 1, 1, 1, 0, 0, 0, 0}
+
+ v := &TFIDFVectorizer{NgramMin: 1, NgramMax: 2, MinDF: 1, MaxDF: 0.9}
+ v.Fit(docs)
+ if len(v.OrderedVocab) == 0 {
+ t.Fatal("vectorizer learned empty vocabulary")
+ }
+
+ vecs := v.Transform(docs)
+ lr := (&LogisticRegression{Lambda: 0.001}).Validate()
+ weights, err := lr.Fit(vecs, labels, nil)
+ if err != nil {
+ t.Fatalf("Fit: %v", err)
+ }
+
+ // Persist and reload through the envelope, as serve does.
+ model := &ModelEnvelope{
+ Vectorizer: v.Vocabulary,
+ OrderedVocab: v.OrderedVocab,
+ Weights: weights,
+ }
+ rv := CreateVectorizerFromModel(model)
+
+ opticsScore := mustScore(t, rv, model, "quantum photon laser experiment")
+ cookingScore := mustScore(t, rv, model, "onion garlic skillet dinner")
+ if !(opticsScore > 0.5 && cookingScore < 0.5) {
+ t.Fatalf("expected separation, got optics=%.3f cooking=%.3f", opticsScore, cookingScore)
+ }
+ if opticsScore <= cookingScore {
+ t.Fatalf("expected optics score above cooking score, got %.3f <= %.3f", opticsScore, cookingScore)
+ }
+}
+
+func mustScore(t *testing.T, v *TFIDFVectorizer, m *ModelEnvelope, title string) float64 {
+ t.Helper()
+ vec := v.Transform([]string{title})
+ s, err := PredictScore(vec[0], m.Weights)
+ if err != nil {
+ t.Fatalf("PredictScore(%q): %v", title, err)
+ }
+ return s
+}
+
+func TestTokenize(t *testing.T) {
+ got := Tokenize("The Fall of the House of Usher!")
+ want := []string{"fall", "house", "usher"}
+ if len(got) != len(want) {
+ t.Fatalf("Tokenize: got %v, want %v", got, want)
+ }
+ for i := range want {
+ if got[i] != want[i] {
+ t.Fatalf("Tokenize: got %v, want %v", got, want)
+ }
+ }
+}
diff --git a/core/model.go b/core/model.go
new file mode 100644
index 0000000..4afcdae
--- /dev/null
+++ b/core/model.go
@@ -0,0 +1,22 @@
+// Vendored from scholscan/core/model.go (rev 7c1a5ef, 2026-08-28).
+// Owned here; no upstream sync.
+// Model envelope persists trained model to JSON. Contains Vectorizer for IDF values,
+// OrderedVocab for feature ordering, and Weights for logistic regression.
+// To score: recreate TFIDFVectorizer, transform, then PredictScore.
+package core
+
+import (
+ "time"
+)
+
+// ModelEnvelope - complete trained model for scoring articles
+type ModelEnvelope struct {
+ Algorithm string `json:"algorithm"`
+ Impl string `json:"impl"`
+ Version string `json:"version"`
+ CreatedAt time.Time `json:"created_at"`
+ Meta map[string]any `json:"meta"`
+ Vectorizer map[string]float64 `json:"vectorizer"`
+ OrderedVocab []string `json:"ordered_vocab"`
+ Weights []float64 `json:"weights"`
+}
diff --git a/core/runs.go b/core/runs.go
new file mode 100644
index 0000000..38e6710
--- /dev/null
+++ b/core/runs.go
@@ -0,0 +1,167 @@
+// runs.jsonl read/write: the serve-side run log. One JSON RunRow appended
+// per non-empty poll batch; kept items double as the RSS render source, so
+// each carries everything a feed item needs (title/link/date/cleaned
+// content). Rotation is generous: past maxRunsBytes the file is rewritten
+// keeping the newest rows, via temp + rename so a crash mid-rotate can't
+// leave a torn log behind.
+package core
+
+import (
+ "bufio"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "time"
+)
+
+// Kept-item kinds. Short strings so runs.jsonl stays human-greppable.
+const (
+ KindRanked = "ranked"
+ KindRunnerUp = "runnerup"
+ KindRandom = "random"
+)
+
+// Rotation knobs. 20MB of runs is years of batches at expected sizes, and
+// 500 retained rows is still far more than the count-based RSS tail (~21)
+// will ever render.
+const (
+ maxRunsBytes = 20 << 20
+ keepRunsOnRotate = 500
+)
+
+// KeptItem is one entry selected into a digest batch, with everything the
+// RSS renderer needs so rendering never touches Miniflux.
+type KeptItem struct {
+ EntryID int64 `json:"entry_id"`
+ URL string `json:"url"`
+ Title string `json:"title"`
+ FeedTitle string `json:"feed_title"`
+ PublishedAt time.Time `json:"published_at"`
+ Content string `json:"content"` // cleaned at poll time: HTML-stripped, <=5KB
+ Score float64 `json:"score"`
+ Kind string `json:"kind"` // KindRanked | KindRunnerUp | KindRandom
+}
+
+// RunRow is one poll batch: what was seen, what was kept, when, and how
+// old the model was. EntriesSeen counts every fetched entry including any
+// skipped before scoring (see RecFeedSkipped).
+type RunRow struct {
+ RanAt time.Time `json:"ran_at"`
+ EntriesSeen int `json:"entries_seen"`
+ RecFeedSkipped int `json:"rec_feed_skipped,omitempty"`
+ Kept []KeptItem `json:"kept"`
+ DurationMs int64 `json:"duration_ms"`
+ ModelCreatedAt time.Time `json:"model_created_at"`
+}
+
+// ReadRuns parses all of path strictly. A missing file is not an error —
+// it just means no runs logged yet (first serve boot).
+func ReadRuns(path string) ([]RunRow, error) {
+ f, err := os.Open(path)
+ if err != nil {
+ if errors.Is(err, os.ErrNotExist) {
+ return nil, nil
+ }
+ return nil, fmt.Errorf("opening runs: %w", err)
+ }
+ defer func() { _ = f.Close() }()
+
+ var rows []RunRow
+ scanner := bufio.NewScanner(f)
+ scanner.Buffer(make([]byte, 64*1024), 4*1024*1024)
+ line := 0
+ for scanner.Scan() {
+ line++
+ text := strings.TrimSpace(scanner.Text())
+ if text == "" {
+ continue
+ }
+ var row RunRow
+ if err := json.Unmarshal([]byte(text), &row); err != nil {
+ return nil, fmt.Errorf("%s:%d: invalid JSON: %w", path, line, err)
+ }
+ if row.RanAt.IsZero() {
+ return nil, fmt.Errorf("%s:%d: ran_at is required", path, line)
+ }
+ for i, k := range row.Kept {
+ if k.EntryID <= 0 {
+ return nil, fmt.Errorf("%s:%d: kept[%d]: entry_id must be positive", path, line, i)
+ }
+ if k.Kind != KindRanked && k.Kind != KindRunnerUp && k.Kind != KindRandom {
+ return nil, fmt.Errorf("%s:%d: kept[%d]: invalid kind %q", path, line, i, k.Kind)
+ }
+ }
+ rows = append(rows, row)
+ }
+ if err := scanner.Err(); err != nil {
+ return nil, fmt.Errorf("reading %s: %w", path, err)
+ }
+ return rows, nil
+}
+
+// ReadRunsTail returns the newest k runs in file order (oldest to newest).
+// k <= 0 reads everything.
+func ReadRunsTail(path string, k int) ([]RunRow, error) {
+ rows, err := ReadRuns(path)
+ if err != nil {
+ return nil, err
+ }
+ if k > 0 && len(rows) > k {
+ rows = rows[len(rows)-k:]
+ }
+ return rows, nil
+}
+
+// AppendRun appends one row, then rotates the file if it has grown past
+// maxRunsBytes. Appending is the only mutation; existing rows are never
+// edited, only dropped by rotation.
+func AppendRun(path string, row RunRow) error {
+ f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
+ if err != nil {
+ return fmt.Errorf("opening runs for append: %w", err)
+ }
+ if err := json.NewEncoder(f).Encode(row); err != nil {
+ _ = f.Close()
+ return fmt.Errorf("writing run row: %w", err)
+ }
+ if err := f.Close(); err != nil {
+ return fmt.Errorf("closing runs: %w", err)
+ }
+
+ info, err := os.Stat(path)
+ if err != nil {
+ return fmt.Errorf("stat runs: %w", err)
+ }
+ if info.Size() <= maxRunsBytes {
+ return nil
+ }
+ rows, err := ReadRuns(path)
+ if err != nil {
+ return err
+ }
+ if len(rows) > keepRunsOnRotate {
+ rows = rows[len(rows)-keepRunsOnRotate:]
+ }
+ tmp := filepath.Clean(path) + ".tmp"
+ w, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
+ if err != nil {
+ return fmt.Errorf("rotating runs: %w", err)
+ }
+ enc := json.NewEncoder(w)
+ for _, r := range rows {
+ if err := enc.Encode(r); err != nil {
+ _ = w.Close()
+ return fmt.Errorf("rotating runs: %w", err)
+ }
+ }
+ if err := w.Close(); err != nil {
+ return fmt.Errorf("rotating runs: %w", err)
+ }
+ if err := os.Rename(tmp, path); err != nil {
+ return fmt.Errorf("rotating runs: %w", err)
+ }
+ return nil
+}
diff --git a/core/scoring.go b/core/scoring.go
new file mode 100644
index 0000000..66b3a3b
--- /dev/null
+++ b/core/scoring.go
@@ -0,0 +1,16 @@
+// Vendored from scholscan/core/scoring.go (rev 7c1a5ef, 2026-08-28).
+// Owned here; no upstream sync.
+// Score conversion utilities.
+//
+// ScoreToScale: Maps probability (0-1) to user-friendly 1-10 scale.
+// Why: Users understand "8/10" better than "0.82 probability".
+package core
+
+import "math"
+
+// ScoreToScale turns probability into 1-10 display score
+func ScoreToScale(rawScore, threshold float64) int {
+ k := 10.0
+ adjustedScore := 1.0 / (1.0 + math.Exp(-k*(rawScore-threshold)))
+ return int(math.Round(1.0 + (adjustedScore * 9.0)))
+}
diff --git a/core/scoring_text.go b/core/scoring_text.go
new file mode 100644
index 0000000..84cd758
--- /dev/null
+++ b/core/scoring_text.go
@@ -0,0 +1,43 @@
+// Scoring-text construction: the document fed to the vectorizer.
+//
+// fluxrec is title-only by design (no body/excerpt ever enters the scoring
+// text). On top of the raw title we inject one synthetic feed-identity token
+// `feed-<slug>` built from the entry's feed_title, so LR can learn a per-feed
+// prior inside the existing TF-IDF + LR model. MinDF shrinks low-data feeds
+// toward zero for free (a feed with <2 labeled entries never enters the
+// vocabulary), so no separate shrinkage-prior machinery.
+//
+// Hyphens, not underscores: the tokenizer keeps [a-zA-Z0-9-] and splits on
+// everything else, so `feed-marginal-revolution` survives as ONE token while
+// `feed_marginal_revolution` would fragment into three. Identical slug
+// construction at train (from labels.jsonl feed_title) and serve (from the
+// entry's feed.title) keeps the feature consistent across both sides.
+package core
+
+import (
+ "regexp"
+ "strings"
+)
+
+var slugSep = regexp.MustCompile(`[^a-z0-9]+`)
+
+// FeedSlug normalizes a feed title into a stable slug of lowercase
+// letters/digits joined by single hyphens, trimmed. Empty/whitespace input
+// returns "". Idempotent: slug(slug(x)) == slug(x).
+func FeedSlug(feedTitle string) string {
+ s := strings.ToLower(strings.TrimSpace(feedTitle))
+ s = slugSep.ReplaceAllString(s, "-")
+ return strings.Trim(s, "-")
+}
+
+// ScoringText builds the document fed to the vectorizer: the entry title plus
+// one synthetic token "feed-<slug>". If the feed title is empty (no slug), the
+// token is omitted and the title alone is returned. No body/excerpt is ever
+// added — title-only is the design.
+func ScoringText(title, feedTitle string) string {
+ slug := FeedSlug(feedTitle)
+ if slug == "" {
+ return title
+ }
+ return title + " feed-" + slug
+}
diff --git a/core/scoring_text_test.go b/core/scoring_text_test.go
new file mode 100644
index 0000000..87b3948
--- /dev/null
+++ b/core/scoring_text_test.go
@@ -0,0 +1,99 @@
+// Tests for feed-identity token and scoring-text construction.
+package core
+
+import (
+ "strings"
+ "testing"
+)
+
+func TestFeedSlug(t *testing.T) {
+ cases := []struct{ in, want string }{
+ {"Marginal Revolution", "marginal-revolution"},
+ {"ACM / IEEE", "acm-ieee"},
+ {" Already-Hyphenated ", "already-hyphenated"},
+ {"---", ""},
+ {"", ""},
+ {"MIT Tech Review #2", "mit-tech-review-2"},
+ {"Über Wörter", "ber-w-rter"}, // non-ascii stripped to hyphens
+ }
+ for _, c := range cases {
+ if got := FeedSlug(c.in); got != c.want {
+ t.Errorf("FeedSlug(%q) = %q, want %q", c.in, got, c.want)
+ }
+ }
+}
+
+func TestFeedSlugIdempotent(t *testing.T) {
+ for _, in := range []string{"Marginal Revolution", "ACM / IEEE", "X---Y", "a b c"} {
+ once := FeedSlug(in)
+ twice := FeedSlug(once)
+ if once != twice {
+ t.Errorf("slug not idempotent: %q → %q → %q", in, once, twice)
+ }
+ }
+}
+
+func TestScoringTextIncludesToken(t *testing.T) {
+ got := ScoringText("A Title", "Marginal Revolution")
+ if !strings.HasSuffix(got, " feed-marginal-revolution") {
+ t.Errorf("expected feed-identity token suffix, got %q", got)
+ }
+}
+
+func TestScoringTextEmptyFeedOmitsToken(t *testing.T) {
+ if got := ScoringText("A Title", ""); got != "A Title" {
+ t.Errorf("empty feed should yield title only, got %q", got)
+ }
+ if got := ScoringText("A Title", " --- "); got != "A Title" {
+ t.Errorf("whitespace-only feed should yield title only, got %q", got)
+ }
+}
+
+// The feed-identity token must round-trip through Tokenize as a SINGLE token
+// (hyphens survive; underscores would fragment). Same feed → shared token;
+// different feeds → different tokens.
+func TestFeedTokenRoundTripsAsSingleToken(t *testing.T) {
+ doc := ScoringText("some title here", "Marginal Revolution")
+ toks := Tokenize(doc)
+ found := false
+ for _, tk := range toks {
+ if tk == "feed-marginal-revolution" {
+ found = true
+ break
+ }
+ }
+ if !found {
+ t.Fatalf("feed token not present as a single token in %v", toks)
+ }
+ // And it must not have fragmented into feed/marginal/revolution.
+ for _, bad := range []string{"feed", "marginal", "revolution"} {
+ for _, tk := range toks {
+ if tk == bad {
+ t.Errorf("token fragmented: %q appeared standalone", bad)
+ }
+ }
+ }
+
+ // Same feed → shared token; different feed → different token.
+ a := ScoringText("x", "Marginal Revolution")
+ b := ScoringText("y", "Marginal Revolution")
+ c := ScoringText("z", "Other Blog")
+ tokA, tokC := feedToken(t, a), feedToken(t, c)
+ if feedToken(t, b) != tokA {
+ t.Error("same feed should share the token")
+ }
+ if tokA == tokC {
+ t.Error("different feeds should differ")
+ }
+}
+
+func feedToken(t *testing.T, doc string) string {
+ t.Helper()
+ for _, tk := range Tokenize(doc) {
+ if strings.HasPrefix(tk, "feed-") {
+ return tk
+ }
+ }
+ t.Fatalf("no feed token in %q", doc)
+ return ""
+}
diff --git a/core/text.go b/core/text.go
new file mode 100644
index 0000000..563abb3
--- /dev/null
+++ b/core/text.go
@@ -0,0 +1,38 @@
+// Vendored from scholscan/core/text.go (rev 7c1a5ef, 2026-08-28).
+// Owned here; no upstream sync.
+// Text processing for RSS feed content.
+// Used for web UI previews and search indexing - not ML (title-only scoring).
+package core
+
+import (
+ "regexp"
+ "strings"
+)
+
+// CleanFeedContent strips HTML, normalizes whitespace, truncates to 5KB
+func CleanFeedContent(content string) string {
+ if content == "" {
+ return ""
+ }
+
+ content = StripHTMLTags(content)
+ content = NormalizeSpace(content)
+
+ maxLength := 5000
+ if len(content) > maxLength {
+ content = content[:maxLength] + "..."
+ }
+
+ return content
+}
+
+// StripHTMLTags removes HTML tags
+func StripHTMLTags(content string) string {
+ re := regexp.MustCompile(`<[^>]*>`)
+ return re.ReplaceAllString(content, "")
+}
+
+// NormalizeSpace collapses whitespace and trims
+func NormalizeSpace(s string) string {
+ return strings.Join(strings.Fields(strings.TrimSpace(s)), " ")
+}
diff --git a/core/types.go b/core/types.go
new file mode 100644
index 0000000..073ee8f
--- /dev/null
+++ b/core/types.go
@@ -0,0 +1,59 @@
+// Core type definitions for fluxrec.
+//
+// Based on scholscan/core/types.go (vendored rev 7c1a5ef) but trimmed hard:
+// academic metadata (authors, journal, DOI, enrichment config) is out.
+// fluxrec's world is Miniflux entries, labels, and scores.
+package core
+
+import (
+ "io"
+ "time"
+)
+
+// Article is a title-carrying item moving through the pipeline
+// (Miniflux entry at serve time, scored candidate in the digest).
+type Article struct {
+ Title string `json:"title"`
+ Content string `json:"content,omitempty"`
+ URL string `json:"url"`
+ Source string `json:"source,omitempty"` // feed title
+ PublishedAt *time.Time `json:"published_at,omitempty"`
+ Score *float64 `json:"score,omitempty"`
+}
+
+// Label values for LabelRow.Label. Short strings so labels.jsonl stays
+// human-greppable.
+const (
+ LabelPos = "pos"
+ LabelNeg = "neg"
+)
+
+// LabelRow is one row of labels.jsonl, the canonical label store.
+// Written by `export`, read by `train`.
+//
+// EntryID is Miniflux's entry ID; for stars made on rec-feed items it resolves
+// to the *original* entry via the guid contract, so a row always describes the
+// original article.
+type LabelRow struct {
+ EntryID int64 `json:"entry_id"`
+ URL string `json:"url"`
+ Title string `json:"title"`
+ FeedID int64 `json:"feed_id"`
+ FeedTitle string `json:"feed_title"`
+ Label string `json:"label"` // LabelPos or LabelNeg
+ CapturedAt time.Time `json:"captured_at"`
+}
+
+// Command defines the interface that all CLI subcommands must implement.
+type Command interface {
+ // Name returns the command name (e.g., "export", "train").
+ Name() string
+
+ // Init parses command-line arguments and initializes the command.
+ // It should return flag.ErrHelp if --help was requested.
+ Init(args []string) error
+
+ // Run executes the command, reading from stdin and writing to stdout.
+ // The command should handle its own error reporting.
+ Run(stdin io.Reader, stdout io.Writer) error
+}
generated by cgit v1.2.3 (git 2.46.0) at 2026-08-31 08:46:41 +0000