1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
|
// labels.jsonl read/write and URL normalization.
//
// labels.jsonl is the canonical label store: one JSON LabelRow per line,
// append-only. ReadLabels is strict (per-line errors with line numbers);
// AppendLabels is the normal writer — rows are only rewritten in place by the
// two commands that deliberately do so, each backing the file up first:
// compact-labels (trimming per-feed negative skew) and export (relabeling a
// starred entry that was stored as a negative).
package core
import (
"bufio"
"encoding/json"
"errors"
"fmt"
"net/url"
"os"
"sort"
"strings"
)
// NormalizeURL collapses trivially-equivalent article links into one dedup
// key: scheme/host lowercased, leading "www." and trailing "/" dropped.
// Unparseable input degrades to a plain lowercase-trim.
func NormalizeURL(u string) string {
u = strings.TrimSpace(u)
pr, err := url.Parse(u)
if err != nil || pr.Host == "" {
return strings.TrimSuffix(strings.ToLower(u), "/")
}
pr.Scheme = strings.ToLower(pr.Scheme)
pr.Host = strings.TrimPrefix(strings.ToLower(pr.Host), "www.")
return strings.TrimSuffix(pr.String(), "/")
}
// ReadLabels parses path strictly. A missing file is not an error — it just
// means no labels yet (first export run).
func ReadLabels(path string) ([]LabelRow, error) {
f, err := os.Open(path)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil, nil
}
return nil, fmt.Errorf("opening labels: %w", err)
}
defer func() { _ = f.Close() }()
var rows []LabelRow
scanner := bufio.NewScanner(f)
scanner.Buffer(make([]byte, 64*1024), 4*1024*1024)
line := 0
for scanner.Scan() {
line++
text := strings.TrimSpace(scanner.Text())
if text == "" {
continue
}
var row LabelRow
if err := json.Unmarshal([]byte(text), &row); err != nil {
return nil, fmt.Errorf("%s:%d: invalid JSON: %w", path, line, err)
}
if row.EntryID <= 0 {
return nil, fmt.Errorf("%s:%d: entry_id must be positive", path, line)
}
if row.URL == "" {
return nil, fmt.Errorf("%s:%d: url is required", path, line)
}
if row.Label != LabelPos && row.Label != LabelNeg {
return nil, fmt.Errorf("%s:%d: label must be %q or %q, got %q", path, line, LabelPos, LabelNeg, row.Label)
}
if row.CapturedAt.IsZero() {
return nil, fmt.Errorf("%s:%d: captured_at is required", path, line)
}
rows = append(rows, row)
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("reading %s: %w", path, err)
}
return rows, nil
}
// SplitByTime partitions rows into train (older) and val (newest valFrac)
// by CapturedAt, the time-split validation strategy: train on older labels,
// evaluate on the most recent window, which is what serve actually faces.
// Sort is stable; ties on CapturedAt break by EntryID for determinism.
// valFrac is clamped: <=0 → all train, >=1 → all val. nVal is computed by
// truncation, so a tiny set may yield an empty val side (callers guard).
func SplitByTime(rows []LabelRow, valFrac float64) (train, val []LabelRow) {
if len(rows) == 0 {
return nil, nil
}
if valFrac <= 0 {
return rows, nil
}
if valFrac >= 1 {
return nil, rows
}
sorted := make([]LabelRow, len(rows))
copy(sorted, rows)
sort.SliceStable(sorted, func(i, j int) bool {
if sorted[i].CapturedAt.Equal(sorted[j].CapturedAt) {
return sorted[i].EntryID < sorted[j].EntryID
}
return sorted[i].CapturedAt.Before(sorted[j].CapturedAt)
})
nVal := int(float64(len(sorted)) * valFrac)
split := len(sorted) - nVal
return sorted[:split], sorted[split:]
}
// AppendLabels appends rows to path (created if needed). Rows already in
// the file are never touched.
func AppendLabels(path string, rows []LabelRow) error {
if len(rows) == 0 {
return nil
}
f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
if err != nil {
return fmt.Errorf("opening labels for append: %w", err)
}
defer func() { _ = f.Close() }()
enc := json.NewEncoder(f)
for _, row := range rows {
if err := enc.Encode(row); err != nil {
return fmt.Errorf("writing label row: %w", err)
}
}
return nil
}
|
