1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
|
// labels.jsonl read/write and URL normalization.
//
// labels.jsonl is the canonical label store: one JSON LabelRow per line,
// append-only. ReadLabels is strict (per-line errors with line numbers);
// AppendLabels is the only writer — rows are never rewritten in place.
package core
import (
"bufio"
"encoding/json"
"errors"
"fmt"
"net/url"
"os"
"sort"
"strings"
)
// NormalizeURL collapses trivially-equivalent article links into one dedup
// key: scheme/host lowercased, leading "www." and trailing "/" dropped.
// Unparseable input degrades to a plain lowercase-trim.
func NormalizeURL(u string) string {
u = strings.TrimSpace(u)
pr, err := url.Parse(u)
if err != nil || pr.Host == "" {
return strings.TrimSuffix(strings.ToLower(u), "/")
}
pr.Scheme = strings.ToLower(pr.Scheme)
pr.Host = strings.TrimPrefix(strings.ToLower(pr.Host), "www.")
return strings.TrimSuffix(pr.String(), "/")
}
// ReadLabels parses path strictly. A missing file is not an error — it just
// means no labels yet (first export run).
func ReadLabels(path string) ([]LabelRow, error) {
f, err := os.Open(path)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil, nil
}
return nil, fmt.Errorf("opening labels: %w", err)
}
defer func() { _ = f.Close() }()
var rows []LabelRow
scanner := bufio.NewScanner(f)
scanner.Buffer(make([]byte, 64*1024), 4*1024*1024)
line := 0
for scanner.Scan() {
line++
text := strings.TrimSpace(scanner.Text())
if text == "" {
continue
}
var row LabelRow
if err := json.Unmarshal([]byte(text), &row); err != nil {
return nil, fmt.Errorf("%s:%d: invalid JSON: %w", path, line, err)
}
if row.EntryID <= 0 {
return nil, fmt.Errorf("%s:%d: entry_id must be positive", path, line)
}
if row.URL == "" {
return nil, fmt.Errorf("%s:%d: url is required", path, line)
}
if row.Label != LabelPos && row.Label != LabelNeg {
return nil, fmt.Errorf("%s:%d: label must be %q or %q, got %q", path, line, LabelPos, LabelNeg, row.Label)
}
if row.CapturedAt.IsZero() {
return nil, fmt.Errorf("%s:%d: captured_at is required", path, line)
}
rows = append(rows, row)
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("reading %s: %w", path, err)
}
return rows, nil
}
// SplitByTime partitions rows into train (older) and val (newest valFrac)
// by CapturedAt, the time-split validation strategy: train on older labels,
// evaluate on the most recent window, which is what serve actually faces.
// Sort is stable; ties on CapturedAt break by EntryID for determinism.
// valFrac is clamped: <=0 → all train, >=1 → all val. nVal is computed by
// truncation, so a tiny set may yield an empty val side (callers guard).
func SplitByTime(rows []LabelRow, valFrac float64) (train, val []LabelRow) {
if len(rows) == 0 {
return nil, nil
}
if valFrac <= 0 {
return rows, nil
}
if valFrac >= 1 {
return nil, rows
}
sorted := make([]LabelRow, len(rows))
copy(sorted, rows)
sort.SliceStable(sorted, func(i, j int) bool {
if sorted[i].CapturedAt.Equal(sorted[j].CapturedAt) {
return sorted[i].EntryID < sorted[j].EntryID
}
return sorted[i].CapturedAt.Before(sorted[j].CapturedAt)
})
nVal := int(float64(len(sorted)) * valFrac)
split := len(sorted) - nVal
return sorted[:split], sorted[split:]
}
// AppendLabels appends rows to path (created if needed). Rows already in
// the file are never touched.
func AppendLabels(path string, rows []LabelRow) error {
if len(rows) == 0 {
return nil
}
f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
if err != nil {
return fmt.Errorf("opening labels for append: %w", err)
}
defer func() { _ = f.Close() }()
enc := json.NewEncoder(f)
for _, row := range rows {
if err := enc.Encode(row); err != nil {
return fmt.Errorf("writing label row: %w", err)
}
}
return nil
}
|
