// labels.jsonl read/write and URL normalization. // // labels.jsonl is the canonical label store: one JSON LabelRow per line, // append-only. ReadLabels is strict (per-line errors with line numbers); // AppendLabels is the only writer — rows are never rewritten in place. package core import ( "bufio" "encoding/json" "errors" "fmt" "net/url" "os" "sort" "strings" ) // NormalizeURL collapses trivially-equivalent article links into one dedup // key: scheme/host lowercased, leading "www." and trailing "/" dropped. // Unparseable input degrades to a plain lowercase-trim. func NormalizeURL(u string) string { u = strings.TrimSpace(u) pr, err := url.Parse(u) if err != nil || pr.Host == "" { return strings.TrimSuffix(strings.ToLower(u), "/") } pr.Scheme = strings.ToLower(pr.Scheme) pr.Host = strings.TrimPrefix(strings.ToLower(pr.Host), "www.") return strings.TrimSuffix(pr.String(), "/") } // ReadLabels parses path strictly. A missing file is not an error — it just // means no labels yet (first export run). func ReadLabels(path string) ([]LabelRow, error) { f, err := os.Open(path) if err != nil { if errors.Is(err, os.ErrNotExist) { return nil, nil } return nil, fmt.Errorf("opening labels: %w", err) } defer func() { _ = f.Close() }() var rows []LabelRow scanner := bufio.NewScanner(f) scanner.Buffer(make([]byte, 64*1024), 4*1024*1024) line := 0 for scanner.Scan() { line++ text := strings.TrimSpace(scanner.Text()) if text == "" { continue } var row LabelRow if err := json.Unmarshal([]byte(text), &row); err != nil { return nil, fmt.Errorf("%s:%d: invalid JSON: %w", path, line, err) } if row.EntryID <= 0 { return nil, fmt.Errorf("%s:%d: entry_id must be positive", path, line) } if row.URL == "" { return nil, fmt.Errorf("%s:%d: url is required", path, line) } if row.Label != LabelPos && row.Label != LabelNeg { return nil, fmt.Errorf("%s:%d: label must be %q or %q, got %q", path, line, LabelPos, LabelNeg, row.Label) } if row.CapturedAt.IsZero() { return nil, fmt.Errorf("%s:%d: captured_at is required", path, line) } rows = append(rows, row) } if err := scanner.Err(); err != nil { return nil, fmt.Errorf("reading %s: %w", path, err) } return rows, nil } // SplitByTime partitions rows into train (older) and val (newest valFrac) // by CapturedAt, the time-split validation strategy: train on older labels, // evaluate on the most recent window, which is what serve actually faces. // Sort is stable; ties on CapturedAt break by EntryID for determinism. // valFrac is clamped: <=0 → all train, >=1 → all val. nVal is computed by // truncation, so a tiny set may yield an empty val side (callers guard). func SplitByTime(rows []LabelRow, valFrac float64) (train, val []LabelRow) { if len(rows) == 0 { return nil, nil } if valFrac <= 0 { return rows, nil } if valFrac >= 1 { return nil, rows } sorted := make([]LabelRow, len(rows)) copy(sorted, rows) sort.SliceStable(sorted, func(i, j int) bool { if sorted[i].CapturedAt.Equal(sorted[j].CapturedAt) { return sorted[i].EntryID < sorted[j].EntryID } return sorted[i].CapturedAt.Before(sorted[j].CapturedAt) }) nVal := int(float64(len(sorted)) * valFrac) split := len(sorted) - nVal return sorted[:split], sorted[split:] } // AppendLabels appends rows to path (created if needed). Rows already in // the file are never touched. func AppendLabels(path string, rows []LabelRow) error { if len(rows) == 0 { return nil } f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) if err != nil { return fmt.Errorf("opening labels for append: %w", err) } defer func() { _ = f.Close() }() enc := json.NewEncoder(f) for _, row := range rows { if err := enc.Encode(row); err != nil { return fmt.Errorf("writing label row: %w", err) } } return nil }