// Tests for labels.jsonl handling and URL normalization. package core import ( "fmt" "os" "path/filepath" "strings" "testing" "time" ) func TestNormalizeURL(t *testing.T) { cases := []struct{ in, want string }{ {"HTTPS://WWW.Example.com/Path/", "https://example.com/Path"}, {"https://example.com/a?x=1&y=2", "https://example.com/a?x=1&y=2"}, {"https://example.com", "https://example.com"}, {" https://EXAMPLE.com/ ", "https://example.com"}, {"https://www.www.example.com/x", "https://www.example.com/x"}, // only one leading www. {"not a url", "not a url"}, } for _, c := range cases { if got := NormalizeURL(c.in); got != c.want { t.Errorf("NormalizeURL(%q) = %q, want %q", c.in, got, c.want) } } } func sampleRow() LabelRow { return LabelRow{ EntryID: 42, URL: "https://example.com/post", Title: "A title", FeedID: 7, FeedTitle: "Feed", Label: LabelPos, CapturedAt: time.Date(2024, 3, 1, 12, 0, 0, 0, time.UTC), } } func TestReadLabelsMissingFile(t *testing.T) { rows, err := ReadLabels(filepath.Join(t.TempDir(), "nope.jsonl")) if err != nil { t.Fatalf("missing file should not error: %v", err) } if len(rows) != 0 { t.Fatalf("expected no rows, got %d", len(rows)) } } func TestAppendThenReadRoundTrip(t *testing.T) { path := filepath.Join(t.TempDir(), "labels.jsonl") r1, r2 := sampleRow(), sampleRow() r2.EntryID, r2.Label = 43, LabelNeg if err := AppendLabels(path, []LabelRow{r1}); err != nil { t.Fatalf("append 1: %v", err) } if err := AppendLabels(path, []LabelRow{r2}); err != nil { t.Fatalf("append 2: %v", err) } rows, err := ReadLabels(path) if err != nil { t.Fatalf("read: %v", err) } if len(rows) != 2 || rows[0].EntryID != 42 || rows[1].Label != LabelNeg { t.Fatalf("round trip mismatch: %+v", rows) } } func TestAppendLabelsNoopOnEmpty(t *testing.T) { path := filepath.Join(t.TempDir(), "labels.jsonl") if err := AppendLabels(path, nil); err != nil { t.Fatalf("noop append: %v", err) } if _, err := os.Stat(path); !os.IsNotExist(err) { t.Fatal("empty append should not create the file") } } func TestSplitByTime(t *testing.T) { mk := func(id int64, day int, label string) LabelRow { return LabelRow{ EntryID: id, URL: "https://example.com/" + fmt.Sprint(id), Title: "T", FeedID: 1, FeedTitle: "F", Label: label, CapturedAt: time.Date(2024, 1, day, 0, 0, 0, 0, time.UTC), } } // 10 rows, days 1..10. val-frac 0.3 → newest 3 (days 8,9,10) are val. rows := []LabelRow{ mk(1, 1, LabelPos), mk(2, 2, LabelNeg), mk(3, 3, LabelPos), mk(4, 4, LabelNeg), mk(5, 5, LabelPos), mk(6, 6, LabelNeg), mk(7, 7, LabelPos), mk(8, 8, LabelNeg), mk(9, 9, LabelPos), mk(10, 10, LabelNeg), } train, val := SplitByTime(rows, 0.3) if len(val) != 3 { t.Fatalf("val size = %d, want 3", len(val)) } if len(train) != 7 { t.Fatalf("train size = %d, want 7", len(train)) } // Val must be the newest (days 8,9,10). wantValDays := map[int]bool{8: true, 9: true, 10: true} for _, r := range val { d := r.CapturedAt.Day() if !wantValDays[d] { t.Errorf("val contains day %d, expected 8/9/10", d) } } // Train must be the older days 1..7. for _, r := range train { if r.CapturedAt.Day() > 7 { t.Errorf("train contains day %d, expected <=7", r.CapturedAt.Day()) } } } func TestSplitByTimeEdgeCases(t *testing.T) { rows := []LabelRow{ {EntryID: 1, CapturedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)}, {EntryID: 2, CapturedAt: time.Date(2024, 1, 2, 0, 0, 0, 0, time.UTC)}, } if train, val := SplitByTime(rows, 0); len(train) != 2 || len(val) != 0 { t.Errorf("valFrac 0: train=%d val=%d, want 2/0", len(train), len(val)) } if train, val := SplitByTime(rows, 1); len(train) != 0 || len(val) != 2 { t.Errorf("valFrac 1: train=%d val=%d, want 0/2", len(train), len(val)) } if train, val := SplitByTime(nil, 0.3); len(train) != 0 || len(val) != 0 { t.Errorf("empty: train=%d val=%d, want 0/0", len(train), len(val)) } } func TestSplitByTimeTieBreakDeterministic(t *testing.T) { // Same CapturedAt, different EntryIDs: tie broken by EntryID ascending. day := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) rows := []LabelRow{ {EntryID: 30, CapturedAt: day}, {EntryID: 10, CapturedAt: day}, {EntryID: 20, CapturedAt: day}, } // Two runs must produce identical splits. train1, val1 := SplitByTime(rows, 0.34) _, val2 := SplitByTime(rows, 0.34) if len(val1) != len(val2) { t.Fatalf("non-deterministic val size: %d vs %d", len(val1), len(val2)) } for i := range val1 { if val1[i].EntryID != val2[i].EntryID { t.Fatalf("non-deterministic tie break at %d: %d vs %d", i, val1[i].EntryID, val2[i].EntryID) } } // Val (newest) should carry the highest EntryIDs. if len(val1) > 0 { maxTrain := int64(0) for _, r := range train1 { if r.EntryID > maxTrain { maxTrain = r.EntryID } } for _, r := range val1 { if r.EntryID <= maxTrain { t.Errorf("val id %d not greater than max train id %d (tie break ascending)", r.EntryID, maxTrain) } } } } func TestReadLabelsStrictErrors(t *testing.T) { cases := []struct { name, body, wantSub string }{ {"bad json", `{not json`, "invalid JSON"}, {"bad id", `{"entry_id":0,"url":"https://x","label":"pos","captured_at":"2024-01-01T00:00:00Z"}`, "entry_id"}, {"no url", `{"entry_id":1,"label":"pos","captured_at":"2024-01-01T00:00:00Z"}`, "url"}, {"bad label", `{"entry_id":1,"url":"https://x","label":"maybe","captured_at":"2024-01-01T00:00:00Z"}`, "label"}, {"no captured_at", `{"entry_id":1,"url":"https://x","label":"pos"}`, "captured_at"}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { path := filepath.Join(t.TempDir(), "labels.jsonl") if err := os.WriteFile(path, []byte(c.body+"\n"), 0o644); err != nil { t.Fatal(err) } _, err := ReadLabels(path) if err == nil { t.Fatal("expected error") } if !strings.Contains(err.Error(), c.wantSub) { t.Errorf("error %q should mention %q", err, c.wantSub) } if !strings.Contains(err.Error(), ":1") { t.Errorf("error %q should carry the line number", err) } }) } }