// Tests for `fluxrec export` against an httptest fake Miniflux. Env vars // point the command's config resolution at the fake — no real credentials. package cmds import ( "bytes" "encoding/json" "fmt" "net/http" "net/http/httptest" "net/url" "os" "path/filepath" "strconv" "strings" "testing" "time" "fluxrec/core" "fluxrec/miniflux" ) // testEntry mirrors the entry fields export consumes. type testFeed struct { ID int64 `json:"id"` Title string `json:"title"` } type testEntry struct { ID int64 `json:"id"` URL string `json:"url"` Title string `json:"title"` Content string `json:"content"` Status string `json:"status"` Starred bool `json:"starred"` PublishedAt time.Time `json:"published_at"` ChangedAt time.Time `json:"changed_at"` Feed testFeed `json:"feed"` } func makeEntries(n int, startID int64, starred bool, feedID int64) []testEntry { base := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) entries := make([]testEntry, n) for i := range entries { entries[i] = testEntry{ ID: startID + int64(i), URL: fmt.Sprintf("https://example.com/post/%d", startID+int64(i)), Title: fmt.Sprintf("Title %d", startID+int64(i)), Status: "read", Starred: starred, PublishedAt: base, ChangedAt: base.Add(time.Duration(i) * time.Hour), Feed: testFeed{ID: feedID, Title: "Fixture Feed"}, } } return entries } func entriesJSON(total int, entries []testEntry) string { b, _ := json.Marshal(entries) return fmt.Sprintf(`{"total":%d,"entries":%s}`, total, b) } // pagedServer serves a starred pool and a read pool (chosen per request by // the starred filter) with limit/offset paging, and records every query. type pagedServer struct { starred []testEntry read []testEntry queries []string } func (p *pagedServer) handler() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { p.queries = append(p.queries, r.URL.RawQuery) q := r.URL.Query() pool := p.read if q.Get("starred") == "true" { pool = p.starred } // Honor published_after server-side, as real Miniflux does, so the // negatives sampler's window is exercised by these tests. if pa, _ := strconv.ParseInt(q.Get("published_after"), 10, 64); pa > 0 { cutoff := time.Unix(pa, 0) filtered := pool[:0:0] for _, e := range pool { if !e.PublishedAt.Before(cutoff) { filtered = append(filtered, e) } } pool = filtered } limit, _ := strconv.Atoi(q.Get("limit")) offset, _ := strconv.Atoi(q.Get("offset")) if limit <= 0 { limit = len(pool) } if offset > len(pool) { offset = len(pool) } end := offset + limit if end > len(pool) { end = len(pool) } w.Header().Set("Content-Type", "application/json") fmt.Fprint(w, entriesJSON(len(pool), pool[offset:end])) } } // runExportAgainst wires a fake Miniflux via env, then runs the export // command with the given extra args, returning stdout. func runExportAgainst(t *testing.T, p *pagedServer, dir string, extra ...string) string { t.Helper() srv := httptest.NewServer(p.handler()) defer srv.Close() t.Setenv("MINIFLUX_URL", srv.URL) t.Setenv("MINIFLUX_TOKEN", "test-token") args := append([]string{ "--labels", filepath.Join(dir, "labels.jsonl"), "--state", filepath.Join(dir, "export_state.json"), "--run-out", filepath.Join(dir, "export_run.json"), }, extra...) cmd := &ExportCommand{} if err := cmd.Init(args); err != nil { t.Fatalf("Init: %v", err) } var out bytes.Buffer if err := cmd.Run(nil, &out); err != nil { t.Fatalf("Run: %v", err) } return out.String() } func TestExportPositivesIdempotent(t *testing.T) { p := &pagedServer{starred: makeEntries(25, 1000, true, 7)} dir := t.TempDir() out1 := runExportAgainst(t, p, dir) if !strings.Contains(out1, "appended=25") { t.Errorf("first run should append 25, got:\n%s", out1) } labels, err := core.ReadLabels(filepath.Join(dir, "labels.jsonl")) if err != nil { t.Fatalf("read labels: %v", err) } if len(labels) != 25 { t.Fatalf("expected 25 rows, got %d", len(labels)) } for _, row := range labels { if row.Label != core.LabelPos { t.Errorf("row label = %q, want pos", row.Label) } if row.EntryID < 1000 || row.EntryID >= 1025 { t.Errorf("unexpected entry id %d", row.EntryID) } } // State cursor must have advanced to the newest changed_at. stateBody, err := os.ReadFile(filepath.Join(dir, "export_state.json")) if err != nil { t.Fatalf("state not written: %v", err) } var st exportState if err := json.Unmarshal(stateBody, &st); err != nil { t.Fatalf("state unparseable: %v", err) } if st.ChangedAfter <= 0 { t.Errorf("state cursor = %d, want a positive timestamp", st.ChangedAfter) } nQueriesAfterRun1 := len(p.queries) out2 := runExportAgainst(t, p, dir) if !strings.Contains(out2, "appended=0") { t.Errorf("second run must append nothing, got:\n%s", out2) } labels2, err := core.ReadLabels(filepath.Join(dir, "labels.jsonl")) if err != nil { t.Fatalf("read labels after run 2: %v", err) } if len(labels2) != 25 { t.Fatalf("second run changed the file: %d rows", len(labels2)) } // Second run must have gone incremental: changed_after in some query // issued by that run (the first run, with no cursor, sends none). incremental := false for _, qq := range p.queries[nQueriesAfterRun1:] { decoded, _ := url.QueryUnescape(qq) if strings.Contains(decoded, "changed_after=") { incremental = true } } if !incremental { t.Errorf("second run sent no changed_after query: %v", p.queries[nQueriesAfterRun1:]) } } func TestExportURLDedup(t *testing.T) { p := &pagedServer{starred: makeEntries(3, 2000, true, 7)} dir := t.TempDir() // Pre-seed a label whose URL is a normalized twin of entry 2000's URL. pre := core.LabelRow{ EntryID: 99999, URL: "HTTPS://www.example.com/post/2000/", Title: "Old", FeedID: 1, FeedTitle: "F", Label: core.LabelPos, CapturedAt: time.Now(), } if err := core.AppendLabels(filepath.Join(dir, "labels.jsonl"), []core.LabelRow{pre}); err != nil { t.Fatal(err) } out := runExportAgainst(t, p, dir) if !strings.Contains(out, "appended=2") || !strings.Contains(out, "url-dup=1") { t.Errorf("expected appended=2 url-dup=1, got:\n%s", out) } } func TestExportRecFeedResolution(t *testing.T) { // Two starred entries ON the recommendations feed (feed id 99). rec := makeEntries(2, 3000, true, 99) rec[0].URL = "https://example.com/post/known" rec[1].URL = "https://example.com/post/unknown" p := &pagedServer{starred: rec} dir := t.TempDir() // The "known" URL is already labeled (as the original entry's star). pre := core.LabelRow{ EntryID: 1234, URL: "https://example.com/post/known", Title: "Known", FeedID: 7, FeedTitle: "Orig", Label: core.LabelPos, CapturedAt: time.Now(), } if err := core.AppendLabels(filepath.Join(dir, "labels.jsonl"), []core.LabelRow{pre}); err != nil { t.Fatal(err) } out := runExportAgainst(t, p, dir, "--rec-feed-id", "99") if !strings.Contains(out, "rec-hit=1") || !strings.Contains(out, "rec-unresolved=1") { t.Errorf("expected rec-hit=1 rec-unresolved=1, got:\n%s", out) } if !strings.Contains(out, "appended=0") { t.Errorf("rec-feed entries must never append rows, got:\n%s", out) } labels, err := core.ReadLabels(filepath.Join(dir, "labels.jsonl")) if err != nil { t.Fatal(err) } if len(labels) != 1 { t.Errorf("labels file should be untouched, has %d rows", len(labels)) } } // negFixture builds a server with `nPos` fresh starred entries and a read // pool of `nFresh` entries inside the typical 45-day window plus `nOld` // entries far outside it (sorted after the fresh ones, as published-desc // would order them). func negFixture(nPos, nFresh, nOld int) *pagedServer { now := time.Now() starred := makeEntries(nPos, 1000, true, 7) for i := range starred { starred[i].PublishedAt = now.Add(-24 * time.Hour) } fresh := makeEntries(nFresh, 5000, false, 8) for i := range fresh { fresh[i].PublishedAt = now.Add(-10 * 24 * time.Hour) } old := makeEntries(nOld, 6000, false, 8) for i := range old { old[i].PublishedAt = now.Add(-100 * 24 * time.Hour) } return &pagedServer{starred: starred, read: append(fresh, old...)} } func negEntryIDs(t *testing.T, dir string) map[int64]bool { t.Helper() labels, err := core.ReadLabels(filepath.Join(dir, "labels.jsonl")) if err != nil { t.Fatal(err) } ids := map[int64]bool{} for _, row := range labels { if row.Label == core.LabelNeg { ids[row.EntryID] = true } } return ids } func TestExportNegatives(t *testing.T) { // 5 positives × ratio 20 = 100 negatives from 150 in-window candidates. p := negFixture(5, 150, 10) dir := t.TempDir() out := runExportAgainst(t, p, dir) if !strings.Contains(out, "candidates=150") || !strings.Contains(out, "appended=100") { t.Errorf("expected candidates=150 appended=100, got:\n%s", out) } labels, err := core.ReadLabels(filepath.Join(dir, "labels.jsonl")) if err != nil { t.Fatal(err) } posIDs := map[int64]bool{} posURLs := map[string]bool{} negCount := 0 for _, row := range labels { if row.Label == core.LabelPos { posIDs[row.EntryID] = true posURLs[core.NormalizeURL(row.URL)] = true } } for _, row := range labels { if row.Label != core.LabelNeg { continue } negCount++ if posIDs[row.EntryID] { t.Errorf("negative %d collides with a positive ID", row.EntryID) } if posURLs[core.NormalizeURL(row.URL)] { t.Errorf("negative %d collides with a positive URL", row.EntryID) } if row.EntryID >= 6000 { t.Errorf("old (out-of-window) entry %d sampled as negative", row.EntryID) } } if negCount != 100 { t.Errorf("expected 100 negatives, got %d", negCount) } // Run summary (latest run, overwritten each time). body, err := os.ReadFile(filepath.Join(dir, "export_run.json")) if err != nil { t.Fatalf("run summary not written: %v", err) } var rs runSummary if err := json.Unmarshal(body, &rs); err != nil { t.Fatalf("run summary unparseable: %v", err) } if rs.PositivesNew != 5 || rs.NegNew != 100 || rs.PosTotal != 5 || rs.NegTotal != 100 { t.Errorf("run summary counts wrong: %+v", rs) } // Second run: pool already at target, so no new negatives. out2 := runExportAgainst(t, p, dir) if !strings.Contains(out2, "appended=0") { t.Errorf("second run should append nothing, got:\n%s", out2) } } func TestExportNegativesSeededDeterministic(t *testing.T) { dir1, dir2 := t.TempDir(), t.TempDir() runExportAgainst(t, negFixture(4, 120, 0), dir1) runExportAgainst(t, negFixture(4, 120, 0), dir2) ids1, ids2 := negEntryIDs(t, dir1), negEntryIDs(t, dir2) if len(ids1) != len(ids2) || len(ids1) == 0 { t.Fatalf("negative counts differ or zero: %d vs %d", len(ids1), len(ids2)) } for id := range ids1 { if !ids2[id] { t.Fatalf("same seed + same pool picked different samples; %d missing in run 2", id) } } } // ── dump-raw tests ── // fakeMiniflux serves small responses for fixture capture and records the // query of each request. type fakeMiniflux struct { queries []string } func (f *fakeMiniflux) handler() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { f.queries = append(f.queries, r.URL.RawQuery) w.Header().Set("Content-Type", "application/json") if r.URL.Query().Get("starred") == "true" { fmt.Fprint(w, entriesJSON(2, makeEntries(2, 1000, true, 1))) } else { fmt.Fprint(w, entriesJSON(3, makeEntries(3, 2000, false, 1))) } } } func TestExportDumpRawWritesFixtures(t *testing.T) { fake := &fakeMiniflux{} srv := httptest.NewServer(fake.handler()) defer srv.Close() t.Setenv("MINIFLUX_URL", srv.URL) t.Setenv("MINIFLUX_TOKEN", "test-token") dir := t.TempDir() cmd := &ExportCommand{} if err := cmd.Init([]string{"--dump-raw", dir}); err != nil { t.Fatalf("Init: %v", err) } var out bytes.Buffer if err := cmd.Run(nil, &out); err != nil { t.Fatalf("Run: %v", err) } for _, want := range []struct { file string nEntries int }{ {"starred_sample.json", 2}, {"read_sample.json", 3}, } { path := filepath.Join(dir, want.file) body, err := os.ReadFile(path) if err != nil { t.Fatalf("fixture %s not written: %v", want.file, err) } var set miniflux.EntrySet if err := json.Unmarshal(body, &set); err != nil { t.Fatalf("fixture %s is not valid entries JSON: %v", want.file, err) } if len(set.Entries) != want.nEntries { t.Errorf("%s: got %d entries, want %d", want.file, len(set.Entries), want.nEntries) } } if len(fake.queries) != 2 { t.Fatalf("expected 2 API calls, got %d", len(fake.queries)) } if !strings.Contains(fake.queries[0], "starred=true") { t.Errorf("first call should filter starred=true, got %q", fake.queries[0]) } if !strings.Contains(fake.queries[1], "status=read") { t.Errorf("second call should filter status=read, got %q", fake.queries[1]) } if !strings.Contains(fake.queries[1], "starred=false") { t.Errorf("negatives call must filter starred=false, got %q", fake.queries[1]) } if !strings.Contains(out.String(), "starred_sample.json") || !strings.Contains(out.String(), "read_sample.json") { t.Errorf("stdout should name both fixtures, got:\n%s", out.String()) } }