1 files changed, 441 insertions, 0 deletions

diff --git a/cmds/export_test.go b/cmds/export_test.go
new file mode 100644
index 0000000..0190d39
--- /dev/null
+++ b/cmds/export_test.go
@@ -0,0 +1,441 @@
+// Tests for `fluxrec export` against an httptest fake Miniflux. Env vars
+// point the command's config resolution at the fake — no real credentials.
+package cmds
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"fluxrec/core"
+	"fluxrec/miniflux"
+)
+
+// testEntry mirrors the entry fields export consumes.
+type testFeed struct {
+	ID    int64  `json:"id"`
+	Title string `json:"title"`
+}
+
+type testEntry struct {
+	ID          int64     `json:"id"`
+	URL         string    `json:"url"`
+	Title       string    `json:"title"`
+	Content     string    `json:"content"`
+	Status      string    `json:"status"`
+	Starred     bool      `json:"starred"`
+	PublishedAt time.Time `json:"published_at"`
+	ChangedAt   time.Time `json:"changed_at"`
+	Feed        testFeed  `json:"feed"`
+}
+
+func makeEntries(n int, startID int64, starred bool, feedID int64) []testEntry {
+	base := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)
+	entries := make([]testEntry, n)
+	for i := range entries {
+		entries[i] = testEntry{
+			ID:          startID + int64(i),
+			URL:         fmt.Sprintf("https://example.com/post/%d", startID+int64(i)),
+			Title:       fmt.Sprintf("Title %d", startID+int64(i)),
+			Status:      "read",
+			Starred:     starred,
+			PublishedAt: base,
+			ChangedAt:   base.Add(time.Duration(i) * time.Hour),
+			Feed:        testFeed{ID: feedID, Title: "Fixture Feed"},
+		}
+	}
+	return entries
+}
+
+func entriesJSON(total int, entries []testEntry) string {
+	b, _ := json.Marshal(entries)
+	return fmt.Sprintf(`{"total":%d,"entries":%s}`, total, b)
+}
+
+// pagedServer serves a starred pool and a read pool (chosen per request by
+// the starred filter) with limit/offset paging, and records every query.
+type pagedServer struct {
+	starred []testEntry
+	read    []testEntry
+	queries []string
+}
+
+func (p *pagedServer) handler() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		p.queries = append(p.queries, r.URL.RawQuery)
+		q := r.URL.Query()
+		pool := p.read
+		if q.Get("starred") == "true" {
+			pool = p.starred
+		}
+		// Honor published_after server-side, as real Miniflux does, so the
+		// negatives sampler's window is exercised by these tests.
+		if pa, _ := strconv.ParseInt(q.Get("published_after"), 10, 64); pa > 0 {
+			cutoff := time.Unix(pa, 0)
+			filtered := pool[:0:0]
+			for _, e := range pool {
+				if !e.PublishedAt.Before(cutoff) {
+					filtered = append(filtered, e)
+				}
+			}
+			pool = filtered
+		}
+		limit, _ := strconv.Atoi(q.Get("limit"))
+		offset, _ := strconv.Atoi(q.Get("offset"))
+		if limit <= 0 {
+			limit = len(pool)
+		}
+		if offset > len(pool) {
+			offset = len(pool)
+		}
+		end := offset + limit
+		if end > len(pool) {
+			end = len(pool)
+		}
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprint(w, entriesJSON(len(pool), pool[offset:end]))
+	}
+}
+
+// runExportAgainst wires a fake Miniflux via env, then runs the export
+// command with the given extra args, returning stdout.
+func runExportAgainst(t *testing.T, p *pagedServer, dir string, extra ...string) string {
+	t.Helper()
+	srv := httptest.NewServer(p.handler())
+	defer srv.Close()
+	t.Setenv("MINIFLUX_URL", srv.URL)
+	t.Setenv("MINIFLUX_TOKEN", "test-token")
+
+	args := append([]string{
+		"--labels", filepath.Join(dir, "labels.jsonl"),
+		"--state", filepath.Join(dir, "export_state.json"),
+		"--run-out", filepath.Join(dir, "export_run.json"),
+	}, extra...)
+
+	cmd := &ExportCommand{}
+	if err := cmd.Init(args); err != nil {
+		t.Fatalf("Init: %v", err)
+	}
+	var out bytes.Buffer
+	if err := cmd.Run(nil, &out); err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+	return out.String()
+}
+
+func TestExportPositivesIdempotent(t *testing.T) {
+	p := &pagedServer{starred: makeEntries(25, 1000, true, 7)}
+	dir := t.TempDir()
+
+	out1 := runExportAgainst(t, p, dir)
+	if !strings.Contains(out1, "appended=25") {
+		t.Errorf("first run should append 25, got:\n%s", out1)
+	}
+	labels, err := core.ReadLabels(filepath.Join(dir, "labels.jsonl"))
+	if err != nil {
+		t.Fatalf("read labels: %v", err)
+	}
+	if len(labels) != 25 {
+		t.Fatalf("expected 25 rows, got %d", len(labels))
+	}
+	for _, row := range labels {
+		if row.Label != core.LabelPos {
+			t.Errorf("row label = %q, want pos", row.Label)
+		}
+		if row.EntryID < 1000 || row.EntryID >= 1025 {
+			t.Errorf("unexpected entry id %d", row.EntryID)
+		}
+	}
+
+	// State cursor must have advanced to the newest changed_at.
+	stateBody, err := os.ReadFile(filepath.Join(dir, "export_state.json"))
+	if err != nil {
+		t.Fatalf("state not written: %v", err)
+	}
+	var st exportState
+	if err := json.Unmarshal(stateBody, &st); err != nil {
+		t.Fatalf("state unparseable: %v", err)
+	}
+	if st.ChangedAfter <= 0 {
+		t.Errorf("state cursor = %d, want a positive timestamp", st.ChangedAfter)
+	}
+
+	nQueriesAfterRun1 := len(p.queries)
+	out2 := runExportAgainst(t, p, dir)
+	if !strings.Contains(out2, "appended=0") {
+		t.Errorf("second run must append nothing, got:\n%s", out2)
+	}
+	labels2, err := core.ReadLabels(filepath.Join(dir, "labels.jsonl"))
+	if err != nil {
+		t.Fatalf("read labels after run 2: %v", err)
+	}
+	if len(labels2) != 25 {
+		t.Fatalf("second run changed the file: %d rows", len(labels2))
+	}
+	// Second run must have gone incremental: changed_after in some query
+	// issued by that run (the first run, with no cursor, sends none).
+	incremental := false
+	for _, qq := range p.queries[nQueriesAfterRun1:] {
+		decoded, _ := url.QueryUnescape(qq)
+		if strings.Contains(decoded, "changed_after=") {
+			incremental = true
+		}
+	}
+	if !incremental {
+		t.Errorf("second run sent no changed_after query: %v", p.queries[nQueriesAfterRun1:])
+	}
+}
+
+func TestExportURLDedup(t *testing.T) {
+	p := &pagedServer{starred: makeEntries(3, 2000, true, 7)}
+	dir := t.TempDir()
+
+	// Pre-seed a label whose URL is a normalized twin of entry 2000's URL.
+	pre := core.LabelRow{
+		EntryID: 99999, URL: "HTTPS://www.example.com/post/2000/", Title: "Old",
+		FeedID: 1, FeedTitle: "F", Label: core.LabelPos,
+		CapturedAt: time.Now(),
+	}
+	if err := core.AppendLabels(filepath.Join(dir, "labels.jsonl"), []core.LabelRow{pre}); err != nil {
+		t.Fatal(err)
+	}
+
+	out := runExportAgainst(t, p, dir)
+	if !strings.Contains(out, "appended=2") || !strings.Contains(out, "url-dup=1") {
+		t.Errorf("expected appended=2 url-dup=1, got:\n%s", out)
+	}
+}
+
+func TestExportRecFeedResolution(t *testing.T) {
+	// Two starred entries ON the recommendations feed (feed id 99).
+	rec := makeEntries(2, 3000, true, 99)
+	rec[0].URL = "https://example.com/post/known"
+	rec[1].URL = "https://example.com/post/unknown"
+	p := &pagedServer{starred: rec}
+	dir := t.TempDir()
+
+	// The "known" URL is already labeled (as the original entry's star).
+	pre := core.LabelRow{
+		EntryID: 1234, URL: "https://example.com/post/known", Title: "Known",
+		FeedID: 7, FeedTitle: "Orig", Label: core.LabelPos,
+		CapturedAt: time.Now(),
+	}
+	if err := core.AppendLabels(filepath.Join(dir, "labels.jsonl"), []core.LabelRow{pre}); err != nil {
+		t.Fatal(err)
+	}
+
+	out := runExportAgainst(t, p, dir, "--rec-feed-id", "99")
+	if !strings.Contains(out, "rec-hit=1") || !strings.Contains(out, "rec-unresolved=1") {
+		t.Errorf("expected rec-hit=1 rec-unresolved=1, got:\n%s", out)
+	}
+	if !strings.Contains(out, "appended=0") {
+		t.Errorf("rec-feed entries must never append rows, got:\n%s", out)
+	}
+	labels, err := core.ReadLabels(filepath.Join(dir, "labels.jsonl"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(labels) != 1 {
+		t.Errorf("labels file should be untouched, has %d rows", len(labels))
+	}
+}
+
+// negFixture builds a server with `nPos` fresh starred entries and a read
+// pool of `nFresh` entries inside the typical 45-day window plus `nOld`
+// entries far outside it (sorted after the fresh ones, as published-desc
+// would order them).
+func negFixture(nPos, nFresh, nOld int) *pagedServer {
+	now := time.Now()
+	starred := makeEntries(nPos, 1000, true, 7)
+	for i := range starred {
+		starred[i].PublishedAt = now.Add(-24 * time.Hour)
+	}
+	fresh := makeEntries(nFresh, 5000, false, 8)
+	for i := range fresh {
+		fresh[i].PublishedAt = now.Add(-10 * 24 * time.Hour)
+	}
+	old := makeEntries(nOld, 6000, false, 8)
+	for i := range old {
+		old[i].PublishedAt = now.Add(-100 * 24 * time.Hour)
+	}
+	return &pagedServer{starred: starred, read: append(fresh, old...)}
+}
+
+func negEntryIDs(t *testing.T, dir string) map[int64]bool {
+	t.Helper()
+	labels, err := core.ReadLabels(filepath.Join(dir, "labels.jsonl"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	ids := map[int64]bool{}
+	for _, row := range labels {
+		if row.Label == core.LabelNeg {
+			ids[row.EntryID] = true
+		}
+	}
+	return ids
+}
+
+func TestExportNegatives(t *testing.T) {
+	// 5 positives × ratio 20 = 100 negatives from 150 in-window candidates.
+	p := negFixture(5, 150, 10)
+	dir := t.TempDir()
+
+	out := runExportAgainst(t, p, dir)
+	if !strings.Contains(out, "candidates=150") || !strings.Contains(out, "appended=100") {
+		t.Errorf("expected candidates=150 appended=100, got:\n%s", out)
+	}
+
+	labels, err := core.ReadLabels(filepath.Join(dir, "labels.jsonl"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	posIDs := map[int64]bool{}
+	posURLs := map[string]bool{}
+	negCount := 0
+	for _, row := range labels {
+		if row.Label == core.LabelPos {
+			posIDs[row.EntryID] = true
+			posURLs[core.NormalizeURL(row.URL)] = true
+		}
+	}
+	for _, row := range labels {
+		if row.Label != core.LabelNeg {
+			continue
+		}
+		negCount++
+		if posIDs[row.EntryID] {
+			t.Errorf("negative %d collides with a positive ID", row.EntryID)
+		}
+		if posURLs[core.NormalizeURL(row.URL)] {
+			t.Errorf("negative %d collides with a positive URL", row.EntryID)
+		}
+		if row.EntryID >= 6000 {
+			t.Errorf("old (out-of-window) entry %d sampled as negative", row.EntryID)
+		}
+	}
+	if negCount != 100 {
+		t.Errorf("expected 100 negatives, got %d", negCount)
+	}
+
+	// Run summary (latest run, overwritten each time).
+	body, err := os.ReadFile(filepath.Join(dir, "export_run.json"))
+	if err != nil {
+		t.Fatalf("run summary not written: %v", err)
+	}
+	var rs runSummary
+	if err := json.Unmarshal(body, &rs); err != nil {
+		t.Fatalf("run summary unparseable: %v", err)
+	}
+	if rs.PositivesNew != 5 || rs.NegNew != 100 || rs.PosTotal != 5 || rs.NegTotal != 100 {
+		t.Errorf("run summary counts wrong: %+v", rs)
+	}
+
+	// Second run: pool already at target, so no new negatives.
+	out2 := runExportAgainst(t, p, dir)
+	if !strings.Contains(out2, "appended=0") {
+		t.Errorf("second run should append nothing, got:\n%s", out2)
+	}
+}
+
+func TestExportNegativesSeededDeterministic(t *testing.T) {
+	dir1, dir2 := t.TempDir(), t.TempDir()
+	runExportAgainst(t, negFixture(4, 120, 0), dir1)
+	runExportAgainst(t, negFixture(4, 120, 0), dir2)
+
+	ids1, ids2 := negEntryIDs(t, dir1), negEntryIDs(t, dir2)
+	if len(ids1) != len(ids2) || len(ids1) == 0 {
+		t.Fatalf("negative counts differ or zero: %d vs %d", len(ids1), len(ids2))
+	}
+	for id := range ids1 {
+		if !ids2[id] {
+			t.Fatalf("same seed + same pool picked different samples; %d missing in run 2", id)
+		}
+	}
+}
+
+// ── dump-raw tests ──
+
+// fakeMiniflux serves small responses for fixture capture and records the
+// query of each request.
+type fakeMiniflux struct {
+	queries []string
+}
+
+func (f *fakeMiniflux) handler() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		f.queries = append(f.queries, r.URL.RawQuery)
+		w.Header().Set("Content-Type", "application/json")
+		if r.URL.Query().Get("starred") == "true" {
+			fmt.Fprint(w, entriesJSON(2, makeEntries(2, 1000, true, 1)))
+		} else {
+			fmt.Fprint(w, entriesJSON(3, makeEntries(3, 2000, false, 1)))
+		}
+	}
+}
+
+func TestExportDumpRawWritesFixtures(t *testing.T) {
+	fake := &fakeMiniflux{}
+	srv := httptest.NewServer(fake.handler())
+	defer srv.Close()
+	t.Setenv("MINIFLUX_URL", srv.URL)
+	t.Setenv("MINIFLUX_TOKEN", "test-token")
+
+	dir := t.TempDir()
+	cmd := &ExportCommand{}
+	if err := cmd.Init([]string{"--dump-raw", dir}); err != nil {
+		t.Fatalf("Init: %v", err)
+	}
+	var out bytes.Buffer
+	if err := cmd.Run(nil, &out); err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+
+	for _, want := range []struct {
+		file     string
+		nEntries int
+	}{
+		{"starred_sample.json", 2},
+		{"read_sample.json", 3},
+	} {
+		path := filepath.Join(dir, want.file)
+		body, err := os.ReadFile(path)
+		if err != nil {
+			t.Fatalf("fixture %s not written: %v", want.file, err)
+		}
+		var set miniflux.EntrySet
+		if err := json.Unmarshal(body, &set); err != nil {
+			t.Fatalf("fixture %s is not valid entries JSON: %v", want.file, err)
+		}
+		if len(set.Entries) != want.nEntries {
+			t.Errorf("%s: got %d entries, want %d", want.file, len(set.Entries), want.nEntries)
+		}
+	}
+
+	if len(fake.queries) != 2 {
+		t.Fatalf("expected 2 API calls, got %d", len(fake.queries))
+	}
+	if !strings.Contains(fake.queries[0], "starred=true") {
+		t.Errorf("first call should filter starred=true, got %q", fake.queries[0])
+	}
+	if !strings.Contains(fake.queries[1], "status=read") {
+		t.Errorf("second call should filter status=read, got %q", fake.queries[1])
+	}
+	if !strings.Contains(fake.queries[1], "starred=false") {
+		t.Errorf("negatives call must filter starred=false, got %q", fake.queries[1])
+	}
+
+	if !strings.Contains(out.String(), "starred_sample.json") ||
+		!strings.Contains(out.String(), "read_sample.json") {
+		t.Errorf("stdout should name both fixtures, got:\n%s", out.String())
+	}
+}