path: root/cmds/export_test.go

// Tests for `fluxrec export` against an httptest fake Miniflux. Env vars
// point the command's config resolution at the fake — no real credentials.
package cmds

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"testing"
	"time"

	"fluxrec/core"
	"fluxrec/miniflux"
)

// testEntry mirrors the entry fields export consumes.
type testFeed struct {
	ID    int64  `json:"id"`
	Title string `json:"title"`
}

type testEntry struct {
	ID          int64     `json:"id"`
	URL         string    `json:"url"`
	Title       string    `json:"title"`
	Content     string    `json:"content"`
	Status      string    `json:"status"`
	Starred     bool      `json:"starred"`
	PublishedAt time.Time `json:"published_at"`
	ChangedAt   time.Time `json:"changed_at"`
	Feed        testFeed  `json:"feed"`
}

func makeEntries(n int, startID int64, starred bool, feedID int64) []testEntry {
	base := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)
	entries := make([]testEntry, n)
	for i := range entries {
		entries[i] = testEntry{
			ID:          startID + int64(i),
			URL:         fmt.Sprintf("https://example.com/post/%d", startID+int64(i)),
			Title:       fmt.Sprintf("Title %d", startID+int64(i)),
			Status:      "read",
			Starred:     starred,
			PublishedAt: base,
			ChangedAt:   base.Add(time.Duration(i) * time.Hour),
			Feed:        testFeed{ID: feedID, Title: "Fixture Feed"},
		}
	}
	return entries
}

func entriesJSON(total int, entries []testEntry) string {
	b, _ := json.Marshal(entries)
	return fmt.Sprintf(`{"total":%d,"entries":%s}`, total, b)
}

// pagedServer serves a starred pool and a read pool (chosen per request by
// the starred filter) with limit/offset paging, and records every query.
type pagedServer struct {
	starred []testEntry
	read    []testEntry
	queries []string
}

func (p *pagedServer) handler() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		p.queries = append(p.queries, r.URL.RawQuery)
		q := r.URL.Query()
		pool := p.read
		if q.Get("starred") == "true" {
			pool = p.starred
		}
		// Honor published_after server-side, as real Miniflux does, so the
		// negatives sampler's window is exercised by these tests.
		if pa, _ := strconv.ParseInt(q.Get("published_after"), 10, 64); pa > 0 {
			cutoff := time.Unix(pa, 0)
			filtered := pool[:0:0]
			for _, e := range pool {
				if !e.PublishedAt.Before(cutoff) {
					filtered = append(filtered, e)
				}
			}
			pool = filtered
		}
		limit, _ := strconv.Atoi(q.Get("limit"))
		offset, _ := strconv.Atoi(q.Get("offset"))
		if limit <= 0 {
			limit = len(pool)
		}
		if offset > len(pool) {
			offset = len(pool)
		}
		end := offset + limit
		if end > len(pool) {
			end = len(pool)
		}
		w.Header().Set("Content-Type", "application/json")
		fmt.Fprint(w, entriesJSON(len(pool), pool[offset:end]))
	}
}

// runExportAgainst wires a fake Miniflux via env, then runs the export
// command with the given extra args, returning stdout.
func runExportAgainst(t *testing.T, p *pagedServer, dir string, extra ...string) string {
	t.Helper()
	srv := httptest.NewServer(p.handler())
	defer srv.Close()
	t.Setenv("MINIFLUX_URL", srv.URL)
	t.Setenv("MINIFLUX_TOKEN", "test-token")

	// Ratio/dedup semantics are tested uncapped; the per-feed ceiling has
	// its own dedicated test (TestExportNegativesFeedCap).
	args := append([]string{
		"--labels", filepath.Join(dir, "labels.jsonl"),
		"--state", filepath.Join(dir, "export_state.json"),
		"--run-out", filepath.Join(dir, "export_run.json"),
		"--neg-feed-cap-pct", "0",
	}, extra...)

	cmd := &ExportCommand{}
	if err := cmd.Init(args); err != nil {
		t.Fatalf("Init: %v", err)
	}
	var out bytes.Buffer
	if err := cmd.Run(nil, &out); err != nil {
		t.Fatalf("Run: %v", err)
	}
	return out.String()
}

func TestExportPositivesIdempotent(t *testing.T) {
	p := &pagedServer{starred: makeEntries(25, 1000, true, 7)}
	dir := t.TempDir()

	out1 := runExportAgainst(t, p, dir)
	if !strings.Contains(out1, "appended=25") {
		t.Errorf("first run should append 25, got:\n%s", out1)
	}
	labels, err := core.ReadLabels(filepath.Join(dir, "labels.jsonl"))
	if err != nil {
		t.Fatalf("read labels: %v", err)
	}
	if len(labels) != 25 {
		t.Fatalf("expected 25 rows, got %d", len(labels))
	}
	for _, row := range labels {
		if row.Label != core.LabelPos {
			t.Errorf("row label = %q, want pos", row.Label)
		}
		if row.EntryID < 1000 || row.EntryID >= 1025 {
			t.Errorf("unexpected entry id %d", row.EntryID)
		}
	}

	// State cursor must have advanced to the newest changed_at.
	stateBody, err := os.ReadFile(filepath.Join(dir, "export_state.json"))
	if err != nil {
		t.Fatalf("state not written: %v", err)
	}
	var st exportState
	if err := json.Unmarshal(stateBody, &st); err != nil {
		t.Fatalf("state unparseable: %v", err)
	}
	if st.ChangedAfter <= 0 {
		t.Errorf("state cursor = %d, want a positive timestamp", st.ChangedAfter)
	}

	nQueriesAfterRun1 := len(p.queries)
	out2 := runExportAgainst(t, p, dir)
	if !strings.Contains(out2, "appended=0") {
		t.Errorf("second run must append nothing, got:\n%s", out2)
	}
	labels2, err := core.ReadLabels(filepath.Join(dir, "labels.jsonl"))
	if err != nil {
		t.Fatalf("read labels after run 2: %v", err)
	}
	if len(labels2) != 25 {
		t.Fatalf("second run changed the file: %d rows", len(labels2))
	}
	// Second run must have gone incremental: changed_after in some query
	// issued by that run (the first run, with no cursor, sends none).
	incremental := false
	for _, qq := range p.queries[nQueriesAfterRun1:] {
		decoded, _ := url.QueryUnescape(qq)
		if strings.Contains(decoded, "changed_after=") {
			incremental = true
		}
	}
	if !incremental {
		t.Errorf("second run sent no changed_after query: %v", p.queries[nQueriesAfterRun1:])
	}
}

func TestExportURLDedup(t *testing.T) {
	p := &pagedServer{starred: makeEntries(3, 2000, true, 7)}
	dir := t.TempDir()

	// Pre-seed a label whose URL is a normalized twin of entry 2000's URL.
	pre := core.LabelRow{
		EntryID: 99999, URL: "HTTPS://www.example.com/post/2000/", Title: "Old",
		FeedID: 1, FeedTitle: "F", Label: core.LabelPos,
		CapturedAt: time.Now(),
	}
	if err := core.AppendLabels(filepath.Join(dir, "labels.jsonl"), []core.LabelRow{pre}); err != nil {
		t.Fatal(err)
	}

	out := runExportAgainst(t, p, dir)
	if !strings.Contains(out, "appended=2") || !strings.Contains(out, "url-dup=1") {
		t.Errorf("expected appended=2 url-dup=1, got:\n%s", out)
	}
}

// Read-then-starred: an entry sampled as a negative, starred later, must flip
// to positive (identity kept, captured_at refreshed) rather than be dropped as
// a duplicate — the star is the signal, and a stale negative row is worse than
// no row at all.
func TestExportRelabelsStarredNegative(t *testing.T) {
	p := &pagedServer{starred: makeEntries(1, 1000, true, 7)}
	dir := t.TempDir()
	labelsPath := filepath.Join(dir, "labels.jsonl")

	old := time.Now().Add(-48 * time.Hour).UTC()
	pre := core.LabelRow{
		EntryID: 1000, URL: "https://example.com/post/1000", Title: "Title 1000",
		FeedID: 7, FeedTitle: "Fixture Feed", Label: core.LabelNeg,
		CapturedAt: old,
	}
	if err := core.AppendLabels(labelsPath, []core.LabelRow{pre}); err != nil {
		t.Fatal(err)
	}

	out := runExportAgainst(t, p, dir)
	if !strings.Contains(out, "relabeled=1") || !strings.Contains(out, "appended=0") {
		t.Errorf("expected relabeled=1 appended=0, got:\n%s", out)
	}

	labels, err := core.ReadLabels(labelsPath)
	if err != nil {
		t.Fatal(err)
	}
	if len(labels) != 1 {
		t.Fatalf("relabel must rewrite in place, got %d rows", len(labels))
	}
	got := labels[0]
	if got.Label != core.LabelPos {
		t.Errorf("label = %q, want pos", got.Label)
	}
	if got.EntryID != 1000 || got.Title != "Title 1000" || got.FeedID != 7 {
		t.Errorf("relabel must keep the row's identity, got %+v", got)
	}
	if !got.CapturedAt.After(old) {
		t.Errorf("captured_at = %v, want refreshed past %v", got.CapturedAt, old)
	}

	// The pre-rewrite file is preserved as a backup.
	bak, err := core.ReadLabels(labelsPath + ".bak")
	if err != nil {
		t.Fatalf("read backup: %v", err)
	}
	if len(bak) != 1 || bak[0].Label != core.LabelNeg {
		t.Errorf("backup should hold the pre-flip negative row, got %+v", bak)
	}

	body, err := os.ReadFile(filepath.Join(dir, "export_run.json"))
	if err != nil {
		t.Fatal(err)
	}
	var rs runSummary
	if err := json.Unmarshal(body, &rs); err != nil {
		t.Fatal(err)
	}
	if rs.Relabeled != 1 || rs.PosTotal != 1 || rs.NegTotal != 0 {
		t.Errorf("run summary should reflect the flip: %+v", rs)
	}
}

// The same flip has to work through rec-feed resolution: the starred row is
// the rec feed's copy, so only the URL ties it to the labeled original.
func TestExportRelabelsRecFeedStar(t *testing.T) {
	rec := makeEntries(1, 3000, true, 99)
	rec[0].URL = "https://example.com/post/known"
	p := &pagedServer{starred: rec}
	dir := t.TempDir()
	labelsPath := filepath.Join(dir, "labels.jsonl")

	pre := core.LabelRow{
		EntryID: 1234, URL: "https://example.com/post/known", Title: "Known",
		FeedID: 7, FeedTitle: "Orig", Label: core.LabelNeg,
		CapturedAt: time.Now().Add(-48 * time.Hour).UTC(),
	}
	if err := core.AppendLabels(labelsPath, []core.LabelRow{pre}); err != nil {
		t.Fatal(err)
	}

	out := runExportAgainst(t, p, dir, "--rec-feed-id", "99")
	if !strings.Contains(out, "relabeled=1") || !strings.Contains(out, "rec-hit=1") {
		t.Errorf("expected relabeled=1 rec-hit=1, got:\n%s", out)
	}

	labels, err := core.ReadLabels(labelsPath)
	if err != nil {
		t.Fatal(err)
	}
	if len(labels) != 1 || labels[0].Label != core.LabelPos {
		t.Fatalf("original row should be flipped to pos, got %+v", labels)
	}
	if labels[0].EntryID != 1234 {
		t.Errorf("relabel must keep the original entry id, got %d", labels[0].EntryID)
	}
}

func TestExportRecFeedResolution(t *testing.T) {
	// Two starred entries ON the recommendations feed (feed id 99).
	rec := makeEntries(2, 3000, true, 99)
	rec[0].URL = "https://example.com/post/known"
	rec[1].URL = "https://example.com/post/unknown"
	p := &pagedServer{starred: rec}
	dir := t.TempDir()

	// The "known" URL is already labeled (as the original entry's star).
	pre := core.LabelRow{
		EntryID: 1234, URL: "https://example.com/post/known", Title: "Known",
		FeedID: 7, FeedTitle: "Orig", Label: core.LabelPos,
		CapturedAt: time.Now(),
	}
	if err := core.AppendLabels(filepath.Join(dir, "labels.jsonl"), []core.LabelRow{pre}); err != nil {
		t.Fatal(err)
	}

	out := runExportAgainst(t, p, dir, "--rec-feed-id", "99")
	if !strings.Contains(out, "rec-hit=1") || !strings.Contains(out, "rec-added=1") {
		t.Errorf("expected rec-hit=1 rec-added=1, got:\n%s", out)
	}
	if !strings.Contains(out, "appended=1") {
		t.Errorf("an unresolved rec-feed star should append its own positive, got:\n%s", out)
	}
	labels, err := core.ReadLabels(filepath.Join(dir, "labels.jsonl"))
	if err != nil {
		t.Fatal(err)
	}
	if len(labels) != 2 {
		t.Fatalf("expected the seeded row plus one added star, got %d rows", len(labels))
	}
	// The resolved star kept the original row (id 1234); the unresolved one
	// is a positive carrying the rec copy's id but the original URL.
	added := labels[1]
	if added.Label != core.LabelPos || added.EntryID != 3001 {
		t.Errorf("added row = %+v, want positive for rec entry 3001", added)
	}
	if added.URL != "https://example.com/post/unknown" {
		t.Errorf("added row URL = %q, want the recommendation's article link", added.URL)
	}
}

// negFixture builds a server with `nPos` fresh starred entries and a read
// pool of `nFresh` entries inside the typical 45-day window plus `nOld`
// entries far outside it (sorted after the fresh ones, as published-desc
// would order them).
func negFixture(nPos, nFresh, nOld int) *pagedServer {
	now := time.Now()
	starred := makeEntries(nPos, 1000, true, 7)
	for i := range starred {
		starred[i].PublishedAt = now.Add(-24 * time.Hour)
	}
	fresh := makeEntries(nFresh, 5000, false, 8)
	for i := range fresh {
		fresh[i].PublishedAt = now.Add(-10 * 24 * time.Hour)
	}
	old := makeEntries(nOld, 6000, false, 8)
	for i := range old {
		old[i].PublishedAt = now.Add(-100 * 24 * time.Hour)
	}
	return &pagedServer{starred: starred, read: append(fresh, old...)}
}

func negEntryIDs(t *testing.T, dir string) map[int64]bool {
	t.Helper()
	labels, err := core.ReadLabels(filepath.Join(dir, "labels.jsonl"))
	if err != nil {
		t.Fatal(err)
	}
	ids := map[int64]bool{}
	for _, row := range labels {
		if row.Label == core.LabelNeg {
			ids[row.EntryID] = true
		}
	}
	return ids
}

func TestExportNegatives(t *testing.T) {
	// 5 positives × ratio 20 = 100 negatives from 150 in-window candidates.
	p := negFixture(5, 150, 10)
	dir := t.TempDir()

	out := runExportAgainst(t, p, dir)
	if !strings.Contains(out, "candidates=150") || !strings.Contains(out, "appended=100") {
		t.Errorf("expected candidates=150 appended=100, got:\n%s", out)
	}

	labels, err := core.ReadLabels(filepath.Join(dir, "labels.jsonl"))
	if err != nil {
		t.Fatal(err)
	}
	posIDs := map[int64]bool{}
	posURLs := map[string]bool{}
	negCount := 0
	for _, row := range labels {
		if row.Label == core.LabelPos {
			posIDs[row.EntryID] = true
			posURLs[core.NormalizeURL(row.URL)] = true
		}
	}
	for _, row := range labels {
		if row.Label != core.LabelNeg {
			continue
		}
		negCount++
		if posIDs[row.EntryID] {
			t.Errorf("negative %d collides with a positive ID", row.EntryID)
		}
		if posURLs[core.NormalizeURL(row.URL)] {
			t.Errorf("negative %d collides with a positive URL", row.EntryID)
		}
		if row.EntryID >= 6000 {
			t.Errorf("old (out-of-window) entry %d sampled as negative", row.EntryID)
		}
	}
	if negCount != 100 {
		t.Errorf("expected 100 negatives, got %d", negCount)
	}

	// Run summary (latest run, overwritten each time).
	body, err := os.ReadFile(filepath.Join(dir, "export_run.json"))
	if err != nil {
		t.Fatalf("run summary not written: %v", err)
	}
	var rs runSummary
	if err := json.Unmarshal(body, &rs); err != nil {
		t.Fatalf("run summary unparseable: %v", err)
	}
	if rs.PositivesNew != 5 || rs.NegNew != 100 || rs.PosTotal != 5 || rs.NegTotal != 100 {
		t.Errorf("run summary counts wrong: %+v", rs)
	}

	// Second run: pool already at target, so no new negatives.
	out2 := runExportAgainst(t, p, dir)
	if !strings.Contains(out2, "appended=0") {
		t.Errorf("second run should append nothing, got:\n%s", out2)
	}
}

// The per-feed ceiling: one feed's negatives are bounded by the cap, the
// budget then flows to other feeds, and the shortfall (vs. the ratio
// target) is reported rather than silently padded.
func TestExportNegativesFeedCap(t *testing.T) {
	now := time.Now()
	starred := makeEntries(10, 1000, true, 7)
	for i := range starred {
		starred[i].PublishedAt = now.Add(-24 * time.Hour)
	}
	// A big firehose pool and a small quiet feed's worth, both in-window.
	big := makeEntries(500, 5000, false, 8)
	small := makeEntries(30, 9000, false, 9)
	for i := range big {
		big[i].PublishedAt = now.Add(-10 * 24 * time.Hour)
	}
	for i := range small {
		small[i].PublishedAt = now.Add(-10 * 24 * time.Hour)
	}
	p := &pagedServer{starred: starred, read: append(big, small...)}
	dir := t.TempDir()

	// 10 positives × ratio 20 = target 200 negatives. Ceiling = 5% × 200 =
	// 10 → floor 50 wins; cap is 50/feed.
	out := runExportAgainst(t, p, dir, "--neg-days", "100000", "--neg-feed-cap-pct", "5")

	labels, err := core.ReadLabels(filepath.Join(dir, "labels.jsonl"))
	if err != nil {
		t.Fatal(err)
	}
	perFeed := map[int64]int{}
	negN := 0
	for _, row := range labels {
		if row.Label == core.LabelNeg {
			perFeed[row.FeedID]++
			negN++
		}
	}
	if perFeed[8] != 50 {
		t.Errorf("firehose feed should be capped at 50, got %d", perFeed[8])
	}
	if perFeed[9] != 30 {
		t.Errorf("quiet feed should give all 30 of its candidates, got %d", perFeed[9])
	}
	if negN != 80 {
		t.Errorf("total negatives = %d, want 80 (50 capped + 30 quiet)", negN)
	}
	if !strings.Contains(out, "capped=") || !strings.Contains(out, "negatives short by 120") {
		t.Errorf("shortfall from capping should be reported:\n%s", out)
	}
	body, err := os.ReadFile(filepath.Join(dir, "export_run.json"))
	if err != nil {
		t.Fatal(err)
	}
	var rs runSummary
	if err := json.Unmarshal(body, &rs); err != nil {
		t.Fatal(err)
	}
	// All 500 firehose candidates walk the permutation; 50 are accepted,
	// the other 450 are skipped by its ceiling (the quiet feed never caps).
	if rs.NegCapped != 450 {
		t.Errorf("neg_capped = %d, want 450", rs.NegCapped)
	}
}

func TestExportNegativesSeededDeterministic(t *testing.T) {
	dir1, dir2 := t.TempDir(), t.TempDir()
	runExportAgainst(t, negFixture(4, 120, 0), dir1)
	runExportAgainst(t, negFixture(4, 120, 0), dir2)

	ids1, ids2 := negEntryIDs(t, dir1), negEntryIDs(t, dir2)
	if len(ids1) != len(ids2) || len(ids1) == 0 {
		t.Fatalf("negative counts differ or zero: %d vs %d", len(ids1), len(ids2))
	}
	for id := range ids1 {
		if !ids2[id] {
			t.Fatalf("same seed + same pool picked different samples; %d missing in run 2", id)
		}
	}
}

// ── dump-raw tests ──

// fakeMiniflux serves small responses for fixture capture and records the
// query of each request.
type fakeMiniflux struct {
	queries []string
}

func (f *fakeMiniflux) handler() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		f.queries = append(f.queries, r.URL.RawQuery)
		w.Header().Set("Content-Type", "application/json")
		if r.URL.Query().Get("starred") == "true" {
			fmt.Fprint(w, entriesJSON(2, makeEntries(2, 1000, true, 1)))
		} else {
			fmt.Fprint(w, entriesJSON(3, makeEntries(3, 2000, false, 1)))
		}
	}
}

func TestExportDumpRawWritesFixtures(t *testing.T) {
	fake := &fakeMiniflux{}
	srv := httptest.NewServer(fake.handler())
	defer srv.Close()
	t.Setenv("MINIFLUX_URL", srv.URL)
	t.Setenv("MINIFLUX_TOKEN", "test-token")

	dir := t.TempDir()
	cmd := &ExportCommand{}
	if err := cmd.Init([]string{"--dump-raw", dir}); err != nil {
		t.Fatalf("Init: %v", err)
	}
	var out bytes.Buffer
	if err := cmd.Run(nil, &out); err != nil {
		t.Fatalf("Run: %v", err)
	}

	for _, want := range []struct {
		file     string
		nEntries int
	}{
		{"starred_sample.json", 2},
		{"read_sample.json", 3},
	} {
		path := filepath.Join(dir, want.file)
		body, err := os.ReadFile(path)
		if err != nil {
			t.Fatalf("fixture %s not written: %v", want.file, err)
		}
		var set miniflux.EntrySet
		if err := json.Unmarshal(body, &set); err != nil {
			t.Fatalf("fixture %s is not valid entries JSON: %v", want.file, err)
		}
		if len(set.Entries) != want.nEntries {
			t.Errorf("%s: got %d entries, want %d", want.file, len(set.Entries), want.nEntries)
		}
	}

	if len(fake.queries) != 2 {
		t.Fatalf("expected 2 API calls, got %d", len(fake.queries))
	}
	if !strings.Contains(fake.queries[0], "starred=true") {
		t.Errorf("first call should filter starred=true, got %q", fake.queries[0])
	}
	if !strings.Contains(fake.queries[1], "status=read") {
		t.Errorf("second call should filter status=read, got %q", fake.queries[1])
	}
	if !strings.Contains(fake.queries[1], "starred=false") {
		t.Errorf("negatives call must filter starred=false, got %q", fake.queries[1])
	}

	if !strings.Contains(out.String(), "starred_sample.json") ||
		!strings.Contains(out.String(), "read_sample.json") {
		t.Errorf("stdout should name both fixtures, got:\n%s", out.String())
	}
}