diff options
Diffstat (limited to 'cmds/export.go')
| -rw-r--r-- | cmds/export.go | 77 |
1 files changed, 56 insertions, 21 deletions
diff --git a/cmds/export.go b/cmds/export.go index 102c94a..203a7ef 100644 --- a/cmds/export.go +++ b/cmds/export.go @@ -22,14 +22,15 @@ import ( ) type ExportCommand struct { - DumpRaw string - Labels string - State string - RunOut string - RecFeedID int64 - NegDays int - NegRatio int - NegSeed int64 + DumpRaw string + Labels string + State string + RunOut string + RecFeedID int64 + NegDays int + NegRatio int + NegSeed int64 + NegFeedCapPct int } func (c *ExportCommand) Name() string { return "export" } @@ -43,7 +44,9 @@ func (c *ExportCommand) Init(args []string) error { Export labels from Miniflux into labels.jsonl: starred entries become positives (incremental via --state cursor), plus a seeded random sample of recent read+unstarred entries as negatives at --neg-ratio per positive. -Each run also writes a JSON summary to --run-out. +No single feed may hold more than --neg-feed-cap-pct percent of the +negative target (floored at 50 rows), so a few firehoses can't own the +negative class. Each run also writes a JSON summary to --run-out. With --dump-raw, instead fetch ~25 starred and ~100 read unstarred entries and write the raw API responses to DIR for offline development fixtures. @@ -58,6 +61,7 @@ and write the raw API responses to DIR for offline development fixtures. fs.IntVar(&c.NegDays, "neg-days", 45, "Only sample negatives published within this many days") fs.IntVar(&c.NegRatio, "neg-ratio", 20, "Target negatives per positive") fs.Int64Var(&c.NegSeed, "neg-seed", 20240301, "Seed for negative sampling (same data + same seed = same sample)") + fs.IntVar(&c.NegFeedCapPct, "neg-feed-cap-pct", 5, "Per-feed negative ceiling as % of target (0 = uncapped; floor 50)") return fs.Parse(args) } @@ -202,11 +206,15 @@ func (c *ExportCommand) runExport(stdout io.Writer) error { } posTotal, negTotal := 0, 0 + // negPerFeed is each feed's existing negative count: the cap is on the + // feed's total share of the pool, so history counts against it. + negPerFeed := make(map[int64]int) for _, row := range existing { if row.Label == core.LabelPos { posTotal++ } else { negTotal++ + negPerFeed[row.FeedID]++ } } posTotal += counts.appended @@ -218,7 +226,7 @@ func (c *ExportCommand) runExport(stdout io.Writer) error { fmt.Fprintln(stdout, "note: rec-feed stars without a matching labeled original were skipped") } - nc, err := c.exportNegatives(client, seenIDs, seenURLs, posTotal, negTotal, now, stdout) + nc, err := c.exportNegatives(client, seenIDs, seenURLs, negPerFeed, posTotal, negTotal, now, stdout) if err != nil { return err } @@ -226,14 +234,15 @@ func (c *ExportCommand) runExport(stdout io.Writer) error { RanAt: now, DurationMs: time.Since(started).Milliseconds(), PositivesSeen: counts.seen, PositivesNew: counts.appended, PosTotal: posTotal, NegCandidates: nc.candidates, NegNew: nc.appended, - NegTotal: negTotal + nc.appended, IDDup: counts.idDup, URLDup: counts.urlDup, + NegTotal: negTotal + nc.appended, NegCapped: nc.capped, + IDDup: counts.idDup, URLDup: counts.urlDup, RecHit: counts.recHit, RecUnresolved: counts.recUnresolved, }) } // negCounts is what the negatives sampler reports back. type negCounts struct { - candidates, appended int + candidates, appended, capped int } // exportNegatives samples read, unstarred entries published within the @@ -242,17 +251,24 @@ type negCounts struct { // (either polarity) is excluded; sampling is seeded so the same candidate // pool + seed reproduce the same sample. // +// negPerFeed carries each feed's existing negative count and is updated as +// rows are added: the --neg-feed-cap-pct ceiling applies to a feed's total +// pool share, so candidates from feeds already at their ceiling are skipped +// and the budget flows to other feeds. A capped-out pool can therefore +// finish short of the ratio target, which is reported, not hidden. +// // The window is enforced server-side via published_after (Miniflux >= 2.0.49, // the same floor the rest of export requires): only in-window entries are // fetched, so the deep archive is never walked and no client-side order // assumption is needed (Miniflux's default order is published_at asc). -func (c *ExportCommand) exportNegatives(client *miniflux.Client, seenIDs map[int64]bool, seenURLs map[string]bool, posTotal, negTotal int, now time.Time, stdout io.Writer) (negCounts, error) { +func (c *ExportCommand) exportNegatives(client *miniflux.Client, seenIDs map[int64]bool, seenURLs map[string]bool, negPerFeed map[int64]int, posTotal, negTotal int, now time.Time, stdout io.Writer) (negCounts, error) { var nc negCounts need := c.NegRatio*posTotal - negTotal if need <= 0 || posTotal == 0 { fmt.Fprintf(stdout, "negatives: skipped (need=%d pos=%d)\n", need, posTotal) return nc, nil } + cap := feedNegCap(c.NegFeedCapPct, c.NegRatio, posTotal) cutoff := now.Add(-time.Duration(c.NegDays) * 24 * time.Hour) unstarred := false @@ -285,17 +301,27 @@ func (c *ExportCommand) exportNegatives(client *miniflux.Client, seenIDs map[int } nc.candidates = len(cands) + // Seeded permutation walk: same candidate pool + seed = same sample. + // The ceiling filters the walk rather than reordering it, so enabling + // or disabling it can't perturb the sampled order of accepted rows. rng := rand.New(rand.NewSource(c.NegSeed)) perm := rng.Perm(len(cands)) - take := min(need, len(cands)) - rows := make([]core.LabelRow, 0, take) - for i := range take { - e := cands[perm[i]] + rows := make([]core.LabelRow, 0, min(need, len(cands))) + for _, idx := range perm { + if len(rows) >= need { + break + } + e := cands[idx] + if cap > 0 && negPerFeed[e.Feed.ID] >= cap { + nc.capped++ + continue + } rows = append(rows, core.LabelRow{ EntryID: e.ID, URL: e.URL, Title: e.Title, FeedID: e.Feed.ID, FeedTitle: e.Feed.Title, Label: core.LabelNeg, CapturedAt: now, }) + negPerFeed[e.Feed.ID]++ seenIDs[e.ID] = true seenURLs[core.NormalizeURL(e.URL)] = true } @@ -304,15 +330,23 @@ func (c *ExportCommand) exportNegatives(client *miniflux.Client, seenIDs map[int } nc.appended = len(rows) - fmt.Fprintf(stdout, "negatives: candidates=%d appended=%d (target=%d:%d)\n", - nc.candidates, nc.appended, c.NegRatio, posTotal) + fmt.Fprintf(stdout, "negatives: candidates=%d appended=%d capped=%d (target=%d:%d cap=%s)\n", + nc.candidates, nc.appended, nc.capped, c.NegRatio, posTotal, capLabel(cap)) if nc.appended < need { - fmt.Fprintf(stdout, "note: only %d usable candidates in the last %d days; negatives short by %d\n", - nc.candidates, c.NegDays, need-nc.appended) + fmt.Fprintf(stdout, "note: negatives short by %d (candidates=%d in the last %d days, capped-out=%d)\n", + need-nc.appended, nc.candidates, c.NegDays, nc.capped) } return nc, nil } +// capLabel renders a per-feed ceiling for the run line ("off" when uncapped). +func capLabel(cap int) string { + if cap <= 0 { + return "off" + } + return fmt.Sprintf("%d/feed", cap) +} + // runSummary is the per-run JSON summary written for visibility; kept to // the latest run (overwrite, not append). type runSummary struct { @@ -324,6 +358,7 @@ type runSummary struct { NegCandidates int `json:"neg_candidates"` NegNew int `json:"neg_new"` NegTotal int `json:"neg_total"` + NegCapped int `json:"neg_capped"` IDDup int `json:"id_dup"` URLDup int `json:"url_dup"` RecHit int `json:"rec_hit"` |