diff options
Diffstat (limited to 'cmds/export.go')
| -rw-r--r-- | cmds/export.go | 83 |
1 files changed, 65 insertions, 18 deletions
diff --git a/cmds/export.go b/cmds/export.go index 203a7ef..9b153ee 100644 --- a/cmds/export.go +++ b/cmds/export.go @@ -2,9 +2,13 @@ // labels.jsonl, the canonical label store. // // Positives: starred entries, incremental via a persisted changed_after -// cursor, deduped by ID then normalized URL. Negatives: seeded random -// sample of recent read+unstarred entries, topped up to a fixed ratio of -// the positive pool. Also carries --dump-raw fixture capture. +// cursor, deduped by ID then normalized URL. A star outranks the read state +// that put an entry in the negative pool: it flips the stored negative row +// (see relabel.go), or — for a fluxrec pick whose original was never +// exported — is recorded as its own positive. +// Negatives: seeded random sample of recent read+unstarred entries, topped +// up to a fixed ratio of the positive pool. Also carries --dump-raw fixture +// capture. package cmds import ( @@ -48,6 +52,12 @@ No single feed may hold more than --neg-feed-cap-pct percent of the negative target (floored at 50 rows), so a few firehoses can't own the negative class. Each run also writes a JSON summary to --run-out. +A star always wins over the read state that produced a negative: a starred +entry already labeled negative is relabeled to positive in place (identity +kept, fresh captured_at, original copied to LABELS.bak), and a star on a +rec-feed item whose original was never exported is added as its own +positive (URL is the original's; title/feed identity are the rec copy's). + With --dump-raw, instead fetch ~25 starred and ~100 read unstarred entries and write the raw API responses to DIR for offline development fixtures. `) @@ -108,7 +118,7 @@ func saveExportState(path string, st exportState) error { // exportCounts is what the run reports to stdout. type exportCounts struct { - seen, appended, idDup, urlDup, recHit, recUnresolved int + seen, appended, idDup, urlDup, recHit, recAdded, relabeled int } func (c *ExportCommand) runExport(stdout io.Writer) error { @@ -143,6 +153,10 @@ func (c *ExportCommand) runExport(stdout io.Writer) error { var counts exportCounts var newRows []core.LabelRow + // Every star seen this run, so rows already in the store can be flipped + // to positive after the walk (relabelStars). + starIDs := make(map[int64]bool) + starURLs := make(map[string]bool) maxChanged := state.ChangedAfter now := time.Now().UTC() @@ -159,16 +173,30 @@ func (c *ExportCommand) runExport(stdout io.Writer) error { if ts := e.ChangedAt.Unix(); ts > maxChanged { maxChanged = ts } - - // Stars on the recommendations feed are not original entries; - // resolve by URL against existing labels. If the original was - // never exported, there is nothing to attach the star to. + starIDs[e.ID] = true + starURLs[core.NormalizeURL(e.URL)] = true + + // Stars on the recommendations feed are not original entries. + // A star outranks whatever the original's read state said, so + // resolve by URL against existing labels when possible (the + // relabel pass below flips a stored negative), and otherwise + // record the star as its own positive: the URL is the original + // article's, only the title/feed identity come from the rec copy. if c.RecFeedID != 0 && e.Feed.ID == c.RecFeedID { if seenURLs[core.NormalizeURL(e.URL)] { counts.recHit++ - } else { - counts.recUnresolved++ + continue } + nu := core.NormalizeURL(e.URL) + newRows = append(newRows, core.LabelRow{ + EntryID: e.ID, URL: e.URL, Title: e.Title, + FeedID: e.Feed.ID, FeedTitle: e.Feed.Title, + Label: core.LabelPos, CapturedAt: now, + }) + seenIDs[e.ID] = true + seenURLs[nu] = true + counts.appended++ + counts.recAdded++ continue } @@ -196,6 +224,18 @@ func (c *ExportCommand) runExport(stdout io.Writer) error { } } + // A star on an entry already in the store as a negative flips that row: + // same identity, label positive, captured_at refreshed so the star reads + // as recent evidence. Rewritten in place, backup first — the rows the + // dedup counters above skipped as duplicates are exactly the ones this + // rescues. + if flipped := relabelStars(existing, starIDs, starURLs, now); flipped > 0 { + if err := rewriteLabels(c.Labels, c.Labels+".bak", existing); err != nil { + return err + } + counts.relabeled = flipped + } + if err := core.AppendLabels(c.Labels, newRows); err != nil { return err } @@ -219,11 +259,16 @@ func (c *ExportCommand) runExport(stdout io.Writer) error { } posTotal += counts.appended - fmt.Fprintf(stdout, "export: seen=%d appended=%d (id-dup=%d url-dup=%d rec-hit=%d rec-unresolved=%d) pos-total=%d\n", - counts.seen, counts.appended, counts.idDup, counts.urlDup, - counts.recHit, counts.recUnresolved, posTotal) - if counts.recUnresolved > 0 { - fmt.Fprintln(stdout, "note: rec-feed stars without a matching labeled original were skipped") + fmt.Fprintf(stdout, "export: seen=%d appended=%d relabeled=%d (id-dup=%d url-dup=%d rec-hit=%d rec-added=%d) pos-total=%d\n", + counts.seen, counts.appended, counts.relabeled, counts.idDup, counts.urlDup, + counts.recHit, counts.recAdded, posTotal) + if counts.relabeled > 0 { + fmt.Fprintf(stdout, "note: relabeled %d starred entries that were stored as negatives (backup: %s)\n", + counts.relabeled, c.Labels+".bak") + } + if counts.recAdded > 0 { + fmt.Fprintf(stdout, "note: %d rec-feed stars had no labeled original and were added as positives\n", + counts.recAdded) } nc, err := c.exportNegatives(client, seenIDs, seenURLs, negPerFeed, posTotal, negTotal, now, stdout) @@ -235,8 +280,9 @@ func (c *ExportCommand) runExport(stdout io.Writer) error { PositivesSeen: counts.seen, PositivesNew: counts.appended, PosTotal: posTotal, NegCandidates: nc.candidates, NegNew: nc.appended, NegTotal: negTotal + nc.appended, NegCapped: nc.capped, - IDDup: counts.idDup, URLDup: counts.urlDup, - RecHit: counts.recHit, RecUnresolved: counts.recUnresolved, + Relabeled: counts.relabeled, + IDDup: counts.idDup, URLDup: counts.urlDup, + RecHit: counts.recHit, RecAdded: counts.recAdded, }) } @@ -359,10 +405,11 @@ type runSummary struct { NegNew int `json:"neg_new"` NegTotal int `json:"neg_total"` NegCapped int `json:"neg_capped"` + Relabeled int `json:"relabeled"` IDDup int `json:"id_dup"` URLDup int `json:"url_dup"` RecHit int `json:"rec_hit"` - RecUnresolved int `json:"rec_unresolved"` + RecAdded int `json:"rec_added"` } func writeRunSummary(path string, s runSummary) error { |
