aboutsummaryrefslogtreecommitdiff
path: root/cmds/export.go
diff options
context:
space:
mode:
Diffstat (limited to 'cmds/export.go')
-rw-r--r--cmds/export.go83
1 files changed, 65 insertions, 18 deletions
diff --git a/cmds/export.go b/cmds/export.go
index 203a7ef..9b153ee 100644
--- a/cmds/export.go
+++ b/cmds/export.go
@@ -2,9 +2,13 @@
// labels.jsonl, the canonical label store.
//
// Positives: starred entries, incremental via a persisted changed_after
-// cursor, deduped by ID then normalized URL. Negatives: seeded random
-// sample of recent read+unstarred entries, topped up to a fixed ratio of
-// the positive pool. Also carries --dump-raw fixture capture.
+// cursor, deduped by ID then normalized URL. A star outranks the read state
+// that put an entry in the negative pool: it flips the stored negative row
+// (see relabel.go), or — for a fluxrec pick whose original was never
+// exported — is recorded as its own positive.
+// Negatives: seeded random sample of recent read+unstarred entries, topped
+// up to a fixed ratio of the positive pool. Also carries --dump-raw fixture
+// capture.
package cmds
import (
@@ -48,6 +52,12 @@ No single feed may hold more than --neg-feed-cap-pct percent of the
negative target (floored at 50 rows), so a few firehoses can't own the
negative class. Each run also writes a JSON summary to --run-out.
+A star always wins over the read state that produced a negative: a starred
+entry already labeled negative is relabeled to positive in place (identity
+kept, fresh captured_at, original copied to LABELS.bak), and a star on a
+rec-feed item whose original was never exported is added as its own
+positive (URL is the original's; title/feed identity are the rec copy's).
+
With --dump-raw, instead fetch ~25 starred and ~100 read unstarred entries
and write the raw API responses to DIR for offline development fixtures.
`)
@@ -108,7 +118,7 @@ func saveExportState(path string, st exportState) error {
// exportCounts is what the run reports to stdout.
type exportCounts struct {
- seen, appended, idDup, urlDup, recHit, recUnresolved int
+ seen, appended, idDup, urlDup, recHit, recAdded, relabeled int
}
func (c *ExportCommand) runExport(stdout io.Writer) error {
@@ -143,6 +153,10 @@ func (c *ExportCommand) runExport(stdout io.Writer) error {
var counts exportCounts
var newRows []core.LabelRow
+ // Every star seen this run, so rows already in the store can be flipped
+ // to positive after the walk (relabelStars).
+ starIDs := make(map[int64]bool)
+ starURLs := make(map[string]bool)
maxChanged := state.ChangedAfter
now := time.Now().UTC()
@@ -159,16 +173,30 @@ func (c *ExportCommand) runExport(stdout io.Writer) error {
if ts := e.ChangedAt.Unix(); ts > maxChanged {
maxChanged = ts
}
-
- // Stars on the recommendations feed are not original entries;
- // resolve by URL against existing labels. If the original was
- // never exported, there is nothing to attach the star to.
+ starIDs[e.ID] = true
+ starURLs[core.NormalizeURL(e.URL)] = true
+
+ // Stars on the recommendations feed are not original entries.
+ // A star outranks whatever the original's read state said, so
+ // resolve by URL against existing labels when possible (the
+ // relabel pass below flips a stored negative), and otherwise
+ // record the star as its own positive: the URL is the original
+ // article's, only the title/feed identity come from the rec copy.
if c.RecFeedID != 0 && e.Feed.ID == c.RecFeedID {
if seenURLs[core.NormalizeURL(e.URL)] {
counts.recHit++
- } else {
- counts.recUnresolved++
+ continue
}
+ nu := core.NormalizeURL(e.URL)
+ newRows = append(newRows, core.LabelRow{
+ EntryID: e.ID, URL: e.URL, Title: e.Title,
+ FeedID: e.Feed.ID, FeedTitle: e.Feed.Title,
+ Label: core.LabelPos, CapturedAt: now,
+ })
+ seenIDs[e.ID] = true
+ seenURLs[nu] = true
+ counts.appended++
+ counts.recAdded++
continue
}
@@ -196,6 +224,18 @@ func (c *ExportCommand) runExport(stdout io.Writer) error {
}
}
+ // A star on an entry already in the store as a negative flips that row:
+ // same identity, label positive, captured_at refreshed so the star reads
+ // as recent evidence. Rewritten in place, backup first — the rows the
+ // dedup counters above skipped as duplicates are exactly the ones this
+ // rescues.
+ if flipped := relabelStars(existing, starIDs, starURLs, now); flipped > 0 {
+ if err := rewriteLabels(c.Labels, c.Labels+".bak", existing); err != nil {
+ return err
+ }
+ counts.relabeled = flipped
+ }
+
if err := core.AppendLabels(c.Labels, newRows); err != nil {
return err
}
@@ -219,11 +259,16 @@ func (c *ExportCommand) runExport(stdout io.Writer) error {
}
posTotal += counts.appended
- fmt.Fprintf(stdout, "export: seen=%d appended=%d (id-dup=%d url-dup=%d rec-hit=%d rec-unresolved=%d) pos-total=%d\n",
- counts.seen, counts.appended, counts.idDup, counts.urlDup,
- counts.recHit, counts.recUnresolved, posTotal)
- if counts.recUnresolved > 0 {
- fmt.Fprintln(stdout, "note: rec-feed stars without a matching labeled original were skipped")
+ fmt.Fprintf(stdout, "export: seen=%d appended=%d relabeled=%d (id-dup=%d url-dup=%d rec-hit=%d rec-added=%d) pos-total=%d\n",
+ counts.seen, counts.appended, counts.relabeled, counts.idDup, counts.urlDup,
+ counts.recHit, counts.recAdded, posTotal)
+ if counts.relabeled > 0 {
+ fmt.Fprintf(stdout, "note: relabeled %d starred entries that were stored as negatives (backup: %s)\n",
+ counts.relabeled, c.Labels+".bak")
+ }
+ if counts.recAdded > 0 {
+ fmt.Fprintf(stdout, "note: %d rec-feed stars had no labeled original and were added as positives\n",
+ counts.recAdded)
}
nc, err := c.exportNegatives(client, seenIDs, seenURLs, negPerFeed, posTotal, negTotal, now, stdout)
@@ -235,8 +280,9 @@ func (c *ExportCommand) runExport(stdout io.Writer) error {
PositivesSeen: counts.seen, PositivesNew: counts.appended,
PosTotal: posTotal, NegCandidates: nc.candidates, NegNew: nc.appended,
NegTotal: negTotal + nc.appended, NegCapped: nc.capped,
- IDDup: counts.idDup, URLDup: counts.urlDup,
- RecHit: counts.recHit, RecUnresolved: counts.recUnresolved,
+ Relabeled: counts.relabeled,
+ IDDup: counts.idDup, URLDup: counts.urlDup,
+ RecHit: counts.recHit, RecAdded: counts.recAdded,
})
}
@@ -359,10 +405,11 @@ type runSummary struct {
NegNew int `json:"neg_new"`
NegTotal int `json:"neg_total"`
NegCapped int `json:"neg_capped"`
+ Relabeled int `json:"relabeled"`
IDDup int `json:"id_dup"`
URLDup int `json:"url_dup"`
RecHit int `json:"rec_hit"`
- RecUnresolved int `json:"rec_unresolved"`
+ RecAdded int `json:"rec_added"`
}
func writeRunSummary(path string, s runSummary) error {