From eff960c54620bfba47d40bc3792ba805dea3dab3 Mon Sep 17 00:00:00 2001 From: Sam Scholten Date: Sun, 30 Aug 2026 20:34:38 +1000 Subject: rss: provenance + score calibration in items Item titles gain a rune-capped source-feed suffix so list views (which only ever show the channel title) display provenance inline. Item bodies open with feed, model score, and the batch-wide score median; the median is computed over every scored entry at poll time and stored on the run row as a nil-safe pointer, so old rows render without it. /api/status exposes the median too. --- cmds/rss.go | 57 +++++++++++++++++++++++++++++++++++++++++------------- cmds/rss_test.go | 22 +++++++++++---------- cmds/serve.go | 23 ++++++++++++++++++++++ cmds/serve_test.go | 17 ++++++++++++++++ core/runs.go | 7 ++++++- 5 files changed, 102 insertions(+), 24 deletions(-) diff --git a/cmds/rss.go b/cmds/rss.go index 3c49513..8679f6f 100644 --- a/cmds/rss.go +++ b/cmds/rss.go @@ -8,10 +8,16 @@ import ( "encoding/xml" "fmt" "time" + "unicode/utf8" "fluxrec/core" ) +// Cap on the source-feed name in item titles: feed titles can be +// arbitrarily long, and the suffix must stay compact so list views keep +// showing the article title itself. +const maxFeedTitleRunes = 40 + const rssXMLHeader = `` + "\n" type rssGUID struct { @@ -40,9 +46,11 @@ type rssDocument struct { } // RenderRSS renders runs (oldest to newest, as read from runs.jsonl) as an -// RSS 2.0 document. The source feed title prefixes each item description so -// provenance is visible in-reader; item content was already cleaned when -// the run row was written. +// RSS 2.0 document. Each item title gets the source feed as a rune-capped +// suffix (readers show the channel title as the source of every item, so +// this is the only in-list provenance), and the description body opens +// with feed + model score + the batch-wide median score. Item content was +// already cleaned when the run row was written. func RenderRSS(runs []core.RunRow) ([]byte, error) { doc := rssDocument{ Version: "2.0", @@ -54,11 +62,11 @@ func RenderRSS(runs []core.RunRow) ([]byte, error) { for i := len(runs) - 1; i >= 0; i-- { for _, k := range runs[i].Kept { doc.Channel.Items = append(doc.Channel.Items, rssItem{ - Title: k.Title, + Title: rssItemTitle(k), Link: k.URL, GUID: rssGUID{IsPermaLink: false, Value: fmt.Sprintf("mf:%d", k.EntryID)}, PubDate: k.PublishedAt.Format(time.RFC1123Z), - Description: rssDescription(k), + Description: rssDescription(k, runs[i].ScoreMedian), }) } } @@ -69,14 +77,37 @@ func RenderRSS(runs []core.RunRow) ([]byte, error) { return append([]byte(rssXMLHeader), body...), nil } -// rssDescription builds the item body: source feed name on the first line, -// then the cleaned content (if any). -func rssDescription(k core.KeptItem) string { - if k.Content == "" { - return k.FeedTitle +// rssItemTitle builds the displayed title: article title first, then the +// (rune-capped) source feed as a suffix so narrow list views truncate the +// provenance, not the title itself. +func rssItemTitle(k core.KeptItem) string { + feed := truncateRunes(k.FeedTitle, maxFeedTitleRunes) + if feed == "" { + return k.Title + } + return k.Title + " — " + feed +} + +func truncateRunes(s string, max int) string { + if utf8.RuneCountInString(s) <= max { + return s } - if k.FeedTitle == "" { - return k.Content + return string([]rune(s)[:max]) + "…" +} + +// rssDescription builds the item body: first a headline line with the +// source feed, model score, and batch-wide median (omitted for run rows +// written before the median was logged), then the cleaned content (if any). +func rssDescription(k core.KeptItem, median *float64) string { + head := fmt.Sprintf("score %.2f", k.Score) + if median != nil { + head += fmt.Sprintf(" · batch median %.2f", *median) + } + if k.FeedTitle != "" { + head = k.FeedTitle + " · " + head + } + if k.Content == "" { + return head } - return k.FeedTitle + "\n\n" + k.Content + return head + "\n\n" + k.Content } diff --git a/cmds/rss_test.go b/cmds/rss_test.go index 406f853..ca216ca 100644 --- a/cmds/rss_test.go +++ b/cmds/rss_test.go @@ -13,20 +13,22 @@ func TestRenderRSSGolden(t *testing.T) { at := func(day, h, m int) time.Time { return time.Date(2024, 3, day, h, m, 0, 0, time.UTC) } + med := 0.55 // over all scored entries in the first batch, not just kept runs := []core.RunRow{ { - RanAt: at(1, 6, 0), EntriesSeen: 40, + RanAt: at(1, 6, 0), EntriesSeen: 40, ScoreMedian: &med, Kept: []core.KeptItem{ {EntryID: 101, URL: "https://example.com/a?x=1&y=2", Title: "Alpha & Omega", FeedTitle: "Feed A", PublishedAt: at(1, 5, 0), Content: "x < y", Score: 0.9, Kind: core.KindRanked}, {EntryID: 102, URL: "https://example.com/b", Title: "Beta", - FeedTitle: "Feed B", PublishedAt: at(1, 5, 30), - Content: "second body", Score: 0.2, Kind: core.KindRandom}, + FeedTitle: "A Feed With An Extremely Long And Descriptive Title Here", + PublishedAt: at(1, 5, 30), + Content: "second body", Score: 0.2, Kind: core.KindRandom}, }, }, { - RanAt: at(1, 12, 0), EntriesSeen: 12, + RanAt: at(1, 12, 0), EntriesSeen: 12, // nil median: pre-field run row Kept: []core.KeptItem{ {EntryID: 103, URL: "https://example.com/c", Title: "Gamma", FeedTitle: "Feed A", PublishedAt: at(1, 11, 0), @@ -45,25 +47,25 @@ func TestRenderRSSGolden(t *testing.T) { fluxrec recommendations Top-scored Miniflux entries, refreshed on a schedule. - Gamma + Gamma — Feed A https://example.com/c mf:103 Fri, 01 Mar 2024 11:00:00 +0000 - Feed A + Feed A · score 0.80 - Alpha & Omega + Alpha & Omega — Feed A https://example.com/a?x=1&y=2 mf:101 Fri, 01 Mar 2024 05:00:00 +0000 - Feed A x < y + Feed A · score 0.90 · batch median 0.55 x < y - Beta + Beta — A Feed With An Extremely Long And Descri… https://example.com/b mf:102 Fri, 01 Mar 2024 05:30:00 +0000 - Feed B second body + A Feed With An Extremely Long And Descriptive Title Here · score 0.20 · batch median 0.55 second body ` diff --git a/cmds/serve.go b/cmds/serve.go index 9d82d38..0df5003 100644 --- a/cmds/serve.go +++ b/cmds/serve.go @@ -257,6 +257,7 @@ func (c *ServeCommand) pollOnce(client *miniflux.Client, vec *core.TFIDFVectoriz Kept: assembleBatch(scored, c.TopN, c.ExploreFrac, rng), DurationMs: time.Since(started).Milliseconds(), ModelCreatedAt: model.CreatedAt, + ScoreMedian: medianScore(scored), } if err := core.AppendRun(c.Runs, row); err != nil { return err @@ -275,6 +276,27 @@ type scoredEntry struct { score float64 } +// medianScore returns the median score across every scored entry in a +// poll — the batch-wide background (typical-entry) calibration rendered +// alongside kept items' scores in the RSS body. Nil when nothing was +// scored, so the run row omits the field rather than logging a fake 0.00. +func medianScore(entries []scoredEntry) *float64 { + if len(entries) == 0 { + return nil + } + scores := make([]float64, len(entries)) + for i, e := range entries { + scores[i] = e.score + } + sort.Float64s(scores) + n := len(scores) + median := scores[n/2] + if n%2 == 0 { + median = (scores[n/2-1] + scores[n/2]) / 2 + } + return &median +} + // assembleBatch ranks scored entries by score descending and selects the // batch: the top (topN minus the exploration budget) are ranked; the // budget splits at a fixed 1/3 : 2/3 into runner-ups (highest-scored just @@ -443,6 +465,7 @@ func (c *ServeCommand) routes(mins []int, modelCreatedAt time.Time) http.Handler "kept": len(last.Kept), "duration_ms": last.DurationMs, "model_created_at": last.ModelCreatedAt, + "score_median": last.ScoreMedian, } } w.Header().Set("Content-Type", "application/json") diff --git a/cmds/serve_test.go b/cmds/serve_test.go index d7df2dd..e8aec19 100644 --- a/cmds/serve_test.go +++ b/cmds/serve_test.go @@ -133,6 +133,9 @@ func TestPollMultiPage(t *testing.T) { if !row.ModelCreatedAt.Equal(model.CreatedAt) { t.Errorf("model_created_at = %v, want %v", row.ModelCreatedAt, model.CreatedAt) } + if row.ScoreMedian == nil { + t.Error("score_median missing on run row (median over all scored entries)") + } } func TestPollCrashLeavesNoState(t *testing.T) { @@ -274,6 +277,20 @@ func TestAssembleBatchFewCandidates(t *testing.T) { } } +func TestMedianScore(t *testing.T) { + if m := medianScore(nil); m != nil { + t.Errorf("empty poll: median = %v, want nil (field omitted)", *m) + } + odd := []scoredEntry{{score: 0.1}, {score: 0.9}, {score: 0.5}} + if m := medianScore(odd); m == nil || *m != 0.5 { + t.Errorf("odd median = %v, want 0.5", m) + } + even := []scoredEntry{{score: 0.1}, {score: 0.5}, {score: 0.9}, {score: 0.2}} + if m := medianScore(even); m == nil || *m != 0.35 { + t.Errorf("even median = %v, want 0.35", m) + } +} + func TestParseRefreshAt(t *testing.T) { mins, err := parseRefreshAt("18:00,06:00, 12:30") if err != nil { diff --git a/core/runs.go b/core/runs.go index 38e6710..0dd4a7f 100644 --- a/core/runs.go +++ b/core/runs.go @@ -47,7 +47,11 @@ type KeptItem struct { // RunRow is one poll batch: what was seen, what was kept, when, and how // old the model was. EntriesSeen counts every fetched entry including any -// skipped before scoring (see RecFeedSkipped). +// skipped before scoring (see RecFeedSkipped). ScoreMedian is the median +// score across every scored entry this poll — the batch's background +// calibration, rendered next to kept items' scores in the RSS body. It's +// a pointer so old rows (and empty polls) read as "absent", never as +// a misleading 0.00. type RunRow struct { RanAt time.Time `json:"ran_at"` EntriesSeen int `json:"entries_seen"` @@ -55,6 +59,7 @@ type RunRow struct { Kept []KeptItem `json:"kept"` DurationMs int64 `json:"duration_ms"` ModelCreatedAt time.Time `json:"model_created_at"` + ScoreMedian *float64 `json:"score_median,omitempty"` } // ReadRuns parses all of path strictly. A missing file is not an error — -- cgit v1.2.3