diff options
Diffstat (limited to 'cmds/train.go')
| -rw-r--r-- | cmds/train.go | 32 |
1 files changed, 32 insertions, 0 deletions
diff --git a/cmds/train.go b/cmds/train.go index 234ef06..4e9a54f 100644 --- a/cmds/train.go +++ b/cmds/train.go @@ -122,6 +122,10 @@ func (c *TrainCommand) Run(stdin io.Reader, stdout io.Writer) error { fmt.Fprintf(stdout, "train: pos=%d neg=%d | val: pos=%d neg=%d | precision@15=%.3f youden_j=%.3f | vocab=%d → %s\n", report.TrainPos, report.TrainNeg, report.ValPos, report.ValNeg, report.PrecisionAt15, report.YoudensJ, report.VocabSize, c.ModelOut) + if report.ValStampShare >= 0.5 { + fmt.Fprintf(stdout, "note: %d val rows (%.0f%%) share captured_at %s; one batch shaped this validation — treat precision@15 as that batch's measurement, not the model's\n", + report.ValStampCount, report.ValStampShare*100, report.ValStamp.Format(time.RFC3339)) + } return nil } @@ -202,6 +206,11 @@ func (c *TrainCommand) trainAndEvaluate(trainRows, valRows []core.LabelRow, now } } + // Val concentration: how much of the validation window is one capture + // batch (a bulk action or a single flood-heavy export). High share means + // precision@15 measured that batch, not steady-state taste. + stampShare, stampCount, stampAt := valStampConcentration(valRows) + vecParams := map[string]any{ "ngram_min": vectorizer.NgramMin, "ngram_max": vectorizer.NgramMax, "min_df": vectorizer.MinDF, "max_df": vectorizer.MaxDF, "vocab_cap": vectorizer.VocabCap, @@ -242,12 +251,32 @@ func (c *TrainCommand) trainAndEvaluate(trainRows, valRows []core.LabelRow, now TrainWindowMin: minCap.UTC(), TrainWindowMax: maxCap.UTC(), ValFrac: c.ValFrac, VocabSize: len(vectorizer.OrderedVocab), PrecisionAt15: precisionAt15, YoudensJ: youdenJ, + ValStampShare: stampShare, ValStampCount: stampCount, ValStamp: stampAt.UTC(), PositiveScores: posStats, NegativeScores: negStats, VectorizerParams: vecParams, ModelParams: modelParams, } return model, report, nil } +// valStampConcentration returns the largest share of validation rows that +// share one captured_at, that stamp's row count, and the stamp itself. +// Deterministic: ties on count pick the newer stamp. +func valStampConcentration(valRows []core.LabelRow) (share float64, count int, stamp time.Time) { + if len(valRows) == 0 { + return 0, 0, time.Time{} + } + byStamp := make(map[time.Time]int) + for _, r := range valRows { + byStamp[r.CapturedAt]++ + } + for ts, n := range byStamp { + if n > count || (n == count && ts.After(stamp)) { + share, count, stamp = float64(n)/float64(len(valRows)), n, ts + } + } + return share, count, stamp +} + // downsampleBalance downsamples the majority class to a 1:1 ratio with the // minority, using a seeded RNG for reproducibility. If already balanced, // returns the inputs unchanged. @@ -423,6 +452,9 @@ type trainReport struct { VocabSize int `json:"vocab_size"` PrecisionAt15 float64 `json:"precision_at_15"` YoudensJ float64 `json:"youdens_j"` + ValStampShare float64 `json:"val_max_stamp_share"` + ValStampCount int `json:"val_max_stamp_count"` + ValStamp time.Time `json:"val_max_stamp"` PositiveScores map[string]float64 `json:"positive_scores"` NegativeScores map[string]float64 `json:"negative_scores"` VectorizerParams map[string]any `json:"vectorizer_params"` |