aboutsummaryrefslogtreecommitdiff
path: root/cmds/train_test.go
diff options
context:
space:
mode:
Diffstat (limited to 'cmds/train_test.go')
-rw-r--r--cmds/train_test.go54
1 files changed, 54 insertions, 0 deletions
diff --git a/cmds/train_test.go b/cmds/train_test.go
index 5a2ecee..2a3bbeb 100644
--- a/cmds/train_test.go
+++ b/cmds/train_test.go
@@ -221,6 +221,60 @@ func TestTrainModelConsumedByScore(t *testing.T) {
}
}
+// Val concentration: when one capture batch dominates the validation window,
+// the report must say so (share/count/stamp) and stdout must flag it.
+func TestTrainReportsValStampConcentration(t *testing.T) {
+ dir := t.TempDir()
+ labelsPath := filepath.Join(dir, "labels.jsonl")
+ syntheticLabels(t, labelsPath, 50, 50)
+
+ // Restamp the newest 25 rows to one captured_at: val-frac 0.3 puts
+ // 30 rows in val, 25 of them from this batch.
+ rows, err := core.ReadLabels(labelsPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ stamp := rows[len(rows)-1].CapturedAt
+ for i := len(rows) - 25; i < len(rows); i++ {
+ rows[i].CapturedAt = stamp
+ }
+ if err := rewriteLabels(labelsPath, "", rows); err != nil {
+ t.Fatal(err)
+ }
+
+ cmd := &TrainCommand{}
+ if err := cmd.Init([]string{labelsPath, "--val-frac", "0.3",
+ "--model-out", filepath.Join(dir, "m.json"),
+ "--report-out", filepath.Join(dir, "r.json")}); err != nil {
+ t.Fatalf("Init: %v", err)
+ }
+ var out bytes.Buffer
+ if err := cmd.Run(nil, &out); err != nil {
+ t.Fatalf("Run: %v", err)
+ }
+
+ body, err := os.ReadFile(filepath.Join(dir, "r.json"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ var report trainReport
+ if err := json.Unmarshal(body, &report); err != nil {
+ t.Fatal(err)
+ }
+ if want := 25.0 / 30.0; report.ValStampShare != want {
+ t.Errorf("val_max_stamp_share = %v, want %v", report.ValStampShare, want)
+ }
+ if report.ValStampCount != 25 {
+ t.Errorf("val_max_stamp_count = %d, want 25", report.ValStampCount)
+ }
+ if !report.ValStamp.Equal(stamp) {
+ t.Errorf("val_max_stamp = %v, want %v", report.ValStamp, stamp)
+ }
+ if !strings.Contains(out.String(), "one batch shaped this validation") {
+ t.Errorf("stdout should flag the concentrated validation:\n%s", out.String())
+ }
+}
+
// Guard: too few validation positives must error loudly.
func TestTrainGuardFewValPositives(t *testing.T) {
dir := t.TempDir()