diff options
Diffstat (limited to 'cmds/train_test.go')
| -rw-r--r-- | cmds/train_test.go | 54 |
1 files changed, 54 insertions, 0 deletions
diff --git a/cmds/train_test.go b/cmds/train_test.go index 5a2ecee..2a3bbeb 100644 --- a/cmds/train_test.go +++ b/cmds/train_test.go @@ -221,6 +221,60 @@ func TestTrainModelConsumedByScore(t *testing.T) { } } +// Val concentration: when one capture batch dominates the validation window, +// the report must say so (share/count/stamp) and stdout must flag it. +func TestTrainReportsValStampConcentration(t *testing.T) { + dir := t.TempDir() + labelsPath := filepath.Join(dir, "labels.jsonl") + syntheticLabels(t, labelsPath, 50, 50) + + // Restamp the newest 25 rows to one captured_at: val-frac 0.3 puts + // 30 rows in val, 25 of them from this batch. + rows, err := core.ReadLabels(labelsPath) + if err != nil { + t.Fatal(err) + } + stamp := rows[len(rows)-1].CapturedAt + for i := len(rows) - 25; i < len(rows); i++ { + rows[i].CapturedAt = stamp + } + if err := rewriteLabels(labelsPath, "", rows); err != nil { + t.Fatal(err) + } + + cmd := &TrainCommand{} + if err := cmd.Init([]string{labelsPath, "--val-frac", "0.3", + "--model-out", filepath.Join(dir, "m.json"), + "--report-out", filepath.Join(dir, "r.json")}); err != nil { + t.Fatalf("Init: %v", err) + } + var out bytes.Buffer + if err := cmd.Run(nil, &out); err != nil { + t.Fatalf("Run: %v", err) + } + + body, err := os.ReadFile(filepath.Join(dir, "r.json")) + if err != nil { + t.Fatal(err) + } + var report trainReport + if err := json.Unmarshal(body, &report); err != nil { + t.Fatal(err) + } + if want := 25.0 / 30.0; report.ValStampShare != want { + t.Errorf("val_max_stamp_share = %v, want %v", report.ValStampShare, want) + } + if report.ValStampCount != 25 { + t.Errorf("val_max_stamp_count = %d, want 25", report.ValStampCount) + } + if !report.ValStamp.Equal(stamp) { + t.Errorf("val_max_stamp = %v, want %v", report.ValStamp, stamp) + } + if !strings.Contains(out.String(), "one batch shaped this validation") { + t.Errorf("stdout should flag the concentrated validation:\n%s", out.String()) + } +} + // Guard: too few validation positives must error loudly. func TestTrainGuardFewValPositives(t *testing.T) { dir := t.TempDir() |
