// Roundtrip test for the vendored TF-IDF + LR pipeline: fit a tiny model, // confirm scores separate the two synthetic topics. Guards against a broken // copy more than against algorithm regressions. package core import ( "testing" ) func TestTFIDFLRRoundtrip(t *testing.T) { // Two separable synthetic "topics": optics vs cooking. docs := []string{ "quantum laser optics photon", "laser photon quantum measurement", "photon optics quantum laser", "optics quantum laser photon", "recipe cooking knife onion", "onion recipe cooking garlic", "cooking garlic onion skillet", "garlic recipe onion cooking", } labels := []float64{1, 1, 1, 1, 0, 0, 0, 0} v := &TFIDFVectorizer{NgramMin: 1, NgramMax: 2, MinDF: 1, MaxDF: 0.9} v.Fit(docs) if len(v.OrderedVocab) == 0 { t.Fatal("vectorizer learned empty vocabulary") } vecs := v.Transform(docs) lr := (&LogisticRegression{Lambda: 0.001}).Validate() weights, err := lr.Fit(vecs, labels, nil) if err != nil { t.Fatalf("Fit: %v", err) } // Persist and reload through the envelope, as serve does. model := &ModelEnvelope{ Vectorizer: v.Vocabulary, OrderedVocab: v.OrderedVocab, Weights: weights, } rv := CreateVectorizerFromModel(model) opticsScore := mustScore(t, rv, model, "quantum photon laser experiment") cookingScore := mustScore(t, rv, model, "onion garlic skillet dinner") if !(opticsScore > 0.5 && cookingScore < 0.5) { t.Fatalf("expected separation, got optics=%.3f cooking=%.3f", opticsScore, cookingScore) } if opticsScore <= cookingScore { t.Fatalf("expected optics score above cooking score, got %.3f <= %.3f", opticsScore, cookingScore) } } func mustScore(t *testing.T, v *TFIDFVectorizer, m *ModelEnvelope, title string) float64 { t.Helper() vec := v.Transform([]string{title}) s, err := PredictScore(vec[0], m.Weights) if err != nil { t.Fatalf("PredictScore(%q): %v", title, err) } return s } func TestTokenize(t *testing.T) { got := Tokenize("The Fall of the House of Usher!") want := []string{"fall", "house", "usher"} if len(got) != len(want) { t.Fatalf("Tokenize: got %v, want %v", got, want) } for i := range want { if got[i] != want[i] { t.Fatalf("Tokenize: got %v, want %v", got, want) } } }