1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
|
// Score command: debugging utility, one trained model vs. stdin titles.
//
// Reads one title per line on stdin, writes {"title", "score"} JSONL on
// stdout. Exists to make the vendored ML core exercisable end-to-end before
// the real commands (export/train/serve) land, and stays useful later for
// poking at model behavior by hand.
package cmds
import (
"bufio"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"fluxrec/core"
)
type ScoreCommand struct {
ModelPath string
}
func (c *ScoreCommand) Name() string { return "score" }
func (c *ScoreCommand) Init(args []string) error {
fs := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
fs.Usage = func() {
fmt.Fprint(fs.Output(), `Usage: fluxrec score --model MODEL
Score stdin titles (one per line) with a trained model, JSONL to stdout.
`)
fs.PrintDefaults()
}
fs.StringVar(&c.ModelPath, "model", "", "Path to model.json (required)")
if err := fs.Parse(args); err != nil {
return err
}
if c.ModelPath == "" {
return errors.New("--model flag is required")
}
clean := filepath.Clean(c.ModelPath)
if strings.Contains(clean, "..") {
return fmt.Errorf("invalid model path %q: must not contain '..'", c.ModelPath)
}
c.ModelPath = clean
return nil
}
type scoreResult struct {
Title string `json:"title"`
Score float64 `json:"score"`
}
func (c *ScoreCommand) Run(stdin io.Reader, stdout io.Writer) error {
model, err := loadModel(c.ModelPath)
if err != nil {
return err
}
vectorizer := core.CreateVectorizerFromModel(model)
enc := json.NewEncoder(stdout)
scanner := bufio.NewScanner(stdin)
scanner.Buffer(make([]byte, 64*1024), 1024*1024)
for scanner.Scan() {
title := strings.TrimSpace(scanner.Text())
if title == "" {
continue
}
if len(title) > 1000 {
return fmt.Errorf("title exceeds 1000 chars")
}
vec := vectorizer.Transform([]string{title})
score, err := core.PredictScore(vec[0], model.Weights)
if err != nil {
return fmt.Errorf("scoring %q: %w", title, err)
}
if err := enc.Encode(scoreResult{Title: title, Score: score}); err != nil {
return err
}
}
return scanner.Err()
}
func loadModel(path string) (*core.ModelEnvelope, error) {
f, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("opening model: %w", err)
}
defer func() { _ = f.Close() }()
var model core.ModelEnvelope
if err := json.NewDecoder(f).Decode(&model); err != nil {
return nil, fmt.Errorf("parsing model %s: %w", path, err)
}
if len(model.Weights) == 0 || len(model.OrderedVocab) == 0 {
return nil, fmt.Errorf("model %s is missing weights or vocabulary", path)
}
return &model, nil
}
|
