From 96ab2fcb2ff442698465389d75390afa91629165 Mon Sep 17 00:00:00 2001
From: Sam Scholten
Date: Sun, 30 Aug 2026 12:50:24 +1000
Subject: fluxrec: personal Miniflux article recommender

---
 core/text.go | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100644 core/text.go

(limited to 'core/text.go')

diff --git a/core/text.go b/core/text.go
new file mode 100644
index 0000000..563abb3
--- /dev/null
+++ b/core/text.go
@@ -0,0 +1,38 @@
+// Vendored from scholscan/core/text.go (rev 7c1a5ef, 2026-08-28).
+// Owned here; no upstream sync.
+// Text processing for RSS feed content.
+// Used for web UI previews and search indexing - not ML (title-only scoring).
+package core
+
+import (
+	"regexp"
+	"strings"
+)
+
+// CleanFeedContent strips HTML, normalizes whitespace, truncates to 5KB
+func CleanFeedContent(content string) string {
+	if content == "" {
+		return ""
+	}
+
+	content = StripHTMLTags(content)
+	content = NormalizeSpace(content)
+
+	maxLength := 5000
+	if len(content) > maxLength {
+		content = content[:maxLength] + "..."
+	}
+
+	return content
+}
+
+// StripHTMLTags removes HTML tags
+func StripHTMLTags(content string) string {
+	re := regexp.MustCompile(`<[^>]*>`)
+	return re.ReplaceAllString(content, "")
+}
+
+// NormalizeSpace collapses whitespace and trims
+func NormalizeSpace(s string) string {
+	return strings.Join(strings.Fields(strings.TrimSpace(s)), " ")
+}
-- 
cgit v1.2.3