fix: sva now extracts and reads properly

This commit is contained in:
gosangam
2026-05-03 14:46:35 +05:30
parent 47a2dc375e
commit 89106efc98
2 changed files with 106 additions and 152 deletions
-18
View File
@@ -93,24 +93,6 @@ async function insertBatched(
for (let i = 0; i < docs.length; i++) {
const sanitized = sanitizeDoc(docs[i]) as Record<string, unknown>;
// Work around BSON driver bug with large string arrays:
// Store paragraphs as a single JSON-encoded string instead of a native
// array. The data layer (lib/sva.ts) already handles stringified JSON
// via normalizeParagraphs().
if (Array.isArray(sanitized.paragraphs)) {
// Stringify, then sanitize the resulting string to ensure clean UTF-8
let jsonStr = JSON.stringify(sanitized.paragraphs);
// NFC normalize the whole JSON string
jsonStr = jsonStr.normalize("NFC");
// Remove any stray control chars or surrogates
jsonStr = jsonStr.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, "");
jsonStr = jsonStr.replace(/[\uD800-\uDFFF]/g, "");
jsonStr = jsonStr.replace(/[\u200B-\u200F\u2028-\u202F\uFEFF]/g, "");
// Round-trip through Buffer to guarantee valid UTF-8 byte sequence
jsonStr = Buffer.from(jsonStr, "utf-8").toString("utf-8");
sanitized.paragraphs = jsonStr;
}
try {
await col.insertOne(sanitized);
} catch (err) {