fix: sva now extracts and reads properly
This commit is contained in:
@@ -93,24 +93,6 @@ async function insertBatched(
|
||||
for (let i = 0; i < docs.length; i++) {
|
||||
const sanitized = sanitizeDoc(docs[i]) as Record<string, unknown>;
|
||||
|
||||
// Work around BSON driver bug with large string arrays:
|
||||
// Store paragraphs as a single JSON-encoded string instead of a native
|
||||
// array. The data layer (lib/sva.ts) already handles stringified JSON
|
||||
// via normalizeParagraphs().
|
||||
if (Array.isArray(sanitized.paragraphs)) {
|
||||
// Stringify, then sanitize the resulting string to ensure clean UTF-8
|
||||
let jsonStr = JSON.stringify(sanitized.paragraphs);
|
||||
// NFC normalize the whole JSON string
|
||||
jsonStr = jsonStr.normalize("NFC");
|
||||
// Remove any stray control chars or surrogates
|
||||
jsonStr = jsonStr.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, "");
|
||||
jsonStr = jsonStr.replace(/[\uD800-\uDFFF]/g, "");
|
||||
jsonStr = jsonStr.replace(/[\u200B-\u200F\u2028-\u202F\uFEFF]/g, "");
|
||||
// Round-trip through Buffer to guarantee valid UTF-8 byte sequence
|
||||
jsonStr = Buffer.from(jsonStr, "utf-8").toString("utf-8");
|
||||
sanitized.paragraphs = jsonStr;
|
||||
}
|
||||
|
||||
try {
|
||||
await col.insertOne(sanitized);
|
||||
} catch (err) {
|
||||
|
||||
Reference in New Issue
Block a user