fix: SVA chapters
This commit is contained in:
gosangam
+139
-42
@@ -1,15 +1,19 @@
|
||||
/**
|
||||
* seed-sva.ts — Push Songs of the Vaiṣṇava Ācāryas JSON data into MongoDB.
|
||||
*
|
||||
* Uses the native MongoDB driver (same pattern as seed-lcl.ts) to avoid
|
||||
* Uses the native MongoDB driver (same pattern as seed-ssr.ts) to avoid
|
||||
* Prisma replica-set write requirements.
|
||||
*
|
||||
* SVA is a prose/song book with 4 parts (chapters) plus intro segments.
|
||||
* We use volume=0 as the sentinel for "no volumes".
|
||||
* SVA is a song book with 4 parent parts, each containing multiple songs
|
||||
* (sub-chapters). Each song is stored as its own prose_chapters document.
|
||||
*
|
||||
* Sub-chapters are stored with slugs like "1-1", "1-2", "2-1", etc.
|
||||
* Each sub-chapter JSON has parentChapter and subChapter fields used to
|
||||
* compute a stable ordering: parentChapter * 100 + subChapter.
|
||||
*
|
||||
* Collections written:
|
||||
* - book_meta → one doc { book: "sva", data: <intro.json> }
|
||||
* - prose_chapters → intro segments + 4 chapters
|
||||
* - book_meta → one doc { book: "sva", data: <intro.json> }
|
||||
* - prose_chapters → 3 intro segments + N sub-chapters (songs)
|
||||
*
|
||||
* Usage:
|
||||
* npx tsx scripts/seed-sva.ts
|
||||
@@ -44,6 +48,40 @@ function readJson<T>(filePath: string): T {
|
||||
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively sanitize all strings in a document to prevent BSON encoding errors.
|
||||
*
|
||||
* Strategy: NFC-normalize every string, strip control characters and lone
|
||||
* surrogates, then round-trip through Buffer to guarantee the bytes are
|
||||
* well-formed UTF-8 that the MongoDB C driver can handle.
|
||||
*/
|
||||
function sanitizeDoc(obj: unknown): unknown {
|
||||
if (typeof obj === "string") {
|
||||
// 1. NFC normalize (composes combining characters)
|
||||
let s = obj.normalize("NFC");
|
||||
// 2. Remove null bytes and C0 control chars (keep \t \n \r)
|
||||
s = s.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, "");
|
||||
// 3. Remove lone surrogates (U+D800–U+DFFF) that are invalid in BSON
|
||||
s = s.replace(/[\uD800-\uDFFF]/g, "");
|
||||
// 4. Remove zero-width characters that can confuse BSON length calculations
|
||||
s = s.replace(/[\u200B-\u200F\u2028-\u202F\uFEFF]/g, "");
|
||||
// 5. Round-trip through Buffer to guarantee valid UTF-8 byte sequence
|
||||
s = Buffer.from(s, "utf-8").toString("utf-8");
|
||||
return s;
|
||||
}
|
||||
if (Array.isArray(obj)) {
|
||||
return obj.map(sanitizeDoc);
|
||||
}
|
||||
if (obj !== null && typeof obj === "object") {
|
||||
const result: Record<string, unknown> = {};
|
||||
for (const [key, value] of Object.entries(obj as Record<string, unknown>)) {
|
||||
result[key] = sanitizeDoc(value);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert documents one at a time with a small delay every N docs.
|
||||
* Uses insertOne to avoid BSON wire-protocol corruption.
|
||||
@@ -53,7 +91,36 @@ async function insertBatched(
|
||||
docs: Record<string, unknown>[],
|
||||
) {
|
||||
for (let i = 0; i < docs.length; i++) {
|
||||
await col.insertOne(docs[i]);
|
||||
const sanitized = sanitizeDoc(docs[i]) as Record<string, unknown>;
|
||||
|
||||
// Work around BSON driver bug with large string arrays:
|
||||
// Store paragraphs as a single JSON-encoded string instead of a native
|
||||
// array. The data layer (lib/sva.ts) already handles stringified JSON
|
||||
// via normalizeParagraphs().
|
||||
if (Array.isArray(sanitized.paragraphs)) {
|
||||
// Stringify, then sanitize the resulting string to ensure clean UTF-8
|
||||
let jsonStr = JSON.stringify(sanitized.paragraphs);
|
||||
// NFC normalize the whole JSON string
|
||||
jsonStr = jsonStr.normalize("NFC");
|
||||
// Remove any stray control chars or surrogates
|
||||
jsonStr = jsonStr.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, "");
|
||||
jsonStr = jsonStr.replace(/[\uD800-\uDFFF]/g, "");
|
||||
jsonStr = jsonStr.replace(/[\u200B-\u200F\u2028-\u202F\uFEFF]/g, "");
|
||||
// Round-trip through Buffer to guarantee valid UTF-8 byte sequence
|
||||
jsonStr = Buffer.from(jsonStr, "utf-8").toString("utf-8");
|
||||
sanitized.paragraphs = jsonStr;
|
||||
}
|
||||
|
||||
try {
|
||||
await col.insertOne(sanitized);
|
||||
} catch (err) {
|
||||
const slug = (sanitized.slug as string) || "unknown";
|
||||
const title = (sanitized.title as string) || "unknown";
|
||||
console.error(
|
||||
`\n ❌ Failed inserting doc ${i} (slug="${slug}", title="${title}")`,
|
||||
);
|
||||
throw err;
|
||||
}
|
||||
if ((i + 1) % 100 === 0) {
|
||||
await new Promise((r) => setTimeout(r, 100));
|
||||
}
|
||||
@@ -71,7 +138,11 @@ type SvaIntroJson = {
|
||||
year: number;
|
||||
description: string;
|
||||
totalChapters: number;
|
||||
chapters: { number: number; title: string }[];
|
||||
chapters: {
|
||||
number: number;
|
||||
title: string;
|
||||
subChapters: { slug: string; title: string }[];
|
||||
}[];
|
||||
};
|
||||
|
||||
type SvaIntroSegmentJson = {
|
||||
@@ -83,10 +154,13 @@ type SvaIntroSegmentJson = {
|
||||
paragraphs: string[];
|
||||
};
|
||||
|
||||
type SvaChapterJson = {
|
||||
type SvaSubChapterJson = {
|
||||
book: string;
|
||||
chapter: number | null;
|
||||
slug: string;
|
||||
parentChapter: number;
|
||||
subChapter: number;
|
||||
title: string;
|
||||
parentTitle: string;
|
||||
type: string;
|
||||
paragraphs: string[];
|
||||
};
|
||||
@@ -104,65 +178,88 @@ async function seedSva(db: ReturnType<MongoClient["db"]>) {
|
||||
|
||||
const intro = readJson<SvaIntroJson>(introPath);
|
||||
|
||||
// ---- Book meta ----
|
||||
// ---- 1. Book meta ----
|
||||
const metaCol = db.collection("book_meta");
|
||||
await metaCol.deleteMany({ book: "sva" });
|
||||
await metaCol.insertOne({ book: "sva", data: intro });
|
||||
console.log(" 📖 SVA book meta inserted");
|
||||
|
||||
// ---- Prose chapters collection ----
|
||||
// ---- 2. Prose chapters collection ----
|
||||
const proseCol = db.collection("prose_chapters");
|
||||
await proseCol.deleteMany({ book: "sva" });
|
||||
|
||||
const docs: Record<string, unknown>[] = [];
|
||||
|
||||
// Intro segments (front-pages, foreword, introduction)
|
||||
const introSegmentFiles = [
|
||||
"front-pages.json",
|
||||
"foreword.json",
|
||||
"introduction.json",
|
||||
];
|
||||
|
||||
const introDir = path.join(SVA_DIR, "intro");
|
||||
if (fs.existsSync(introDir)) {
|
||||
const introFiles = fs
|
||||
.readdirSync(introDir)
|
||||
.filter((f) => f.endsWith(".json"))
|
||||
.sort();
|
||||
|
||||
for (const file of introFiles) {
|
||||
const seg = readJson<SvaIntroSegmentJson>(path.join(introDir, file));
|
||||
docs.push({
|
||||
book: "sva",
|
||||
volume: NO_VOLUME,
|
||||
chapter: null,
|
||||
title: seg.title,
|
||||
type: seg.type || "intro-segment",
|
||||
slug: `intro-${seg.slug || file.replace(".json", "")}`,
|
||||
order: seg.order ?? 0,
|
||||
paragraphs: seg.paragraphs,
|
||||
});
|
||||
for (const file of introSegmentFiles) {
|
||||
const filePath = path.join(introDir, file);
|
||||
if (!fs.existsSync(filePath)) {
|
||||
console.warn(` ⚠️ Intro segment not found, skipping: ${file}`);
|
||||
continue;
|
||||
}
|
||||
console.log(` 📄 Intro segments: ${introFiles.length}`);
|
||||
}
|
||||
|
||||
// Chapters (01.json – 04.json)
|
||||
const seg = readJson<SvaIntroSegmentJson>(filePath);
|
||||
|
||||
docs.push({
|
||||
book: "sva",
|
||||
volume: NO_VOLUME,
|
||||
chapter: null,
|
||||
title: seg.title,
|
||||
type: "intro-segment",
|
||||
slug: `intro-${seg.slug}`,
|
||||
order: seg.order,
|
||||
paragraphs: seg.paragraphs,
|
||||
});
|
||||
}
|
||||
console.log(` 📄 Intro segments: ${docs.length}`);
|
||||
|
||||
// Sub-chapters (songs): 1-1.json, 1-2.json, 2-1.json, etc.
|
||||
const chaptersDir = path.join(SVA_DIR, "chapters");
|
||||
let songCount = 0;
|
||||
if (fs.existsSync(chaptersDir)) {
|
||||
const chapterFiles = fs
|
||||
.readdirSync(chaptersDir)
|
||||
.filter((f) => f.endsWith(".json"))
|
||||
.sort();
|
||||
.sort((a, b) => {
|
||||
// Sort by parentChapter then subChapter numerically
|
||||
const parseSlug = (name: string) => {
|
||||
const slug = name.replace(".json", "");
|
||||
const parts = slug.split("-");
|
||||
return {
|
||||
parent: parseInt(parts[0], 10),
|
||||
sub: parseInt(parts[1], 10),
|
||||
};
|
||||
};
|
||||
const aa = parseSlug(a);
|
||||
const bb = parseSlug(b);
|
||||
if (aa.parent !== bb.parent) return aa.parent - bb.parent;
|
||||
return aa.sub - bb.sub;
|
||||
});
|
||||
|
||||
for (const file of chapterFiles) {
|
||||
const ch = readJson<SvaChapterJson>(path.join(chaptersDir, file));
|
||||
const chNum = ch.chapter ?? parseInt(file.replace(".json", ""), 10);
|
||||
const sc = readJson<SvaSubChapterJson>(path.join(chaptersDir, file));
|
||||
const order = sc.parentChapter * 100 + sc.subChapter;
|
||||
|
||||
docs.push({
|
||||
book: "sva",
|
||||
volume: NO_VOLUME,
|
||||
chapter: chNum,
|
||||
title: ch.title,
|
||||
type: ch.type || "chapter",
|
||||
slug: String(chNum),
|
||||
order: 100 + chNum,
|
||||
paragraphs: ch.paragraphs,
|
||||
chapter: null, // null since slug is a string like "1-1"
|
||||
title: sc.title,
|
||||
type: "sub-chapter",
|
||||
slug: sc.slug,
|
||||
order,
|
||||
paragraphs: sc.paragraphs,
|
||||
});
|
||||
songCount++;
|
||||
}
|
||||
console.log(` 📖 Chapters: ${chapterFiles.length}`);
|
||||
console.log(` 🎵 Songs (sub-chapters): ${songCount}`);
|
||||
}
|
||||
|
||||
// Insert all docs
|
||||
|
||||
Reference in New Issue
Block a user