krispire/spsays-training
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: kb both volumes

Browse Source
This commit is contained in:
gosangam
2026-04-16 00:48:48 +05:30
parent abb7cc49d0
commit d88ababd76
15 changed files with 2298 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/seed-db.ts
+191 -27
View File
@@ -49,6 +49,26 @@ function readAllJsonInDir(dir: string): unknown[] {
.map((f) => readJson(path.join(dir, f)));
}
/**
* Insert documents one at a time with a small delay every N docs.
*
* Uses insertOne to avoid BSON wire-protocol corruption that occurs
* with insertMany on certain MongoDB server/proxy configurations.
* A brief pause every 100 docs prevents connection overload.
*/
async function insertBatched(
col: ReturnType<ReturnType<MongoClient["db"]>["collection"]>,
docs: Record<string, unknown>[],
) {
for (let i = 0; i < docs.length; i++) {
await col.insertOne(docs[i]);
// Pause briefly every 100 docs to give the connection breathing room
if ((i + 1) % 100 === 0) {
await new Promise((r) => setTimeout(r, 100));
}
}
}
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
@@ -95,6 +115,17 @@ type CcVerse = {
purport: string[];
};
type ProseChapterJson = {
book: string;
volume: number;
chapter?: number;
title: string;
type: string;
slug?: string;
order?: number;
paragraphs: string[];
};
type Sb1962Verse = {
book_title: string;
chapter: number;
@@ -178,9 +209,7 @@ async function seedBg(db: ReturnType<MongoClient["db"]>) {
}
}
if (docs.length > 0) {
await versesCol.insertMany(docs, { ordered: false });
}
await insertBatched(versesCol, docs);
console.log(` ✅ BG: ${docs.length} verses`);
}
@@ -227,14 +256,14 @@ async function seedSb(db: ReturnType<MongoClient["db"]>) {
.filter((d) => fs.statSync(path.join(chaptersDir, d)).isDirectory())
.sort();
const docs: Record<string, unknown>[] = [];
const cantoDocs: Record<string, unknown>[] = [];
for (const chDir of chapterDirs) {
const versesDir = path.join(chaptersDir, chDir, "verses");
const verses = readAllJsonInDir(versesDir) as SbVerse[];
for (const v of verses) {
docs.push({
cantoDocs.push({
book: "sb",
division: String(v.canto),
chapter: v.chapter,
@@ -252,11 +281,9 @@ async function seedSb(db: ReturnType<MongoClient["db"]>) {
}
}
if (docs.length > 0) {
await versesCol.insertMany(docs, { ordered: false });
}
totalCount += docs.length;
console.log(` Canto ${cantoDir}: ${docs.length} verses`);
await insertBatched(versesCol, cantoDocs);
totalCount += cantoDocs.length;
console.log(` Canto ${cantoDir}: ${cantoDocs.length} verses`);
}
console.log(` ✅ SB: ${totalCount} verses`);
@@ -305,14 +332,14 @@ async function seedCc(db: ReturnType<MongoClient["db"]>) {
.filter((d) => fs.statSync(path.join(chaptersDir, d)).isDirectory())
.sort();
const docs: Record<string, unknown>[] = [];
const lilaDocs: Record<string, unknown>[] = [];
for (const chDir of chapterDirs) {
const versesDir = path.join(chaptersDir, chDir, "verses");
const verses = readAllJsonInDir(versesDir) as CcVerse[];
for (const v of verses) {
docs.push({
lilaDocs.push({
book: "cc",
division: v.lila,
chapter: v.chapter,
@@ -330,11 +357,9 @@ async function seedCc(db: ReturnType<MongoClient["db"]>) {
}
}
if (docs.length > 0) {
await versesCol.insertMany(docs, { ordered: false });
}
totalCount += docs.length;
console.log(` Līlā ${lilaDir}: ${docs.length} verses`);
await insertBatched(versesCol, lilaDocs);
totalCount += lilaDocs.length;
console.log(` Līlā ${lilaDir}: ${lilaDocs.length} verses`);
}
console.log(` ✅ CC: ${totalCount} verses`);
@@ -397,12 +422,143 @@ async function seedSb1962(db: ReturnType<MongoClient["db"]>) {
}
}
if (docs.length > 0) {
await versesCol.insertMany(docs, { ordered: false });
}
await insertBatched(versesCol, docs);
console.log(` ✅ SB 1962: ${docs.length} verses`);
}
// ---------------------------------------------------------------------------
// Kṛṣṇa Book
// ---------------------------------------------------------------------------
async function seedKb(db: ReturnType<MongoClient["db"]>) {
const introPath = path.join(LIB_DIR, "books", "kb", "intro.json");
if (!fs.existsSync(introPath)) {
console.log(" ⏭ KB intro.json not found, skipping");
return;
}
const intro = readJson(introPath);
// Book meta
const metaCol = db.collection("book_meta");
await metaCol.deleteMany({ book: "kb" });
await metaCol.insertOne({ book: "kb", data: intro });
console.log(" 📖 KB book meta inserted");
// Prose chapters collection
const proseCol = db.collection("prose_chapters");
await proseCol.deleteMany({ book: "kb" });
const docs: Record<string, unknown>[] = [];
// Volume 1 intro segments
const vol1IntroDir = path.join(LIB_DIR, "books", "kb", "vol-1", "intro");
if (fs.existsSync(vol1IntroDir)) {
const introFiles = fs
.readdirSync(vol1IntroDir)
.filter((f) => f.endsWith(".json"))
.sort();
const introDocsV1: Record<string, unknown>[] = [];
for (const file of introFiles) {
const seg = readJson<ProseChapterJson>(path.join(vol1IntroDir, file));
introDocsV1.push({
book: "kb",
volume: seg.volume ?? 1,
chapter: seg.chapter ?? null,
title: seg.title,
type: seg.type || "intro-segment",
slug: seg.slug || file.replace(".json", ""),
order: seg.order ?? 0,
paragraphs: seg.paragraphs,
});
}
await insertBatched(proseCol, introDocsV1);
console.log(` Vol 1 intro segments: ${introFiles.length}`);
}
// Volume 1 chapters
const vol1ChapDir = path.join(LIB_DIR, "books", "kb", "vol-1", "chapters");
if (fs.existsSync(vol1ChapDir)) {
const chapterFiles = fs
.readdirSync(vol1ChapDir)
.filter((f) => f.endsWith(".json"))
.sort();
const chapDocsV1: Record<string, unknown>[] = [];
for (const file of chapterFiles) {
const ch = readJson<ProseChapterJson>(path.join(vol1ChapDir, file));
chapDocsV1.push({
book: "kb",
volume: ch.volume ?? 1,
chapter: ch.chapter ?? null,
title: ch.title,
type: ch.type || "chapter",
slug: String(ch.chapter ?? file.replace(".json", "")).padStart(2, "0"),
order: 100 + (ch.chapter ?? 0),
paragraphs: ch.paragraphs,
});
}
await insertBatched(proseCol, chapDocsV1);
console.log(` Vol 1 chapters: ${chapterFiles.length}`);
}
// Volume 2 intro segments
const vol2IntroDir = path.join(LIB_DIR, "books", "kb", "vol-2", "intro");
if (fs.existsSync(vol2IntroDir)) {
const introFiles = fs
.readdirSync(vol2IntroDir)
.filter((f) => f.endsWith(".json"))
.sort();
const introDocsV2: Record<string, unknown>[] = [];
for (const file of introFiles) {
const seg = readJson<ProseChapterJson>(path.join(vol2IntroDir, file));
introDocsV2.push({
book: "kb",
volume: seg.volume ?? 2,
chapter: seg.chapter ?? null,
title: seg.title,
type: seg.type || "intro-segment",
slug: seg.slug || file.replace(".json", ""),
order: seg.order ?? 0,
paragraphs: seg.paragraphs,
});
}
await insertBatched(proseCol, introDocsV2);
console.log(` Vol 2 intro segments: ${introFiles.length}`);
}
// Volume 2 chapters
const vol2ChapDir = path.join(LIB_DIR, "books", "kb", "vol-2", "chapters");
if (fs.existsSync(vol2ChapDir)) {
const chapterFiles = fs
.readdirSync(vol2ChapDir)
.filter((f) => f.endsWith(".json"))
.sort();
const chapDocsV2: Record<string, unknown>[] = [];
for (const file of chapterFiles) {
const ch = readJson<ProseChapterJson>(path.join(vol2ChapDir, file));
chapDocsV2.push({
book: "kb",
volume: ch.volume ?? 2,
chapter: ch.chapter ?? null,
title: ch.title,
type: ch.type || "chapter",
slug: String(ch.chapter ?? file.replace(".json", "")).padStart(2, "0"),
order: 100 + (ch.chapter ?? 0),
paragraphs: ch.paragraphs,
});
}
await insertBatched(proseCol, chapDocsV2);
console.log(` Vol 2 chapters: ${chapterFiles.length}`);
}
const totalDocs = await proseCol.countDocuments({ book: "kb" });
console.log(` ✅ KB: ${totalDocs} prose entries`);
}
// ---------------------------------------------------------------------------
// Glossary
// ---------------------------------------------------------------------------
@@ -419,16 +575,14 @@ async function seedGlossary(db: ReturnType<MongoClient["db"]>) {
const col = db.collection("glossary");
await col.deleteMany({});
const docs = data.entries.map((entry) => ({
const glossaryDocs = data.entries.map((entry) => ({
term: entry.term,
definition: entry.definition,
definitionHtml: entry.definition_html,
}));
if (docs.length > 0) {
await col.insertMany(docs, { ordered: false });
}
console.log(` ✅ Glossary: ${docs.length} entries`);
await insertBatched(col, glossaryDocs);
console.log(` ✅ Glossary: ${glossaryDocs.length} entries`);
}
// ---------------------------------------------------------------------------
@@ -453,6 +607,12 @@ async function createIndexes(db: ReturnType<MongoClient["db"]>) {
const glossaryCol = db.collection("glossary");
await glossaryCol.createIndex({ term: 1 }, { unique: true });
const proseCol = db.collection("prose_chapters");
await proseCol.createIndex({ book: 1 });
await proseCol.createIndex({ book: 1, volume: 1 });
await proseCol.createIndex({ book: 1, volume: 1, slug: 1 }, { unique: true });
await proseCol.createIndex({ book: 1, volume: 1, type: 1 });
console.log(" ✅ Indexes created");
}
@@ -487,11 +647,15 @@ async function main() {
await seedSb1962(db);
console.log();
console.log("[5/6] Glossary");
console.log("[5/7] Kṛṣṇa Book");
await seedKb(db);
console.log();
console.log("[6/7] Glossary");
await seedGlossary(db);
console.log();
console.log("[6/6] Indexes");
console.log("[7/7] Indexes");
await createIndexes(db);
console.log();