-
- Part {ch.number}
-
+ {/* Parts — hierarchical parent → song (sub-chapter) structure */}
+ {meta.chapters.map((ch) => (
+
+
+ Part {ch.number}: {ch.title}
+
+
+
+
+ {ch.subChapters.map((sub) => (
+
+
- {ch.title}
+ {sub.title}
-
-
-
- ))}
-
-
-
-
+
+
+ ))}
+
+
+
+
+ ))}
{/* Back Matter */}
{backMatter.length > 0 && (
diff --git a/lib/sva.ts b/lib/sva.ts
index df1ef201..d582a306 100644
--- a/lib/sva.ts
+++ b/lib/sva.ts
@@ -1,9 +1,10 @@
/**
* Data access layer for Songs of the Vaiṣṇava Ācāryas (SVA).
*
- * Reads from MongoDB via Prisma, same as the rest of the app.
- * SVA is a prose/song book with 4 parts (chapters) plus intro segments.
- * We use volume=0 as the sentinel for "no volumes".
+ * Reads from MongoDB via Prisma.
+ * SVA is a song book with 4 parent parts, each containing multiple songs
+ * (sub-chapters). Each song is stored as its own prose_chapters document
+ * with slugs like "1-1", "1-2", "2-1", etc.
*
* When the DB isn't seeded yet, falls back to reading local JSON files
* from lib/jsons/books/sva/.
@@ -22,11 +23,11 @@ import path from "path";
// Constants
// ---------------------------------------------------------------------------
+const BOOK = "sva";
+
/** Volume sentinel for books without volumes. */
const NO_VOLUME = 0;
-const BOOK = "sva";
-
const SVA_JSON_DIR = path.join(process.cwd(), "lib", "jsons", "books", "sva");
// ---------------------------------------------------------------------------
@@ -37,15 +38,21 @@ export type SvaMeta = {
book: string;
title: string;
author: string;
- year?: number;
+ year: number;
description: string;
totalChapters: number;
- chapters: SvaChapterEntry[];
+ chapters: SvaParentChapter[];
};
-export type SvaChapterEntry = {
+export type SvaParentChapter = {
number: number;
title: string;
+ subChapters: SvaSubChapterEntry[];
+};
+
+export type SvaSubChapterEntry = {
+ slug: string; // "1-1", "1-2", etc.
+ title: string;
};
export type SvaIntroSegment = {
@@ -59,27 +66,29 @@ export type SvaIntroSegment = {
export type SvaChapter = {
book: string;
- chapter: number | null;
+ chapter: string; // "1-1", "2-1", etc.
+ parentChapter: number;
+ subChapter: number;
title: string;
+ parentTitle: string;
type: string;
- slug?: string;
paragraphs: string[];
};
/** Union type used in sidebar / table-of-contents navigation. */
export type SvaSidebarItem = {
- /** URL-safe slug: "intro-front-pages", "intro-foreword", "1", "4" */
+ /** URL-safe slug: "intro-front-pages", "1-1", "parent-1", etc. */
slug: string;
/** Human-readable label for the sidebar */
label: string;
- /** "intro-segment" | "chapter" */
+ /** "intro-segment" | "parent" | "sub-chapter" */
type: string;
- /** Chapter number, or null for non-chapter items */
- chapter: number | null;
+ /** Parent chapter number, or null for intro segments */
+ parentChapter: number | null;
};
// ---------------------------------------------------------------------------
-// HTML sanitisation (mirrors lib/api.ts)
+// HTML sanitisation
// ---------------------------------------------------------------------------
function sanitizeHtml(html: string): string {
@@ -148,13 +157,29 @@ function getLocalIntroSegment(rawSlug: string): SvaIntroSegment | null {
return readLocalJson(filePath);
}
-function getLocalChapter(chapterNum: number): SvaChapter | null {
- const filePath = path.join(
- SVA_JSON_DIR,
- "chapters",
- String(chapterNum).padStart(2, "0") + ".json",
- );
- return readLocalJson(filePath);
+function getLocalSubChapter(slug: string): SvaChapter | null {
+ const filePath = path.join(SVA_JSON_DIR, "chapters", `${slug}.json`);
+ const raw = readLocalJson<{
+ book: string;
+ slug: string;
+ parentChapter: number;
+ subChapter: number;
+ title: string;
+ parentTitle: string;
+ type: string;
+ paragraphs: string[];
+ }>(filePath);
+ if (!raw) return null;
+ return {
+ book: raw.book,
+ chapter: raw.slug,
+ parentChapter: raw.parentChapter,
+ subChapter: raw.subChapter,
+ title: raw.title,
+ parentTitle: raw.parentTitle,
+ type: raw.type,
+ paragraphs: raw.paragraphs,
+ };
}
// ---------------------------------------------------------------------------
@@ -162,11 +187,15 @@ function getLocalChapter(chapterNum: number): SvaChapter | null {
// ---------------------------------------------------------------------------
/**
- * Book-level metadata (title, author, chapter list, etc.)
+ * Book-level metadata (title, author, chapter list with sub-chapters, etc.)
*/
export async function getSvaMeta(): Promise {
- const meta = await prisma.bookMeta.findUnique({ where: { book: BOOK } });
- if (meta) return meta.data as unknown as SvaMeta;
+ try {
+ const meta = await prisma.bookMeta.findUnique({ where: { book: BOOK } });
+ if (meta) return meta.data as unknown as SvaMeta;
+ } catch {
+ // Prisma BSON decode error — fall through to local JSON
+ }
// Fallback to local JSON
const introPath = path.join(SVA_JSON_DIR, "intro.json");
@@ -181,20 +210,24 @@ export async function getSvaMeta(): Promise {
* All intro segments, ordered by their `order` field.
*/
export async function getSvaIntroSegments(): Promise {
- const results = await prisma.proseChapter.findMany({
- where: { book: BOOK, volume: NO_VOLUME, type: "intro-segment" },
- orderBy: { order: "asc" },
- });
+ try {
+ const results = await prisma.proseChapter.findMany({
+ where: { book: BOOK, volume: NO_VOLUME, type: "intro-segment" },
+ orderBy: { order: "asc" },
+ });
- if (results.length > 0) {
- return results.map((doc) => ({
- slug: doc.slug,
- title: doc.title,
- type: doc.type,
- book: doc.book,
- order: doc.order,
- paragraphs: normalizeParagraphs(doc.paragraphs),
- }));
+ if (results.length > 0) {
+ return results.map((doc) => ({
+ slug: doc.slug,
+ title: doc.title,
+ type: doc.type,
+ book: doc.book,
+ order: doc.order,
+ paragraphs: normalizeParagraphs(doc.paragraphs),
+ }));
+ }
+ } catch {
+ // Prisma BSON decode error — fall through to local JSON
}
// Fallback to local JSON
@@ -203,14 +236,16 @@ export async function getSvaIntroSegments(): Promise {
/**
* A single intro segment by slug.
- * Accepts both "intro-foreword" (URL slug) and "foreword" (raw slug).
+ * Accepts both "intro-front-pages" (URL slug) and "front-pages" (raw slug).
*/
export async function getSvaIntroSegment(
slug: string,
): Promise {
// DB stores with "intro-" prefix; accept either form
const dbSlug = slug.startsWith("intro-") ? slug : `intro-${slug}`;
- const rawSlug = slug.startsWith("intro-") ? slug.replace(/^intro-/, "") : slug;
+ const rawSlug = slug.startsWith("intro-")
+ ? slug.replace(/^intro-/, "")
+ : slug;
const doc = await prisma.proseChapter.findUnique({
where: {
@@ -240,42 +275,54 @@ export async function getSvaIntroSegment(
}
// ---------------------------------------------------------------------------
-// Chapters
+// Chapters (sub-chapters like "1-1", "2-3", etc.)
// ---------------------------------------------------------------------------
/**
- * A numbered chapter (1–4).
+ * A sub-chapter (song) by its slug (e.g. "1-1", "2-3").
*/
-export async function getSvaChapter(
- chapterNum: number,
-): Promise {
- if (chapterNum < 1 || chapterNum > 4 || !Number.isInteger(chapterNum)) {
- return null;
- }
-
- const doc = await prisma.proseChapter.findUnique({
- where: {
- book_volume_slug: {
- book: BOOK,
- volume: NO_VOLUME,
- slug: String(chapterNum),
+export async function getSvaChapter(slug: string): Promise {
+ try {
+ const doc = await prisma.proseChapter.findUnique({
+ where: {
+ book_volume_slug: {
+ book: BOOK,
+ volume: NO_VOLUME,
+ slug,
+ },
},
- },
- });
+ });
- if (doc) {
- return {
- book: doc.book,
- chapter: doc.chapter,
- title: doc.title,
- type: doc.type,
- slug: doc.slug,
- paragraphs: normalizeParagraphs(doc.paragraphs),
- };
+ if (doc) {
+ // Parse parentChapter and subChapter from slug "X-Y"
+ const parts = slug.split("-");
+ const parentChapter = parseInt(parts[0], 10);
+ const subChapter = parseInt(parts[1], 10);
+
+ // Derive parentTitle from meta if available
+ const meta = await getSvaMeta();
+ const parentEntry = meta?.chapters.find(
+ (ch) => ch.number === parentChapter,
+ );
+ const parentTitle = parentEntry?.title ?? "";
+
+ return {
+ book: doc.book,
+ chapter: slug,
+ parentChapter,
+ subChapter,
+ title: doc.title,
+ parentTitle,
+ type: doc.type,
+ paragraphs: normalizeParagraphs(doc.paragraphs),
+ };
+ }
+ } catch {
+ // Prisma BSON decode error — fall through to local JSON
}
// Fallback to local JSON
- const local = getLocalChapter(chapterNum);
+ const local = getLocalSubChapter(slug);
if (!local) return null;
return {
...local,
@@ -290,10 +337,10 @@ export async function getSvaChapter(
/**
* Get content by URL slug.
*
- * "intro-front-pages" → intro segment
- * "intro-foreword" → intro segment
- * "intro-introduction" → intro segment
- * "1" … "4" → chapter (part)
+ * "intro-front-pages" → intro segment
+ * "intro-foreword" → intro segment
+ * "intro-introduction" → intro segment
+ * "1-1", "2-3", etc. → sub-chapter (song)
*/
export async function getSvaContent(
slug: string,
@@ -302,12 +349,8 @@ export async function getSvaContent(
return getSvaIntroSegment(slug);
}
- const num = Number(slug);
- if (!Number.isNaN(num) && Number.isInteger(num) && num >= 1 && num <= 4) {
- return getSvaChapter(num);
- }
-
- return null;
+ // Sub-chapter slug like "1-1", "2-3"
+ return getSvaChapter(slug);
}
// ---------------------------------------------------------------------------
@@ -315,131 +358,172 @@ export async function getSvaContent(
// ---------------------------------------------------------------------------
/**
- * Build the full sidebar navigation list.
+ * Build the full sidebar navigation list with hierarchical structure.
*
- * Order: front-pages → foreword → introduction → parts 1–4
+ * Order: front-pages → foreword → introduction
+ * → parent-1 (label) → 1-1, 1-2, ... → parent-2 (label) → 2-1, 2-2, ...
+ *
+ * "parent" type items are non-clickable headings. Only "intro-segment" and
+ * "sub-chapter" items are navigable pages.
*/
export async function getSvaSidebarItems(): Promise {
- const [meta, allDocs] = await Promise.all([
- getSvaMeta(),
- prisma.proseChapter.findMany({
+ let allDocs: {
+ slug: string;
+ title: string;
+ type: string;
+ chapter: number | null;
+ order: number;
+ }[] = [];
+
+ const meta = await getSvaMeta();
+
+ try {
+ allDocs = await prisma.proseChapter.findMany({
where: { book: BOOK, volume: NO_VOLUME },
orderBy: { order: "asc" },
- select: { slug: true, title: true, type: true, chapter: true, order: true },
- }),
- ]);
-
- // If we got DB results, build from those
- if (meta && allDocs.length > 0) {
- return buildSidebarFromDocs(meta, allDocs);
+ select: {
+ slug: true,
+ title: true,
+ type: true,
+ chapter: true,
+ order: true,
+ },
+ });
+ } catch {
+ // Prisma BSON decode error — build sidebar from meta alone
}
- // Fallback: build from local JSON
- return buildSidebarFromLocal(meta);
-}
+ if (!meta) {
+ // Try building from local JSON
+ return buildSidebarFromLocal();
+ }
-function buildSidebarFromDocs(
- meta: SvaMeta,
- allDocs: { slug: string; title: string; type: string; chapter: number | null; order: number | null }[],
-): SvaSidebarItem[] {
const items: SvaSidebarItem[] = [];
// Intro segments that come BEFORE chapters (order < 99)
- for (const doc of allDocs) {
- if (doc.type === "intro-segment" && (doc.order ?? 0) < 99) {
+ if (allDocs.length > 0) {
+ for (const doc of allDocs) {
+ if (doc.type === "intro-segment" && (doc.order ?? 0) < 99) {
+ items.push({
+ slug: doc.slug,
+ label: doc.title,
+ type: "intro-segment",
+ parentChapter: null,
+ });
+ }
+ }
+ } else {
+ // Fallback: local intro segments
+ const localIntros = getLocalIntroSegments();
+ for (const seg of localIntros) {
+ if ((seg.order ?? 0) < 99) {
+ items.push({
+ slug: `intro-${seg.slug}`,
+ label: seg.title,
+ type: "intro-segment",
+ parentChapter: null,
+ });
+ }
+ }
+ }
+
+ // Chapters — grouped by parent chapter with sub-chapters
+ for (const ch of meta.chapters) {
+ // Parent heading (non-clickable label)
+ items.push({
+ slug: `parent-${ch.number}`,
+ label: `Part ${ch.number}: ${ch.title}`,
+ type: "parent",
+ parentChapter: ch.number,
+ });
+
+ // Sub-chapters (songs) under this parent
+ for (const sub of ch.subChapters) {
items.push({
- slug: doc.slug,
- label: doc.title,
- type: "intro-segment",
- chapter: null,
+ slug: sub.slug,
+ label: sub.title,
+ type: "sub-chapter",
+ parentChapter: ch.number,
});
}
}
- // Chapters 1–4 (use meta.chapters for consistent title/ordering)
- for (const ch of meta.chapters) {
- items.push({
- slug: String(ch.number),
- label: `Part ${ch.number}: ${ch.title}`,
- type: "chapter",
- chapter: ch.number,
- });
- }
-
// Intro segments that come AFTER chapters (order >= 99)
- for (const doc of allDocs) {
- if (doc.type === "intro-segment" && (doc.order ?? 0) >= 99) {
- items.push({
- slug: doc.slug,
- label: doc.title,
- type: "intro-segment",
- chapter: null,
- });
+ if (allDocs.length > 0) {
+ for (const doc of allDocs) {
+ if (doc.type === "intro-segment" && (doc.order ?? 0) >= 99) {
+ items.push({
+ slug: doc.slug,
+ label: doc.title,
+ type: "intro-segment",
+ parentChapter: null,
+ });
+ }
}
}
return items;
}
-function buildSidebarFromLocal(meta: SvaMeta | null): SvaSidebarItem[] {
+function buildSidebarFromLocal(): SvaSidebarItem[] {
const items: SvaSidebarItem[] = [];
- // Local intro segments (front matter)
+ // Intro segments
const introSegs = getLocalIntroSegments();
- const frontMatter = introSegs.filter((s) => (s.order ?? 0) < 99);
- const backMatter = introSegs.filter((s) => (s.order ?? 0) >= 99);
-
- for (const seg of frontMatter) {
- items.push({
- slug: `intro-${seg.slug}`,
- label: seg.title,
- type: "intro-segment",
- chapter: null,
- });
- }
-
- // Chapters from meta or generate default 1-4
- if (meta && meta.chapters) {
- for (const ch of meta.chapters) {
+ for (const seg of introSegs) {
+ if ((seg.order ?? 0) < 99) {
items.push({
- slug: String(ch.number),
- label: `Part ${ch.number}: ${ch.title}`,
- type: "chapter",
- chapter: ch.number,
+ slug: `intro-${seg.slug}`,
+ label: seg.title,
+ type: "intro-segment",
+ parentChapter: null,
});
}
- } else {
- // Fallback: scan local chapter JSONs
- const chaptersDir = path.join(SVA_JSON_DIR, "chapters");
- if (fs.existsSync(chaptersDir)) {
- const files = fs
- .readdirSync(chaptersDir)
- .filter((f) => f.endsWith(".json"))
- .sort();
- for (const file of files) {
- const ch = readLocalJson(path.join(chaptersDir, file));
- if (ch) {
- const num = ch.chapter ?? parseInt(file.replace(".json", ""), 10);
- items.push({
- slug: String(num),
- label: `Part ${num}: ${ch.title}`,
- type: "chapter",
- chapter: num,
- });
- }
+ }
+
+ // Load meta for chapter structure
+ const introPath = path.join(SVA_JSON_DIR, "intro.json");
+ const meta = readLocalJson(introPath);
+
+ if (meta?.chapters) {
+ for (const ch of meta.chapters) {
+ items.push({
+ slug: `parent-${ch.number}`,
+ label: `Part ${ch.number}: ${ch.title}`,
+ type: "parent",
+ parentChapter: ch.number,
+ });
+
+ for (const sub of ch.subChapters) {
+ items.push({
+ slug: sub.slug,
+ label: sub.title,
+ type: "sub-chapter",
+ parentChapter: ch.number,
+ });
}
}
}
// Back matter
- for (const seg of backMatter) {
- items.push({
- slug: `intro-${seg.slug}`,
- label: seg.title,
- type: "intro-segment",
- chapter: null,
- });
+ for (const seg of introSegs) {
+ if ((seg.order ?? 0) >= 99) {
+ items.push({
+ slug: `intro-${seg.slug}`,
+ label: seg.title,
+ type: "intro-segment",
+ parentChapter: null,
+ });
+ }
}
return items;
}
+
+/**
+ * Get the flat list of navigable sidebar items (excludes "parent" headings).
+ * Used for prev/next navigation.
+ */
+export function getNavigableItems(items: SvaSidebarItem[]): SvaSidebarItem[] {
+ return items.filter((item) => item.type !== "parent");
+}
diff --git a/scripts/__pycache__/sva_extract.cpython-313.pyc b/scripts/__pycache__/sva_extract.cpython-313.pyc
new file mode 100644
index 00000000..e4ca5ac1
Binary files /dev/null and b/scripts/__pycache__/sva_extract.cpython-313.pyc differ
diff --git a/scripts/seed-sva.ts b/scripts/seed-sva.ts
index 85b8fd48..512650e0 100644
--- a/scripts/seed-sva.ts
+++ b/scripts/seed-sva.ts
@@ -1,15 +1,19 @@
/**
* seed-sva.ts — Push Songs of the Vaiṣṇava Ācāryas JSON data into MongoDB.
*
- * Uses the native MongoDB driver (same pattern as seed-lcl.ts) to avoid
+ * Uses the native MongoDB driver (same pattern as seed-ssr.ts) to avoid
* Prisma replica-set write requirements.
*
- * SVA is a prose/song book with 4 parts (chapters) plus intro segments.
- * We use volume=0 as the sentinel for "no volumes".
+ * SVA is a song book with 4 parent parts, each containing multiple songs
+ * (sub-chapters). Each song is stored as its own prose_chapters document.
+ *
+ * Sub-chapters are stored with slugs like "1-1", "1-2", "2-1", etc.
+ * Each sub-chapter JSON has parentChapter and subChapter fields used to
+ * compute a stable ordering: parentChapter * 100 + subChapter.
*
* Collections written:
- * - book_meta → one doc { book: "sva", data: }
- * - prose_chapters → intro segments + 4 chapters
+ * - book_meta → one doc { book: "sva", data: }
+ * - prose_chapters → 3 intro segments + N sub-chapters (songs)
*
* Usage:
* npx tsx scripts/seed-sva.ts
@@ -44,6 +48,40 @@ function readJson(filePath: string): T {
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
}
+/**
+ * Recursively sanitize all strings in a document to prevent BSON encoding errors.
+ *
+ * Strategy: NFC-normalize every string, strip control characters and lone
+ * surrogates, then round-trip through Buffer to guarantee the bytes are
+ * well-formed UTF-8 that the MongoDB C driver can handle.
+ */
+function sanitizeDoc(obj: unknown): unknown {
+ if (typeof obj === "string") {
+ // 1. NFC normalize (composes combining characters)
+ let s = obj.normalize("NFC");
+ // 2. Remove null bytes and C0 control chars (keep \t \n \r)
+ s = s.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, "");
+ // 3. Remove lone surrogates (U+D800–U+DFFF) that are invalid in BSON
+ s = s.replace(/[\uD800-\uDFFF]/g, "");
+ // 4. Remove zero-width characters that can confuse BSON length calculations
+ s = s.replace(/[\u200B-\u200F\u2028-\u202F\uFEFF]/g, "");
+ // 5. Round-trip through Buffer to guarantee valid UTF-8 byte sequence
+ s = Buffer.from(s, "utf-8").toString("utf-8");
+ return s;
+ }
+ if (Array.isArray(obj)) {
+ return obj.map(sanitizeDoc);
+ }
+ if (obj !== null && typeof obj === "object") {
+ const result: Record = {};
+ for (const [key, value] of Object.entries(obj as Record)) {
+ result[key] = sanitizeDoc(value);
+ }
+ return result;
+ }
+ return obj;
+}
+
/**
* Insert documents one at a time with a small delay every N docs.
* Uses insertOne to avoid BSON wire-protocol corruption.
@@ -53,7 +91,36 @@ async function insertBatched(
docs: Record[],
) {
for (let i = 0; i < docs.length; i++) {
- await col.insertOne(docs[i]);
+ const sanitized = sanitizeDoc(docs[i]) as Record;
+
+ // Work around BSON driver bug with large string arrays:
+ // Store paragraphs as a single JSON-encoded string instead of a native
+ // array. The data layer (lib/sva.ts) already handles stringified JSON
+ // via normalizeParagraphs().
+ if (Array.isArray(sanitized.paragraphs)) {
+ // Stringify, then sanitize the resulting string to ensure clean UTF-8
+ let jsonStr = JSON.stringify(sanitized.paragraphs);
+ // NFC normalize the whole JSON string
+ jsonStr = jsonStr.normalize("NFC");
+ // Remove any stray control chars or surrogates
+ jsonStr = jsonStr.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, "");
+ jsonStr = jsonStr.replace(/[\uD800-\uDFFF]/g, "");
+ jsonStr = jsonStr.replace(/[\u200B-\u200F\u2028-\u202F\uFEFF]/g, "");
+ // Round-trip through Buffer to guarantee valid UTF-8 byte sequence
+ jsonStr = Buffer.from(jsonStr, "utf-8").toString("utf-8");
+ sanitized.paragraphs = jsonStr;
+ }
+
+ try {
+ await col.insertOne(sanitized);
+ } catch (err) {
+ const slug = (sanitized.slug as string) || "unknown";
+ const title = (sanitized.title as string) || "unknown";
+ console.error(
+ `\n ❌ Failed inserting doc ${i} (slug="${slug}", title="${title}")`,
+ );
+ throw err;
+ }
if ((i + 1) % 100 === 0) {
await new Promise((r) => setTimeout(r, 100));
}
@@ -71,7 +138,11 @@ type SvaIntroJson = {
year: number;
description: string;
totalChapters: number;
- chapters: { number: number; title: string }[];
+ chapters: {
+ number: number;
+ title: string;
+ subChapters: { slug: string; title: string }[];
+ }[];
};
type SvaIntroSegmentJson = {
@@ -83,10 +154,13 @@ type SvaIntroSegmentJson = {
paragraphs: string[];
};
-type SvaChapterJson = {
+type SvaSubChapterJson = {
book: string;
- chapter: number | null;
+ slug: string;
+ parentChapter: number;
+ subChapter: number;
title: string;
+ parentTitle: string;
type: string;
paragraphs: string[];
};
@@ -104,65 +178,88 @@ async function seedSva(db: ReturnType) {
const intro = readJson(introPath);
- // ---- Book meta ----
+ // ---- 1. Book meta ----
const metaCol = db.collection("book_meta");
await metaCol.deleteMany({ book: "sva" });
await metaCol.insertOne({ book: "sva", data: intro });
console.log(" 📖 SVA book meta inserted");
- // ---- Prose chapters collection ----
+ // ---- 2. Prose chapters collection ----
const proseCol = db.collection("prose_chapters");
await proseCol.deleteMany({ book: "sva" });
const docs: Record[] = [];
// Intro segments (front-pages, foreword, introduction)
+ const introSegmentFiles = [
+ "front-pages.json",
+ "foreword.json",
+ "introduction.json",
+ ];
+
const introDir = path.join(SVA_DIR, "intro");
- if (fs.existsSync(introDir)) {
- const introFiles = fs
- .readdirSync(introDir)
- .filter((f) => f.endsWith(".json"))
- .sort();
-
- for (const file of introFiles) {
- const seg = readJson(path.join(introDir, file));
- docs.push({
- book: "sva",
- volume: NO_VOLUME,
- chapter: null,
- title: seg.title,
- type: seg.type || "intro-segment",
- slug: `intro-${seg.slug || file.replace(".json", "")}`,
- order: seg.order ?? 0,
- paragraphs: seg.paragraphs,
- });
+ for (const file of introSegmentFiles) {
+ const filePath = path.join(introDir, file);
+ if (!fs.existsSync(filePath)) {
+ console.warn(` ⚠️ Intro segment not found, skipping: ${file}`);
+ continue;
}
- console.log(` 📄 Intro segments: ${introFiles.length}`);
- }
- // Chapters (01.json – 04.json)
+ const seg = readJson(filePath);
+
+ docs.push({
+ book: "sva",
+ volume: NO_VOLUME,
+ chapter: null,
+ title: seg.title,
+ type: "intro-segment",
+ slug: `intro-${seg.slug}`,
+ order: seg.order,
+ paragraphs: seg.paragraphs,
+ });
+ }
+ console.log(` 📄 Intro segments: ${docs.length}`);
+
+ // Sub-chapters (songs): 1-1.json, 1-2.json, 2-1.json, etc.
const chaptersDir = path.join(SVA_DIR, "chapters");
+ let songCount = 0;
if (fs.existsSync(chaptersDir)) {
const chapterFiles = fs
.readdirSync(chaptersDir)
.filter((f) => f.endsWith(".json"))
- .sort();
+ .sort((a, b) => {
+ // Sort by parentChapter then subChapter numerically
+ const parseSlug = (name: string) => {
+ const slug = name.replace(".json", "");
+ const parts = slug.split("-");
+ return {
+ parent: parseInt(parts[0], 10),
+ sub: parseInt(parts[1], 10),
+ };
+ };
+ const aa = parseSlug(a);
+ const bb = parseSlug(b);
+ if (aa.parent !== bb.parent) return aa.parent - bb.parent;
+ return aa.sub - bb.sub;
+ });
for (const file of chapterFiles) {
- const ch = readJson(path.join(chaptersDir, file));
- const chNum = ch.chapter ?? parseInt(file.replace(".json", ""), 10);
+ const sc = readJson(path.join(chaptersDir, file));
+ const order = sc.parentChapter * 100 + sc.subChapter;
+
docs.push({
book: "sva",
volume: NO_VOLUME,
- chapter: chNum,
- title: ch.title,
- type: ch.type || "chapter",
- slug: String(chNum),
- order: 100 + chNum,
- paragraphs: ch.paragraphs,
+ chapter: null, // null since slug is a string like "1-1"
+ title: sc.title,
+ type: "sub-chapter",
+ slug: sc.slug,
+ order,
+ paragraphs: sc.paragraphs,
});
+ songCount++;
}
- console.log(` 📖 Chapters: ${chapterFiles.length}`);
+ console.log(` 🎵 Songs (sub-chapters): ${songCount}`);
}
// Insert all docs
diff --git a/scripts/sva_extract.py b/scripts/sva_extract.py
index 7e381cd1..8a0496b6 100644
--- a/scripts/sva_extract.py
+++ b/scripts/sva_extract.py
@@ -1,13 +1,21 @@
#!/usr/bin/env python3
"""Extract Songs of the Vaiṣṇava Ācāryas (1972) from other.html.
-The book has 3 intro segments (front-pages, foreword, introduction),
-and 4 chapters (Parts 1–4).
+Hierarchical sub-chapter extraction: each song within a part becomes its OWN
+chapter JSON file with a parent-child relationship.
+
+Parent chapters (Parts 1–4) are containers only — no content of their own.
+
+Numbering: Part 1, Song 1 → slug "1-1", Part 2, Song 3 → slug "2-3", etc.
+
+Song boundaries are delimited by p49 bookmarks with text like:
+ "SVA 1: Śrī Guru Praṇāma"
+ "SVA 2: Aruṇodaya-kīrtana"
Output:
- lib/jsons/books/sva/intro.json — Book metadata
- lib/jsons/books/sva/intro/*.json — Intro segments
- lib/jsons/books/sva/chapters/*.json — Chapters (01–04)
+ lib/jsons/books/sva/intro.json — Book metadata with hierarchical chapters
+ lib/jsons/books/sva/intro/*.json — Intro segments (front-pages, foreword, introduction)
+ lib/jsons/books/sva/chapters/*.json — Sub-chapter files (1-1.json, 1-2.json, 2-1.json, ...)
Usage:
python3 scripts/sva_extract.py
@@ -17,6 +25,7 @@ import json
import os
import re
import sys
+from collections import defaultdict
from bs4 import BeautifulSoup, NavigableString, Tag
@@ -39,13 +48,17 @@ SCAGOUDY_TO_IAST = {
"ç": "ś",
"Ç": "Ś",
"ì": "ṅ",
+ "Ì": "Ṅ",
"ï": "ñ",
+ "Ï": "Ñ",
"ü": "ū",
"Ü": "Ū",
"é": "ī",
"É": "Ī",
"à": "ṁ",
+ "À": "Ṁ",
"ù": "ḥ",
+ "Ù": "Ḥ",
"è": "ṝ",
"û": "ṁl",
"ÿ": "ḷ",
@@ -69,6 +82,18 @@ def decode_html_content(tag: Tag) -> str:
)
# Clean up Apple-tab-span leftovers (tab characters)
inner = inner.replace("\t", " ")
+ # Force valid UTF-8 by round-tripping through encode/decode with replacement
+ inner = inner.encode("utf-8", errors="replace").decode("utf-8", errors="replace")
+ # Strip null bytes and other control characters that break BSON
+ inner = re.sub(r"[\x00-\x08\x0b\x0c\x0e-\x1f]", "", inner)
+ # Remove lone surrogates (U+D800–U+DFFF) that are invalid in BSON/MongoDB
+ inner = re.sub(r"[\ud800-\udfff]", "", inner)
+ # Merge adjacent bold tags: → (nothing)
+ for _ in range(5):
+ inner = re.sub(r"\s*", "", inner)
+ # Remove empty bold and italic tags
+ inner = re.sub(r"\s*", "", inner)
+ inner = re.sub(r"\s*", "", inner)
# Normalize whitespace
inner = re.sub(r"\s+", " ", inner).strip()
return inner
@@ -138,10 +163,16 @@ SONG_TITLE_CLASS = "p27"
SONG_TITLE_VARIANT_CLASS = "p169"
# Verse line classes (italic, centered)
-VERSE_PADA_CLASSES = frozenset(["p5", "p53", "p24", "p66", "p255"])
+VERSE_PADA_CLASSES = frozenset(["p5", "p53", "p24", "p66", "p255", "p134"])
# Verse number classes (e.g. "3)", "4)", "[1]") — centered but NOT verse text
-VERSE_NUMBER_CLASS = "p248"
+VERSE_NUMBER_CLASSES = frozenset(["p248", "p30"])
+
+# Translation variant classes (bold, centered) — in addition to p56
+TRANSLATION_VARIANT_CLASSES = frozenset(["p86", "p240", "p242"])
+
+# Reference/purport note classes that should be centered
+CENTERED_NOTE_CLASSES = frozenset(["p241", "p22"])
# Synonyms text
SYNONYMS_CLASS = "p54"
@@ -246,6 +277,18 @@ SKIP_CLASSES = frozenset(
)
+# ---------------------------------------------------------------------------
+# Clean parent titles (pre-decoded IAST)
+# ---------------------------------------------------------------------------
+
+CLEAN_PARENT_TITLES = {
+ 1: "Prayers and Praṇāmas",
+ 2: "Songs of Śrīla Bhaktivinoda Ṭhākura",
+ 3: "Songs of Śrīla Narottama dāsa Ṭhākura",
+ 4: "Songs by other Ācāryas",
+}
+
+
# ---------------------------------------------------------------------------
# Extraction helpers
# ---------------------------------------------------------------------------
@@ -337,12 +380,12 @@ def process_table_t3(table: Tag) -> list:
joined = " ".join(verse_lines)
result.append(f'
{joined}
')
- # Right cell: English translation
+ # Right cell: English translation — centered
right_ps = tds[1].find_all("p")
for p in right_ps:
html = decode_html_content(p)
if html and html.strip() and html.strip() != "\xa0":
- result.append(html)
+ result.append(f'
{html}
')
return result
@@ -367,7 +410,7 @@ def extract_paragraphs(
start_idx: int,
end_idx: int,
) -> list:
- """Extract paragraphs for a section (chapter, intro segment).
+ """Extract paragraphs for a section (song, intro segment).
Handles all CSS classes within the SVA content.
Handles
and
elements.
@@ -460,6 +503,19 @@ def extract_paragraphs(
# Non-verse paragraph — flush any accumulated verse first
flush_verse()
+ # Short p6 content (verse numbers like "1)", "2)") should be centered
+ # Long p6 content (body paragraphs) stays left-aligned
+ if cls == "p6":
+ html = decode_html_content(elem)
+ if html:
+ plain = re.sub(r"<[^>]+>", "", html).strip()
+ # Short content that looks like a verse number or bracketed ref
+ if len(plain) <= 10 and re.match(r"^[\[\(]?\d+[\]\)]?\.?$", plain):
+ result.append(f'
{html}
')
+ else:
+ result.append(html)
+ continue
+
# Front pages text (p18)
if cls == FRONT_PAGES_CLASS:
html = decode_html_content(elem)
@@ -467,29 +523,29 @@ def extract_paragraphs(
result.append(html)
continue
- # Section heading (p31) — bold
+ # Section heading (p31) — bold, centered
if cls == SECTION_HEADING_CLASS:
html = decode_html_content(elem)
if html:
if not html.strip().startswith(""):
html = f"{html}"
- result.append(html)
+ result.append(f'
{html}
')
continue
- # Song title (p27) — bold
+ # Song title (p27) — bold, centered
if cls == SONG_TITLE_CLASS:
html = decode_html_content(elem)
if html:
if not html.strip().startswith(""):
html = f"{html}"
- result.append(html)
+ result.append(f'
{html}
')
continue
- # Song title variant (p169) — bold italic with author
+ # Song title variant (p169) — bold italic with author, centered
if cls == SONG_TITLE_VARIANT_CLASS:
html = decode_html_content(elem)
if html:
- result.append(html)
+ result.append(f'
{html}
')
continue
# Synonyms (p54)
@@ -499,8 +555,25 @@ def extract_paragraphs(
result.append(html)
continue
- # Verse number heading (p248) — e.g. "3)", "4)", "[1]" — center it
- if cls == VERSE_NUMBER_CLASS:
+ # Verse number heading (p248, p30) — e.g. "3)", "4)", "[1]" — center it
+ if cls in VERSE_NUMBER_CLASSES:
+ flush_verse()
+ html = decode_html_content(elem)
+ if html:
+ result.append(f'
{html}
')
+ continue
+
+ # Translation variant classes (p86, p240, p242) — bold, centered
+ if cls in TRANSLATION_VARIANT_CLASSES:
+ flush_verse()
+ html = decode_html_content(elem)
+ if html:
+ result.append(f'
{html}
')
+ continue
+
+ # Centered note classes (p241)
+ if cls in CENTERED_NOTE_CLASSES:
+ flush_verse()
html = decode_html_content(elem)
if html:
result.append(f'
')
continue
- # SYNONYMS header (p165)
+ # SYNONYMS header (p165) — centered
if cls == SYNONYMS_HEADER_CLASS:
html = decode_html_content(elem)
if html:
- result.append(html)
+ result.append(f'
{html}
')
continue
- # TRANSLATION header (p55)
+ # TRANSLATION header (p55) — centered
if cls == TRANSLATION_HEADER_CLASS:
html = decode_html_content(elem)
if html:
- result.append(html)
+ result.append(f'
{html}
')
continue
# Mantra text (p19)
@@ -578,11 +651,11 @@ def extract_paragraphs(
result.append(html)
continue
- # Reference notes
+ # Reference notes (p187, p231, p233, p237, p32, p251) — centered
if cls in REFERENCE_CLASSES:
html = decode_html_content(elem)
if html:
- result.append(html)
+ result.append(f'
{html}
')
continue
# Bold text classes
@@ -592,11 +665,11 @@ def extract_paragraphs(
result.append(html)
continue
- # Synonym notes (p247)
+ # Synonym note (p247) — centered
if cls == SYNONYM_NOTES_CLASS:
html = decode_html_content(elem)
if html:
- result.append(html)
+ result.append(f'
{html}
')
continue
# Bold headings (p224, p223)
@@ -686,8 +759,8 @@ def extract_sva(soup: BeautifulSoup) -> dict:
Returns a dict with:
"intro_segments": list of intro segment dicts
- "chapters": list of chapter dicts
- "chapter_listing": list of {number, title} for intro.json
+ "sub_chapters": list of sub-chapter dicts (one per song)
+ "parent_chapters": list of parent chapter info for intro.json
"""
# Get all top-level
and
elements
all_elements = list(soup.find_all(["p", "table"]))
@@ -700,18 +773,22 @@ def extract_sva(soup: BeautifulSoup) -> dict:
# Step 2: Find all SVA bookmarks
bookmarks = find_sva_bookmarks(all_elements, sva_start, sva_end)
print(f" Found {len(bookmarks)} SVA bookmarks")
- for pos, text, cls in bookmarks[:20]:
+ for pos, text, cls in bookmarks[:30]:
print(f" Elem ~{pos} [{cls}]: {text[:80]}")
- if len(bookmarks) > 20:
- print(f" ... and {len(bookmarks) - 20} more")
+ if len(bookmarks) > 30:
+ print(f" ... and {len(bookmarks) - 30} more")
- # Step 3: Categorize bookmarks into sections
+ # Step 3: Categorize bookmarks
# p13 bookmarks define major sections:
# "SVA: Foreword" → foreword
# "SVA: Introduction" → introduction
# "SVA 2: ..." → Part 2 start
# "SVA 3: ..." → Part 3 start
# "SVA 4: ..." → Part 4 start
+ # p49 bookmarks define songs within parts:
+ # "SVA 1: ..." → songs in Part 1
+ # "SVA 2: ..." → songs in Part 2
+ # etc.
foreword_pos = None
introduction_pos = None
@@ -719,18 +796,8 @@ def extract_sva(soup: BeautifulSoup) -> dict:
part3_pos = None
part4_pos = None
- # p49 bookmarks define songs within parts:
- # "SVA 1: ..." → songs in Part 1
- # "SVA 2: ..." → songs in Part 2
- # "SVA 3: ..." → songs in Part 3
- # "SVA 4: ..." → songs in Part 4
-
- # Also find the first SVA 1: bookmark (start of Part 1)
first_part1_pos = None
- # Find "SVA: A Glimpse..." bookmark in intro
- glimpse_pos = None
-
for pos, text, cls in bookmarks:
if cls == BOOKMARK_CLASS: # p13
if text.strip() == "SVA: Foreword":
@@ -747,8 +814,6 @@ def extract_sva(soup: BeautifulSoup) -> dict:
if cls == SONG_BOOKMARK_CLASS: # p49
if text.strip().startswith("SVA 1:") and first_part1_pos is None:
first_part1_pos = pos
- if "Glimpse" in text or "Life of" in text:
- glimpse_pos = pos
print(f"\n Section boundaries:")
print(f" Book start: elem {sva_start}")
@@ -821,63 +886,133 @@ def extract_sva(soup: BeautifulSoup) -> dict:
"paragraphs": intro_paras,
}
- # Step 5: Extract chapters (Parts 1–4)
- chapter_boundaries = []
-
- if first_part1_pos is not None:
- ch1_end = part2_pos if part2_pos else sva_end
- chapter_boundaries.append((1, "Prayers and Praṇāmas", first_part1_pos, ch1_end))
-
- if part2_pos is not None:
- ch2_end = part3_pos if part3_pos else sva_end
- chapter_boundaries.append(
- (2, "Songs of Śrīla Bhaktivinoda Ṭhākura", part2_pos, ch2_end)
- )
-
- if part3_pos is not None:
- ch3_end = part4_pos if part4_pos else sva_end
- chapter_boundaries.append(
- (3, "Songs of Śrīla Narottama dāsa Ṭhākura", part3_pos, ch3_end)
- )
-
- if part4_pos is not None:
- chapter_boundaries.append((4, "Songs by other Ācāryas", part4_pos, sva_end))
-
- chapters = []
- for ch_num, ch_title, ch_start, ch_end in chapter_boundaries:
- # Start extraction from the element after the p13 bookmark (for parts 2-4)
- # or from the p49 bookmark itself for part 1
- content_start = ch_start
- if ch_num >= 2:
- # For parts 2-4, skip the p13 bookmark and the p31 heading that follows
- content_start = ch_start + 1
-
- paragraphs = extract_paragraphs(all_elements, content_start, ch_end)
-
- ch = {
- "book": "sva",
- "chapter": ch_num,
- "title": ch_title,
- "type": "chapter",
- "paragraphs": paragraphs,
- }
- chapters.append(ch)
-
- # Step 6: Build results
intro_segments = [front_pages]
if foreword:
intro_segments.append(foreword)
if introduction:
intro_segments.append(introduction)
- chapter_listing = [
- {"number": ch["chapter"], "title": ch["title"]} for ch in chapters
- ]
+ # Step 5: Collect all p49 song bookmarks grouped by part number
+ song_bookmarks_by_part = defaultdict(list)
+
+ for pos, text, cls in bookmarks:
+ if cls != SONG_BOOKMARK_CLASS:
+ continue
+ # Parse: "SVA 1: Śrī Guru Praṇāma" → part_num=1, title="Śrī Guru Praṇāma"
+ m = re.match(r"SVA\s+(\d+):\s+(.+)", text.strip())
+ if m:
+ part_num = int(m.group(1))
+ song_title = m.group(2).strip()
+ song_bookmarks_by_part[part_num].append(
+ {
+ "pos": pos,
+ "title": song_title,
+ "raw_text": text.strip(),
+ }
+ )
+
+ # Step 6: Define part boundaries
+ part_boundaries = []
+
+ if first_part1_pos is not None:
+ ch1_end = part2_pos if part2_pos else sva_end
+ part_boundaries.append((1, first_part1_pos, ch1_end))
+
+ if part2_pos is not None:
+ ch2_end = part3_pos if part3_pos else sva_end
+ part_boundaries.append((2, part2_pos, ch2_end))
+
+ if part3_pos is not None:
+ ch3_end = part4_pos if part4_pos else sva_end
+ part_boundaries.append((3, part3_pos, ch3_end))
+
+ if part4_pos is not None:
+ part_boundaries.append((4, part4_pos, sva_end))
+
+ # Step 7: Extract sub-chapters (songs) for each part
+ sub_chapters = []
+ parent_chapters = []
+
+ for part_num, part_start, part_end in part_boundaries:
+ parent_title = CLEAN_PARENT_TITLES.get(part_num, f"Part {part_num}")
+
+ songs = song_bookmarks_by_part.get(part_num, [])
+ # Sort by position
+ songs.sort(key=lambda s: s["pos"])
+
+ sub_entries = []
+ song_counter = 0
+
+ for si, song in enumerate(songs):
+ song_title = decode_scagoudy(song["title"])
+
+ # Detect "Purport to..." bookmarks — merge into preceding song
+ is_purport = song_title.lower().startswith("purport to")
+ if (
+ is_purport
+ and sub_chapters
+ and sub_chapters[-1]["parentChapter"] == part_num
+ ):
+ # Merge this purport's content into the last song
+ content_start = song["pos"] + 1
+ if si + 1 < len(songs):
+ content_end = songs[si + 1]["pos"]
+ else:
+ content_end = part_end
+
+ purport_paragraphs = extract_paragraphs(
+ all_elements, content_start, content_end
+ )
+ sub_chapters[-1]["paragraphs"].extend(purport_paragraphs)
+ continue
+
+ song_counter += 1
+ slug = f"{part_num}-{song_counter}"
+
+ # Content range: from this bookmark to next song bookmark (or part end)
+ # We start AFTER the bookmark element itself
+ content_start = song["pos"] + 1
+ if si + 1 < len(songs):
+ content_end = songs[si + 1]["pos"]
+ else:
+ content_end = part_end
+
+ paragraphs = extract_paragraphs(all_elements, content_start, content_end)
+
+ sub_chapters.append(
+ {
+ "book": "sva",
+ "slug": slug,
+ "parentChapter": part_num,
+ "subChapter": song_counter,
+ "title": song_title,
+ "parentTitle": parent_title,
+ "type": "sub-chapter",
+ "paragraphs": paragraphs,
+ }
+ )
+ sub_entries.append({"slug": slug, "title": song_title})
+
+ parent_chapters.append(
+ {
+ "number": part_num,
+ "title": parent_title,
+ "subChapters": sub_entries,
+ }
+ )
+
+ print(f"\n Part {part_num}: {parent_title} → {len(sub_entries)} songs")
+ for entry in sub_entries:
+ matching = [sc for sc in sub_chapters if sc["slug"] == entry["slug"]]
+ para_count = len(matching[0]["paragraphs"]) if matching else 0
+ print(
+ f" {entry['slug']:>5}: {para_count:>3} paragraphs — {entry['title']}"
+ )
return {
"intro_segments": intro_segments,
- "chapters": chapters,
- "chapter_listing": chapter_listing,
+ "sub_chapters": sub_chapters,
+ "parent_chapters": parent_chapters,
}
@@ -910,8 +1045,8 @@ def main():
result = extract_sva(soup)
intro_segments = result["intro_segments"]
- chapters = result["chapters"]
- chapter_listing = result["chapter_listing"]
+ sub_chapters = result["sub_chapters"]
+ parent_chapters = result["parent_chapters"]
# --- Write intro.json ---
intro_meta = {
@@ -921,7 +1056,7 @@ def main():
"year": 1972,
"description": "Bengali devotional songs by the great Vaiṣṇava teachers, with English translations and commentary.",
"totalChapters": 4,
- "chapters": chapter_listing,
+ "chapters": parent_chapters,
}
intro_path = os.path.join(output_base, "intro.json")
write_json(intro_meta, intro_path)
@@ -933,75 +1068,82 @@ def main():
filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
write_json(seg, filepath)
print(
- f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}"
+ f" {seg['order']:>2}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}"
)
- # --- Write chapters ---
- print(f"\n--- Chapters ---")
- for ch in chapters:
- ch_num_str = str(ch["chapter"]).zfill(2)
- filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
- write_json(ch, filepath)
+ # --- Write sub-chapter files (songs) ---
+ print(f"\n--- Songs (Sub-Chapters) ---")
+ for sc in sub_chapters:
+ slug = sc["slug"]
+ filepath = os.path.join(output_base, "chapters", f"{slug}.json")
+ write_json(sc, filepath)
print(
- f" Part {ch['chapter']}: {len(ch['paragraphs']):>3} paragraphs — "
- f"{ch['title']}"
+ f" {slug:>5}: {len(sc['paragraphs']):>3} paragraphs — "
+ f"[Part {sc['parentChapter']}] {sc['title']}"
)
# --- Summary ---
total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments)
- total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
+ total_chapter_paras = sum(len(sc["paragraphs"]) for sc in sub_chapters)
total_paras = total_intro_paras + total_chapter_paras
print(f"\n{'=' * 60}")
print(f" EXTRACTION SUMMARY")
print(f"{'=' * 60}")
print(f" Intro segments: {len(intro_segments)}")
- for seg in intro_segments:
- print(
- f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs"
- )
- print(f" Chapters (Parts): {len(chapters)}")
- for ch in chapters:
- print(
- f" Part {ch['chapter']}: {len(ch['paragraphs'])} paragraphs — {ch['title']}"
- )
+ print(f" Parent chapters: {len(parent_chapters)}")
+ print(f" Songs (sub-ch): {len(sub_chapters)}")
print(f" Total paragraphs: {total_paras}")
print(f" Intro: {total_intro_paras}")
- print(f" Chapters: {total_chapter_paras}")
- if chapters:
- print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
+ print(f" Songs: {total_chapter_paras}")
+ if sub_chapters:
+ print(f" Avg paras/song: {total_chapter_paras / len(sub_chapters):.1f}")
print(f" Output directory: {output_base}/")
+ # --- Per-parent breakdown ---
+ print(f"\n--- Part Breakdown ---")
+ for pc in parent_chapters:
+ part_num = pc["number"]
+ sub_count = len(pc["subChapters"])
+ part_paras = sum(
+ len(sc["paragraphs"])
+ for sc in sub_chapters
+ if sc["parentChapter"] == part_num
+ )
+ print(
+ f" Part {part_num}: {sub_count:>2} songs, {part_paras:>4} paragraphs — {pc['title']}"
+ )
+
# --- Samples ---
- print(f"\n--- Sample: Part 1 (Prayers and Praṇāmas) ---")
- if chapters:
- ch1 = chapters[0]
- for i, p in enumerate(ch1["paragraphs"][:5]):
- preview = re.sub(r"<[^>]+>", "", p)[:120]
+ print(f"\n--- Sample: Front Pages ---")
+ if intro_segments:
+ front = intro_segments[0]
+ for i, p in enumerate(front["paragraphs"][:5]):
+ preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
- if len(chapters) >= 2:
- print(f"\n--- Sample: Part 2 (Songs of Bhaktivinoda Ṭhākura) ---")
- ch2 = chapters[1]
- for i, p in enumerate(ch2["paragraphs"][:5]):
- preview = re.sub(r"<[^>]+>", "", p)[:120]
+ if sub_chapters:
+ first_sc = sub_chapters[0]
+ print(f"\n--- Sample: {first_sc['slug']} ({first_sc['title']}) ---")
+ for i, p in enumerate(first_sc["paragraphs"][:5]):
+ preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
- if len(chapters) >= 4:
- print(f"\n--- Sample: Part 4 (Songs by other Ācāryas) ---")
- ch4 = chapters[3]
- for i, p in enumerate(ch4["paragraphs"][:5]):
- preview = re.sub(r"<[^>]+>", "", p)[:120]
+ if len(sub_chapters) > 1:
+ last_sc = sub_chapters[-1]
+ print(f"\n--- Sample: {last_sc['slug']} ({last_sc['title']}) ---")
+ for i, p in enumerate(last_sc["paragraphs"][:5]):
+ preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
# --- Quality Checks ---
print(f"\n--- Quality Checks ---")
# Check all 4 parts present
- expected_chapters = set(range(1, 5))
- actual_chapters = set(ch["chapter"] for ch in chapters)
- missing = expected_chapters - actual_chapters
- extra = actual_chapters - expected_chapters
+ expected_parents = set(range(1, 5))
+ actual_parents = {pc["number"] for pc in parent_chapters}
+ missing = expected_parents - actual_parents
+ extra = actual_parents - expected_parents
if missing:
print(f" ⚠ Missing parts: {sorted(missing)}")
if extra:
@@ -1009,7 +1151,7 @@ def main():
if not missing and not extra:
print(f" ✅ All 4 parts present")
- # Check intro segments
+ # Check all 3 intro segments present
expected_intros = {"front-pages", "foreword", "introduction"}
actual_intros = {s["slug"] for s in intro_segments}
missing_intros = expected_intros - actual_intros
@@ -1018,17 +1160,24 @@ def main():
else:
print(f" ✅ All 3 intro segments present")
- # Check empty chapters
- empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
- if empty_chapters:
- print(f" ⚠ Empty parts (0 paragraphs): {empty_chapters}")
+ # Check empty sub-chapters
+ empty_scs = [sc["slug"] for sc in sub_chapters if len(sc["paragraphs"]) == 0]
+ if empty_scs:
+ print(f" ⚠ Empty songs (0 paragraphs): {empty_scs}")
else:
- print(f" ✅ No empty parts")
+ print(f" ✅ No empty songs")
+
+ # Check empty intro segments
+ empty_intros = [s["slug"] for s in intro_segments if len(s["paragraphs"]) == 0]
+ if empty_intros:
+ print(f" ⚠ Empty intro segments: {empty_intros}")
+ else:
+ print(f" ✅ No empty intro segments")
# Check ScaGoudy decoding
encoded_chars = set("äåöëòçüéàùè")
suspect_count = 0
- all_sections = chapters + intro_segments
+ all_sections = sub_chapters + intro_segments
for section in all_sections:
for p in section["paragraphs"]:
plain = re.sub(r"<[^>]+>", "", p)
@@ -1045,16 +1194,16 @@ def main():
# Check verse blocks
verse_block_count = 0
- for ch in chapters:
- for p in ch["paragraphs"]:
+ for sc in sub_chapters:
+ for p in sc["paragraphs"]:
if '
' in p:
verse_block_count += 1
- print(f" ℹ {verse_block_count} verse blocks in chapters")
+ print(f" ℹ {verse_block_count} verse blocks in songs")
# Check table-derived content
table_verse_count = 0
- for ch in chapters:
- for p in ch["paragraphs"]:
+ for sc in sub_chapters:
+ for p in sc["paragraphs"]:
if '
' in p and " " in p:
table_verse_count += 1
print(f" ℹ {table_verse_count} multi-line verse blocks (likely from tables)")
@@ -1077,11 +1226,11 @@ def main():
# Sample verse blocks
print(f"\n--- Sample Verse Blocks ---")
shown = 0
- for ch in chapters:
- for p in ch["paragraphs"]:
+ for sc in sub_chapters:
+ for p in sc["paragraphs"]:
if '
' in p:
preview = re.sub(r"<[^>]+>", "", p)[:120]
- print(f" Part {ch['chapter']}: {preview}...")
+ print(f" {sc['slug']} ({sc['title']}): {preview}...")
shown += 1
if shown >= 5:
break
@@ -1089,24 +1238,92 @@ def main():
break
# Song title check
- print(f"\n--- Song Titles in Chapters ---")
- song_count = 0
- sample_songs = []
- for ch in chapters:
- ch_songs = 0
- for p in ch["paragraphs"]:
- # Bold italic song titles
- if "" in p and "" in p:
- plain = re.sub(r"<[^>]+>", "", p).strip()
- if len(plain) < 100:
- ch_songs += 1
- if len(sample_songs) < 8:
- sample_songs.append(f" Part {ch['chapter']}: {plain}")
- song_count += ch_songs
- print(f" Part {ch['chapter']}: ~{ch_songs} song titles")
- print(f" Total song titles: ~{song_count}")
- for s in sample_songs:
- print(s)
+ print(f"\n--- Song Titles ---")
+ for pc in parent_chapters:
+ print(f" Part {pc['number']}: {pc['title']}")
+ for sub in pc["subChapters"]:
+ print(f" {sub['slug']:>5}: {sub['title']}")
+
+ # Verify slug numbering continuity within each parent
+ print(f"\n--- Slug Numbering Check ---")
+ for pc in parent_chapters:
+ part_num = pc["number"]
+ slugs = [s["slug"] for s in pc["subChapters"]]
+ expected_slugs = [f"{part_num}-{i + 1}" for i in range(len(slugs))]
+ if slugs == expected_slugs:
+ print(
+ f" ✅ Part {part_num}: {len(slugs)} songs, slugs {slugs[0]}..{slugs[-1]}"
+ )
+ else:
+ print(
+ f" ⚠ Part {part_num}: slug mismatch! expected {expected_slugs}, got {slugs}"
+ )
+
+ # Sub-chapter size check
+ print(f"\n--- Song Size Check ---")
+ very_small = []
+ very_large = []
+ for sc in sub_chapters:
+ count = len(sc["paragraphs"])
+ if count < 1:
+ very_small.append((sc["slug"], count, sc["title"]))
+ elif count > 200:
+ very_large.append((sc["slug"], count, sc["title"]))
+ if very_small:
+ print(f" ⚠ Very small songs (0 paragraphs):")
+ for slug, count, title in very_small:
+ print(f" {slug}: {count} paragraphs — {title}")
+ else:
+ print(f" ✅ All songs have ≥ 1 paragraph")
+ if very_large:
+ print(f" ⚠ Very large songs (> 200 paragraphs):")
+ for slug, count, title in very_large:
+ print(f" {slug}: {count} paragraphs — {title}")
+ else:
+ print(f" ✅ All songs have ≤ 200 paragraphs")
+
+ # Sub-chapter title decoding check
+ print(f"\n--- Song Title Decoding Check ---")
+ bad_titles = 0
+ for sc in sub_chapters:
+ title = sc["title"]
+ if "å" in title or "ë" in title or "ñ" in title or "ä" in title:
+ print(f" ⚠ {sc['slug']} title has un-decoded ScaGoudy: {title}")
+ bad_titles += 1
+ if bad_titles == 0:
+ print(f" ✅ All {len(sub_chapters)} song titles properly decoded")
+
+ # Check for residual span tags
+ span_count = 0
+ for sc in sub_chapters:
+ for p in sc["paragraphs"]:
+ if 'class="' in p and "]+>", "", p)
+ if is_dash_line(plain):
+ dash_count += 1
+ for seg in intro_segments:
+ for p in seg["paragraphs"]:
+ plain = re.sub(r"<[^>]+>", "", p)
+ if is_dash_line(plain):
+ dash_count += 1
+ if dash_count == 0:
+ print(f" ✅ No dashed-line paragraphs in output")
+ else:
+ print(f" ⚠ {dash_count} dashed-line paragraphs found in output")
print(f"\nDone! ✨")