fix: SVA chapters
This commit is contained in:
@@ -3,6 +3,7 @@ import { notFound } from "next/navigation";
|
|||||||
import {
|
import {
|
||||||
getSvaContent,
|
getSvaContent,
|
||||||
getSvaSidebarItems,
|
getSvaSidebarItems,
|
||||||
|
getNavigableItems,
|
||||||
type SvaChapter,
|
type SvaChapter,
|
||||||
type SvaSidebarItem,
|
type SvaSidebarItem,
|
||||||
} from "@/lib/sva";
|
} from "@/lib/sva";
|
||||||
@@ -28,18 +29,24 @@ export default async function SvaChapterPage({
|
|||||||
if (!content) notFound();
|
if (!content) notFound();
|
||||||
|
|
||||||
const sidebarItems = await getSvaSidebarItems();
|
const sidebarItems = await getSvaSidebarItems();
|
||||||
const currentIdx = sidebarItems.findIndex((item) => item.slug === slug);
|
|
||||||
|
|
||||||
const prev = currentIdx > 0 ? sidebarItems[currentIdx - 1] : null;
|
// Build a flat list of navigable items (skip "parent" type — they're labels, not pages)
|
||||||
|
const navigableItems = getNavigableItems(sidebarItems);
|
||||||
|
const currentIdx = navigableItems.findIndex((item) => item.slug === slug);
|
||||||
|
|
||||||
|
const prev = currentIdx > 0 ? navigableItems[currentIdx - 1] : null;
|
||||||
const next =
|
const next =
|
||||||
currentIdx < sidebarItems.length - 1 ? sidebarItems[currentIdx + 1] : null;
|
currentIdx < navigableItems.length - 1
|
||||||
|
? navigableItems[currentIdx + 1]
|
||||||
|
: null;
|
||||||
|
|
||||||
// Build the page title
|
// Determine if this is a sub-chapter (has parentChapter)
|
||||||
const isChapter =
|
const isSubChapter =
|
||||||
"chapter" in content &&
|
"parentChapter" in content &&
|
||||||
typeof (content as SvaChapter).chapter === "number" &&
|
typeof (content as SvaChapter).parentChapter === "number";
|
||||||
(content as SvaChapter).chapter !== null;
|
const parentChapterNum = isSubChapter
|
||||||
const chapterNum = isChapter ? (content as SvaChapter).chapter : null;
|
? (content as SvaChapter).parentChapter
|
||||||
|
: null;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="min-h-screen bg-background">
|
<div className="min-h-screen bg-background">
|
||||||
@@ -51,8 +58,8 @@ export default async function SvaChapterPage({
|
|||||||
← Songs of the Vaiṣṇava Ācāryas
|
← Songs of the Vaiṣṇava Ācāryas
|
||||||
</Link>
|
</Link>
|
||||||
<h1 className="mt-3 text-2xl font-bold text-foreground">
|
<h1 className="mt-3 text-2xl font-bold text-foreground">
|
||||||
{chapterNum != null && (
|
{parentChapterNum != null && (
|
||||||
<span className="text-primary">Part {chapterNum}: </span>
|
<span className="text-primary">Part {parentChapterNum}: </span>
|
||||||
)}
|
)}
|
||||||
{content.title}
|
{content.title}
|
||||||
</h1>
|
</h1>
|
||||||
@@ -111,33 +118,21 @@ function SvaSidebar({
|
|||||||
items: SvaSidebarItem[];
|
items: SvaSidebarItem[];
|
||||||
currentSlug: string;
|
currentSlug: string;
|
||||||
}) {
|
}) {
|
||||||
const frontMatterItems: SvaSidebarItem[] = [];
|
// Split items by section
|
||||||
const chapterItems: SvaSidebarItem[] = [];
|
const frontMatter = items.filter((i) => i.type === "intro-segment");
|
||||||
const backMatterItems: SvaSidebarItem[] = [];
|
const parents = items.filter((i) => i.type === "parent");
|
||||||
|
const subChapters = items.filter((i) => i.type === "sub-chapter");
|
||||||
let pastChapters = false;
|
|
||||||
|
|
||||||
for (const item of items) {
|
|
||||||
if (item.type === "chapter") {
|
|
||||||
pastChapters = true;
|
|
||||||
chapterItems.push(item);
|
|
||||||
} else if (!pastChapters) {
|
|
||||||
frontMatterItems.push(item);
|
|
||||||
} else {
|
|
||||||
backMatterItems.push(item);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<aside className="hidden lg:block w-56 shrink-0 sticky top-[89px] self-start max-h-[calc(100vh-89px)] overflow-y-auto py-8 pr-4">
|
<aside className="hidden lg:block w-56 shrink-0 sticky top-[89px] self-start max-h-[calc(100vh-89px)] overflow-y-auto py-8 pr-4">
|
||||||
{/* Front Matter */}
|
{/* Front Matter */}
|
||||||
{frontMatterItems.length > 0 && (
|
{frontMatter.length > 0 && (
|
||||||
<>
|
<>
|
||||||
<h2 className="text-xs font-bold uppercase tracking-wider text-muted-foreground mb-3 px-2">
|
<h2 className="text-xs font-bold uppercase tracking-wider text-muted-foreground mb-3 px-2">
|
||||||
Front Matter
|
Front Matter
|
||||||
</h2>
|
</h2>
|
||||||
<nav className="flex flex-col gap-0.5 mb-4">
|
<nav className="flex flex-col gap-0.5 mb-4">
|
||||||
{frontMatterItems.map((item) => (
|
{frontMatter.map((item) => (
|
||||||
<SidebarLink
|
<SidebarLink
|
||||||
key={item.slug}
|
key={item.slug}
|
||||||
item={item}
|
item={item}
|
||||||
@@ -148,36 +143,45 @@ function SvaSidebar({
|
|||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
{/* Parts */}
|
{/* Parts — grouped by parent */}
|
||||||
{chapterItems.length > 0 && (
|
{parents.length > 0 && (
|
||||||
<>
|
<>
|
||||||
<h2 className="text-xs font-bold uppercase tracking-wider text-muted-foreground mb-3 px-2">
|
<h2 className="text-xs font-bold uppercase tracking-wider text-muted-foreground mb-3 px-2">
|
||||||
Parts
|
Songs
|
||||||
</h2>
|
</h2>
|
||||||
<nav className="flex flex-col gap-0.5 mb-4">
|
<nav className="flex flex-col gap-0.5 mb-4">
|
||||||
{chapterItems.map((item) => (
|
{parents.map((parent) => (
|
||||||
<SidebarLink
|
<div key={parent.slug}>
|
||||||
key={item.slug}
|
{/* Parent heading — not a link, just a bold label */}
|
||||||
item={item}
|
<div className="px-2 py-1.5 text-xs font-bold text-muted-foreground mt-3 first:mt-0">
|
||||||
isActive={item.slug === currentSlug}
|
{parent.label}
|
||||||
/>
|
</div>
|
||||||
|
{/* Sub-chapters (songs) indented */}
|
||||||
|
{subChapters
|
||||||
|
.filter((sub) => sub.parentChapter === parent.parentChapter)
|
||||||
|
.map((sub) => {
|
||||||
|
const isActive = sub.slug === currentSlug;
|
||||||
|
return (
|
||||||
|
<Link
|
||||||
|
key={sub.slug}
|
||||||
|
href={`/books/sva/${sub.slug}`}
|
||||||
|
className={cn(
|
||||||
|
"pl-4 px-2 py-1.5 rounded-md text-sm transition-colors truncate block",
|
||||||
|
isActive
|
||||||
|
? "bg-primary/10 text-primary font-medium"
|
||||||
|
: "text-muted-foreground hover:bg-muted hover:text-foreground",
|
||||||
|
)}
|
||||||
|
title={sub.label}
|
||||||
|
>
|
||||||
|
{sub.label}
|
||||||
|
</Link>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
))}
|
))}
|
||||||
</nav>
|
</nav>
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
{/* Back Matter */}
|
|
||||||
{backMatterItems.length > 0 && (
|
|
||||||
<nav className="flex flex-col gap-0.5">
|
|
||||||
{backMatterItems.map((item) => (
|
|
||||||
<SidebarLink
|
|
||||||
key={item.slug}
|
|
||||||
item={item}
|
|
||||||
isActive={item.slug === currentSlug}
|
|
||||||
/>
|
|
||||||
))}
|
|
||||||
</nav>
|
|
||||||
)}
|
|
||||||
</aside>
|
</aside>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -189,15 +193,6 @@ function SidebarLink({
|
|||||||
item: SvaSidebarItem;
|
item: SvaSidebarItem;
|
||||||
isActive: boolean;
|
isActive: boolean;
|
||||||
}) {
|
}) {
|
||||||
let label: string;
|
|
||||||
if (item.type === "chapter" && item.chapter != null) {
|
|
||||||
// Extract just the title portion after "Part N: "
|
|
||||||
const titlePart = item.label.replace(/^Part \d+:\s*/, "");
|
|
||||||
label = `${item.chapter}. ${titlePart}`;
|
|
||||||
} else {
|
|
||||||
label = item.label;
|
|
||||||
}
|
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<Link
|
<Link
|
||||||
href={`/books/sva/${item.slug}`}
|
href={`/books/sva/${item.slug}`}
|
||||||
@@ -207,9 +202,9 @@ function SidebarLink({
|
|||||||
? "bg-primary/10 text-primary font-medium"
|
? "bg-primary/10 text-primary font-medium"
|
||||||
: "text-muted-foreground hover:bg-muted hover:text-foreground",
|
: "text-muted-foreground hover:bg-muted hover:text-foreground",
|
||||||
)}
|
)}
|
||||||
title={label}
|
title={item.label}
|
||||||
>
|
>
|
||||||
{label}
|
{item.label}
|
||||||
</Link>
|
</Link>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -231,26 +226,11 @@ function SvaNav({
|
|||||||
}) {
|
}) {
|
||||||
const mobileNavGroups: MobileNavGroup[] = [];
|
const mobileNavGroups: MobileNavGroup[] = [];
|
||||||
|
|
||||||
const frontMatterItems: SvaSidebarItem[] = [];
|
const frontMatter = sidebarItems.filter((i) => i.type === "intro-segment");
|
||||||
const chapterItems: SvaSidebarItem[] = [];
|
if (frontMatter.length > 0) {
|
||||||
const backMatterItems: SvaSidebarItem[] = [];
|
|
||||||
let pastChapters = false;
|
|
||||||
|
|
||||||
for (const item of sidebarItems) {
|
|
||||||
if (item.type === "chapter") {
|
|
||||||
pastChapters = true;
|
|
||||||
chapterItems.push(item);
|
|
||||||
} else if (!pastChapters) {
|
|
||||||
frontMatterItems.push(item);
|
|
||||||
} else {
|
|
||||||
backMatterItems.push(item);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (frontMatterItems.length > 0) {
|
|
||||||
mobileNavGroups.push({
|
mobileNavGroups.push({
|
||||||
title: "Front Matter",
|
title: "Front Matter",
|
||||||
items: frontMatterItems.map((i) => ({
|
items: frontMatter.map((i) => ({
|
||||||
slug: i.slug,
|
slug: i.slug,
|
||||||
label: i.label,
|
label: i.label,
|
||||||
href: `/books/sva/${i.slug}`,
|
href: `/books/sva/${i.slug}`,
|
||||||
@@ -259,26 +239,19 @@ function SvaNav({
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const chapterItems = sidebarItems.filter(
|
||||||
|
(i) => i.type === "parent" || i.type === "sub-chapter",
|
||||||
|
);
|
||||||
if (chapterItems.length > 0) {
|
if (chapterItems.length > 0) {
|
||||||
mobileNavGroups.push({
|
mobileNavGroups.push({
|
||||||
title: "Parts",
|
title: "Songs",
|
||||||
items: chapterItems.map((i) => ({
|
items: chapterItems.map((i) => ({
|
||||||
slug: i.slug,
|
slug: i.slug,
|
||||||
label: i.label,
|
label: i.label,
|
||||||
href: `/books/sva/${i.slug}`,
|
href: `/books/sva/${i.slug}`,
|
||||||
type: i.type,
|
type: i.type,
|
||||||
})),
|
parentChapter: i.parentChapter,
|
||||||
});
|
isParent: i.type === "parent",
|
||||||
}
|
|
||||||
|
|
||||||
if (backMatterItems.length > 0) {
|
|
||||||
mobileNavGroups.push({
|
|
||||||
title: "Back Matter",
|
|
||||||
items: backMatterItems.map((i) => ({
|
|
||||||
slug: i.slug,
|
|
||||||
label: i.label,
|
|
||||||
href: `/books/sva/${i.slug}`,
|
|
||||||
type: i.type,
|
|
||||||
})),
|
})),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -301,9 +274,7 @@ function SvaNav({
|
|||||||
>
|
>
|
||||||
<ChevronLeft className="size-4" />
|
<ChevronLeft className="size-4" />
|
||||||
<span className="hidden sm:inline truncate max-w-[180px]">
|
<span className="hidden sm:inline truncate max-w-[180px]">
|
||||||
{prev.type === "chapter" && prev.chapter != null
|
{prev.label}
|
||||||
? `Part ${prev.chapter}`
|
|
||||||
: prev.label}
|
|
||||||
</span>
|
</span>
|
||||||
<span className="sm:hidden">Previous</span>
|
<span className="sm:hidden">Previous</span>
|
||||||
</Link>
|
</Link>
|
||||||
@@ -321,9 +292,7 @@ function SvaNav({
|
|||||||
)}
|
)}
|
||||||
>
|
>
|
||||||
<span className="hidden sm:inline truncate max-w-[180px]">
|
<span className="hidden sm:inline truncate max-w-[180px]">
|
||||||
{next.type === "chapter" && next.chapter != null
|
{next.label}
|
||||||
? `Part ${next.chapter}`
|
|
||||||
: next.label}
|
|
||||||
</span>
|
</span>
|
||||||
<span className="sm:hidden">Next</span>
|
<span className="sm:hidden">Next</span>
|
||||||
<ChevronRight className="size-4" />
|
<ChevronRight className="size-4" />
|
||||||
|
|||||||
+25
-28
@@ -12,7 +12,7 @@ export default async function SvaPage() {
|
|||||||
|
|
||||||
const introSegments = await getSvaIntroSegments();
|
const introSegments = await getSvaIntroSegments();
|
||||||
|
|
||||||
// Split intro segments into front matter (order < 99) and back matter (order >= 99)
|
// Split intro segments into front matter (before chapters) and back matter (after)
|
||||||
const frontMatter = introSegments.filter((seg) => seg.order < 99);
|
const frontMatter = introSegments.filter((seg) => seg.order < 99);
|
||||||
const backMatter = introSegments.filter((seg) => seg.order >= 99);
|
const backMatter = introSegments.filter((seg) => seg.order >= 99);
|
||||||
|
|
||||||
@@ -69,35 +69,32 @@ export default async function SvaPage() {
|
|||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
{/* Chapters (Parts) */}
|
{/* Parts — hierarchical parent → song (sub-chapter) structure */}
|
||||||
<div>
|
{meta.chapters.map((ch) => (
|
||||||
<h2 className="text-sm font-bold uppercase tracking-wider text-muted-foreground mb-3">
|
<div key={ch.number}>
|
||||||
Parts
|
<h2 className="text-sm font-bold uppercase tracking-wider text-muted-foreground mb-3">
|
||||||
</h2>
|
Part {ch.number}: {ch.title}
|
||||||
<Card className="divide-y divide-border">
|
</h2>
|
||||||
<CardContent className="p-0">
|
<Card className="divide-y divide-border">
|
||||||
<ul className="divide-y divide-border">
|
<CardContent className="p-0">
|
||||||
{meta.chapters.map((ch) => (
|
<ul className="divide-y divide-border">
|
||||||
<li key={ch.number}>
|
{ch.subChapters.map((sub) => (
|
||||||
<Link
|
<li key={sub.slug}>
|
||||||
href={`/books/sva/${ch.number}`}
|
<Link
|
||||||
className="flex flex-col sm:flex-row sm:items-center sm:justify-between px-5 py-4 gap-1 sm:gap-0 hover:bg-muted transition-colors"
|
href={`/books/sva/${sub.slug}`}
|
||||||
>
|
className="flex flex-col sm:flex-row sm:items-center sm:justify-between px-5 py-4 gap-1 sm:gap-0 hover:bg-muted transition-colors"
|
||||||
<div>
|
>
|
||||||
<span className="text-sm text-primary font-medium">
|
|
||||||
Part {ch.number}
|
|
||||||
</span>
|
|
||||||
<p className="font-medium text-foreground">
|
<p className="font-medium text-foreground">
|
||||||
{ch.title}
|
{sub.title}
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</Link>
|
||||||
</Link>
|
</li>
|
||||||
</li>
|
))}
|
||||||
))}
|
</ul>
|
||||||
</ul>
|
</CardContent>
|
||||||
</CardContent>
|
</Card>
|
||||||
</Card>
|
</div>
|
||||||
</div>
|
))}
|
||||||
|
|
||||||
{/* Back Matter */}
|
{/* Back Matter */}
|
||||||
{backMatter.length > 0 && (
|
{backMatter.length > 0 && (
|
||||||
|
|||||||
+251
-167
@@ -1,9 +1,10 @@
|
|||||||
/**
|
/**
|
||||||
* Data access layer for Songs of the Vaiṣṇava Ācāryas (SVA).
|
* Data access layer for Songs of the Vaiṣṇava Ācāryas (SVA).
|
||||||
*
|
*
|
||||||
* Reads from MongoDB via Prisma, same as the rest of the app.
|
* Reads from MongoDB via Prisma.
|
||||||
* SVA is a prose/song book with 4 parts (chapters) plus intro segments.
|
* SVA is a song book with 4 parent parts, each containing multiple songs
|
||||||
* We use volume=0 as the sentinel for "no volumes".
|
* (sub-chapters). Each song is stored as its own prose_chapters document
|
||||||
|
* with slugs like "1-1", "1-2", "2-1", etc.
|
||||||
*
|
*
|
||||||
* When the DB isn't seeded yet, falls back to reading local JSON files
|
* When the DB isn't seeded yet, falls back to reading local JSON files
|
||||||
* from lib/jsons/books/sva/.
|
* from lib/jsons/books/sva/.
|
||||||
@@ -22,11 +23,11 @@ import path from "path";
|
|||||||
// Constants
|
// Constants
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
const BOOK = "sva";
|
||||||
|
|
||||||
/** Volume sentinel for books without volumes. */
|
/** Volume sentinel for books without volumes. */
|
||||||
const NO_VOLUME = 0;
|
const NO_VOLUME = 0;
|
||||||
|
|
||||||
const BOOK = "sva";
|
|
||||||
|
|
||||||
const SVA_JSON_DIR = path.join(process.cwd(), "lib", "jsons", "books", "sva");
|
const SVA_JSON_DIR = path.join(process.cwd(), "lib", "jsons", "books", "sva");
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -37,15 +38,21 @@ export type SvaMeta = {
|
|||||||
book: string;
|
book: string;
|
||||||
title: string;
|
title: string;
|
||||||
author: string;
|
author: string;
|
||||||
year?: number;
|
year: number;
|
||||||
description: string;
|
description: string;
|
||||||
totalChapters: number;
|
totalChapters: number;
|
||||||
chapters: SvaChapterEntry[];
|
chapters: SvaParentChapter[];
|
||||||
};
|
};
|
||||||
|
|
||||||
export type SvaChapterEntry = {
|
export type SvaParentChapter = {
|
||||||
number: number;
|
number: number;
|
||||||
title: string;
|
title: string;
|
||||||
|
subChapters: SvaSubChapterEntry[];
|
||||||
|
};
|
||||||
|
|
||||||
|
export type SvaSubChapterEntry = {
|
||||||
|
slug: string; // "1-1", "1-2", etc.
|
||||||
|
title: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
export type SvaIntroSegment = {
|
export type SvaIntroSegment = {
|
||||||
@@ -59,27 +66,29 @@ export type SvaIntroSegment = {
|
|||||||
|
|
||||||
export type SvaChapter = {
|
export type SvaChapter = {
|
||||||
book: string;
|
book: string;
|
||||||
chapter: number | null;
|
chapter: string; // "1-1", "2-1", etc.
|
||||||
|
parentChapter: number;
|
||||||
|
subChapter: number;
|
||||||
title: string;
|
title: string;
|
||||||
|
parentTitle: string;
|
||||||
type: string;
|
type: string;
|
||||||
slug?: string;
|
|
||||||
paragraphs: string[];
|
paragraphs: string[];
|
||||||
};
|
};
|
||||||
|
|
||||||
/** Union type used in sidebar / table-of-contents navigation. */
|
/** Union type used in sidebar / table-of-contents navigation. */
|
||||||
export type SvaSidebarItem = {
|
export type SvaSidebarItem = {
|
||||||
/** URL-safe slug: "intro-front-pages", "intro-foreword", "1", "4" */
|
/** URL-safe slug: "intro-front-pages", "1-1", "parent-1", etc. */
|
||||||
slug: string;
|
slug: string;
|
||||||
/** Human-readable label for the sidebar */
|
/** Human-readable label for the sidebar */
|
||||||
label: string;
|
label: string;
|
||||||
/** "intro-segment" | "chapter" */
|
/** "intro-segment" | "parent" | "sub-chapter" */
|
||||||
type: string;
|
type: string;
|
||||||
/** Chapter number, or null for non-chapter items */
|
/** Parent chapter number, or null for intro segments */
|
||||||
chapter: number | null;
|
parentChapter: number | null;
|
||||||
};
|
};
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// HTML sanitisation (mirrors lib/api.ts)
|
// HTML sanitisation
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
function sanitizeHtml(html: string): string {
|
function sanitizeHtml(html: string): string {
|
||||||
@@ -148,13 +157,29 @@ function getLocalIntroSegment(rawSlug: string): SvaIntroSegment | null {
|
|||||||
return readLocalJson<SvaIntroSegment>(filePath);
|
return readLocalJson<SvaIntroSegment>(filePath);
|
||||||
}
|
}
|
||||||
|
|
||||||
function getLocalChapter(chapterNum: number): SvaChapter | null {
|
function getLocalSubChapter(slug: string): SvaChapter | null {
|
||||||
const filePath = path.join(
|
const filePath = path.join(SVA_JSON_DIR, "chapters", `${slug}.json`);
|
||||||
SVA_JSON_DIR,
|
const raw = readLocalJson<{
|
||||||
"chapters",
|
book: string;
|
||||||
String(chapterNum).padStart(2, "0") + ".json",
|
slug: string;
|
||||||
);
|
parentChapter: number;
|
||||||
return readLocalJson<SvaChapter>(filePath);
|
subChapter: number;
|
||||||
|
title: string;
|
||||||
|
parentTitle: string;
|
||||||
|
type: string;
|
||||||
|
paragraphs: string[];
|
||||||
|
}>(filePath);
|
||||||
|
if (!raw) return null;
|
||||||
|
return {
|
||||||
|
book: raw.book,
|
||||||
|
chapter: raw.slug,
|
||||||
|
parentChapter: raw.parentChapter,
|
||||||
|
subChapter: raw.subChapter,
|
||||||
|
title: raw.title,
|
||||||
|
parentTitle: raw.parentTitle,
|
||||||
|
type: raw.type,
|
||||||
|
paragraphs: raw.paragraphs,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -162,11 +187,15 @@ function getLocalChapter(chapterNum: number): SvaChapter | null {
|
|||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Book-level metadata (title, author, chapter list, etc.)
|
* Book-level metadata (title, author, chapter list with sub-chapters, etc.)
|
||||||
*/
|
*/
|
||||||
export async function getSvaMeta(): Promise<SvaMeta | null> {
|
export async function getSvaMeta(): Promise<SvaMeta | null> {
|
||||||
const meta = await prisma.bookMeta.findUnique({ where: { book: BOOK } });
|
try {
|
||||||
if (meta) return meta.data as unknown as SvaMeta;
|
const meta = await prisma.bookMeta.findUnique({ where: { book: BOOK } });
|
||||||
|
if (meta) return meta.data as unknown as SvaMeta;
|
||||||
|
} catch {
|
||||||
|
// Prisma BSON decode error — fall through to local JSON
|
||||||
|
}
|
||||||
|
|
||||||
// Fallback to local JSON
|
// Fallback to local JSON
|
||||||
const introPath = path.join(SVA_JSON_DIR, "intro.json");
|
const introPath = path.join(SVA_JSON_DIR, "intro.json");
|
||||||
@@ -181,20 +210,24 @@ export async function getSvaMeta(): Promise<SvaMeta | null> {
|
|||||||
* All intro segments, ordered by their `order` field.
|
* All intro segments, ordered by their `order` field.
|
||||||
*/
|
*/
|
||||||
export async function getSvaIntroSegments(): Promise<SvaIntroSegment[]> {
|
export async function getSvaIntroSegments(): Promise<SvaIntroSegment[]> {
|
||||||
const results = await prisma.proseChapter.findMany({
|
try {
|
||||||
where: { book: BOOK, volume: NO_VOLUME, type: "intro-segment" },
|
const results = await prisma.proseChapter.findMany({
|
||||||
orderBy: { order: "asc" },
|
where: { book: BOOK, volume: NO_VOLUME, type: "intro-segment" },
|
||||||
});
|
orderBy: { order: "asc" },
|
||||||
|
});
|
||||||
|
|
||||||
if (results.length > 0) {
|
if (results.length > 0) {
|
||||||
return results.map((doc) => ({
|
return results.map((doc) => ({
|
||||||
slug: doc.slug,
|
slug: doc.slug,
|
||||||
title: doc.title,
|
title: doc.title,
|
||||||
type: doc.type,
|
type: doc.type,
|
||||||
book: doc.book,
|
book: doc.book,
|
||||||
order: doc.order,
|
order: doc.order,
|
||||||
paragraphs: normalizeParagraphs(doc.paragraphs),
|
paragraphs: normalizeParagraphs(doc.paragraphs),
|
||||||
}));
|
}));
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Prisma BSON decode error — fall through to local JSON
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fallback to local JSON
|
// Fallback to local JSON
|
||||||
@@ -203,14 +236,16 @@ export async function getSvaIntroSegments(): Promise<SvaIntroSegment[]> {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* A single intro segment by slug.
|
* A single intro segment by slug.
|
||||||
* Accepts both "intro-foreword" (URL slug) and "foreword" (raw slug).
|
* Accepts both "intro-front-pages" (URL slug) and "front-pages" (raw slug).
|
||||||
*/
|
*/
|
||||||
export async function getSvaIntroSegment(
|
export async function getSvaIntroSegment(
|
||||||
slug: string,
|
slug: string,
|
||||||
): Promise<SvaIntroSegment | null> {
|
): Promise<SvaIntroSegment | null> {
|
||||||
// DB stores with "intro-" prefix; accept either form
|
// DB stores with "intro-" prefix; accept either form
|
||||||
const dbSlug = slug.startsWith("intro-") ? slug : `intro-${slug}`;
|
const dbSlug = slug.startsWith("intro-") ? slug : `intro-${slug}`;
|
||||||
const rawSlug = slug.startsWith("intro-") ? slug.replace(/^intro-/, "") : slug;
|
const rawSlug = slug.startsWith("intro-")
|
||||||
|
? slug.replace(/^intro-/, "")
|
||||||
|
: slug;
|
||||||
|
|
||||||
const doc = await prisma.proseChapter.findUnique({
|
const doc = await prisma.proseChapter.findUnique({
|
||||||
where: {
|
where: {
|
||||||
@@ -240,42 +275,54 @@ export async function getSvaIntroSegment(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Chapters
|
// Chapters (sub-chapters like "1-1", "2-3", etc.)
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A numbered chapter (1–4).
|
* A sub-chapter (song) by its slug (e.g. "1-1", "2-3").
|
||||||
*/
|
*/
|
||||||
export async function getSvaChapter(
|
export async function getSvaChapter(slug: string): Promise<SvaChapter | null> {
|
||||||
chapterNum: number,
|
try {
|
||||||
): Promise<SvaChapter | null> {
|
const doc = await prisma.proseChapter.findUnique({
|
||||||
if (chapterNum < 1 || chapterNum > 4 || !Number.isInteger(chapterNum)) {
|
where: {
|
||||||
return null;
|
book_volume_slug: {
|
||||||
}
|
book: BOOK,
|
||||||
|
volume: NO_VOLUME,
|
||||||
const doc = await prisma.proseChapter.findUnique({
|
slug,
|
||||||
where: {
|
},
|
||||||
book_volume_slug: {
|
|
||||||
book: BOOK,
|
|
||||||
volume: NO_VOLUME,
|
|
||||||
slug: String(chapterNum),
|
|
||||||
},
|
},
|
||||||
},
|
});
|
||||||
});
|
|
||||||
|
|
||||||
if (doc) {
|
if (doc) {
|
||||||
return {
|
// Parse parentChapter and subChapter from slug "X-Y"
|
||||||
book: doc.book,
|
const parts = slug.split("-");
|
||||||
chapter: doc.chapter,
|
const parentChapter = parseInt(parts[0], 10);
|
||||||
title: doc.title,
|
const subChapter = parseInt(parts[1], 10);
|
||||||
type: doc.type,
|
|
||||||
slug: doc.slug,
|
// Derive parentTitle from meta if available
|
||||||
paragraphs: normalizeParagraphs(doc.paragraphs),
|
const meta = await getSvaMeta();
|
||||||
};
|
const parentEntry = meta?.chapters.find(
|
||||||
|
(ch) => ch.number === parentChapter,
|
||||||
|
);
|
||||||
|
const parentTitle = parentEntry?.title ?? "";
|
||||||
|
|
||||||
|
return {
|
||||||
|
book: doc.book,
|
||||||
|
chapter: slug,
|
||||||
|
parentChapter,
|
||||||
|
subChapter,
|
||||||
|
title: doc.title,
|
||||||
|
parentTitle,
|
||||||
|
type: doc.type,
|
||||||
|
paragraphs: normalizeParagraphs(doc.paragraphs),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Prisma BSON decode error — fall through to local JSON
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fallback to local JSON
|
// Fallback to local JSON
|
||||||
const local = getLocalChapter(chapterNum);
|
const local = getLocalSubChapter(slug);
|
||||||
if (!local) return null;
|
if (!local) return null;
|
||||||
return {
|
return {
|
||||||
...local,
|
...local,
|
||||||
@@ -290,10 +337,10 @@ export async function getSvaChapter(
|
|||||||
/**
|
/**
|
||||||
* Get content by URL slug.
|
* Get content by URL slug.
|
||||||
*
|
*
|
||||||
* "intro-front-pages" → intro segment
|
* "intro-front-pages" → intro segment
|
||||||
* "intro-foreword" → intro segment
|
* "intro-foreword" → intro segment
|
||||||
* "intro-introduction" → intro segment
|
* "intro-introduction" → intro segment
|
||||||
* "1" … "4" → chapter (part)
|
* "1-1", "2-3", etc. → sub-chapter (song)
|
||||||
*/
|
*/
|
||||||
export async function getSvaContent(
|
export async function getSvaContent(
|
||||||
slug: string,
|
slug: string,
|
||||||
@@ -302,12 +349,8 @@ export async function getSvaContent(
|
|||||||
return getSvaIntroSegment(slug);
|
return getSvaIntroSegment(slug);
|
||||||
}
|
}
|
||||||
|
|
||||||
const num = Number(slug);
|
// Sub-chapter slug like "1-1", "2-3"
|
||||||
if (!Number.isNaN(num) && Number.isInteger(num) && num >= 1 && num <= 4) {
|
return getSvaChapter(slug);
|
||||||
return getSvaChapter(num);
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -315,131 +358,172 @@ export async function getSvaContent(
|
|||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Build the full sidebar navigation list.
|
* Build the full sidebar navigation list with hierarchical structure.
|
||||||
*
|
*
|
||||||
* Order: front-pages → foreword → introduction → parts 1–4
|
* Order: front-pages → foreword → introduction
|
||||||
|
* → parent-1 (label) → 1-1, 1-2, ... → parent-2 (label) → 2-1, 2-2, ...
|
||||||
|
*
|
||||||
|
* "parent" type items are non-clickable headings. Only "intro-segment" and
|
||||||
|
* "sub-chapter" items are navigable pages.
|
||||||
*/
|
*/
|
||||||
export async function getSvaSidebarItems(): Promise<SvaSidebarItem[]> {
|
export async function getSvaSidebarItems(): Promise<SvaSidebarItem[]> {
|
||||||
const [meta, allDocs] = await Promise.all([
|
let allDocs: {
|
||||||
getSvaMeta(),
|
slug: string;
|
||||||
prisma.proseChapter.findMany({
|
title: string;
|
||||||
|
type: string;
|
||||||
|
chapter: number | null;
|
||||||
|
order: number;
|
||||||
|
}[] = [];
|
||||||
|
|
||||||
|
const meta = await getSvaMeta();
|
||||||
|
|
||||||
|
try {
|
||||||
|
allDocs = await prisma.proseChapter.findMany({
|
||||||
where: { book: BOOK, volume: NO_VOLUME },
|
where: { book: BOOK, volume: NO_VOLUME },
|
||||||
orderBy: { order: "asc" },
|
orderBy: { order: "asc" },
|
||||||
select: { slug: true, title: true, type: true, chapter: true, order: true },
|
select: {
|
||||||
}),
|
slug: true,
|
||||||
]);
|
title: true,
|
||||||
|
type: true,
|
||||||
// If we got DB results, build from those
|
chapter: true,
|
||||||
if (meta && allDocs.length > 0) {
|
order: true,
|
||||||
return buildSidebarFromDocs(meta, allDocs);
|
},
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
// Prisma BSON decode error — build sidebar from meta alone
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fallback: build from local JSON
|
if (!meta) {
|
||||||
return buildSidebarFromLocal(meta);
|
// Try building from local JSON
|
||||||
}
|
return buildSidebarFromLocal();
|
||||||
|
}
|
||||||
|
|
||||||
function buildSidebarFromDocs(
|
|
||||||
meta: SvaMeta,
|
|
||||||
allDocs: { slug: string; title: string; type: string; chapter: number | null; order: number | null }[],
|
|
||||||
): SvaSidebarItem[] {
|
|
||||||
const items: SvaSidebarItem[] = [];
|
const items: SvaSidebarItem[] = [];
|
||||||
|
|
||||||
// Intro segments that come BEFORE chapters (order < 99)
|
// Intro segments that come BEFORE chapters (order < 99)
|
||||||
for (const doc of allDocs) {
|
if (allDocs.length > 0) {
|
||||||
if (doc.type === "intro-segment" && (doc.order ?? 0) < 99) {
|
for (const doc of allDocs) {
|
||||||
|
if (doc.type === "intro-segment" && (doc.order ?? 0) < 99) {
|
||||||
|
items.push({
|
||||||
|
slug: doc.slug,
|
||||||
|
label: doc.title,
|
||||||
|
type: "intro-segment",
|
||||||
|
parentChapter: null,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Fallback: local intro segments
|
||||||
|
const localIntros = getLocalIntroSegments();
|
||||||
|
for (const seg of localIntros) {
|
||||||
|
if ((seg.order ?? 0) < 99) {
|
||||||
|
items.push({
|
||||||
|
slug: `intro-${seg.slug}`,
|
||||||
|
label: seg.title,
|
||||||
|
type: "intro-segment",
|
||||||
|
parentChapter: null,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Chapters — grouped by parent chapter with sub-chapters
|
||||||
|
for (const ch of meta.chapters) {
|
||||||
|
// Parent heading (non-clickable label)
|
||||||
|
items.push({
|
||||||
|
slug: `parent-${ch.number}`,
|
||||||
|
label: `Part ${ch.number}: ${ch.title}`,
|
||||||
|
type: "parent",
|
||||||
|
parentChapter: ch.number,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Sub-chapters (songs) under this parent
|
||||||
|
for (const sub of ch.subChapters) {
|
||||||
items.push({
|
items.push({
|
||||||
slug: doc.slug,
|
slug: sub.slug,
|
||||||
label: doc.title,
|
label: sub.title,
|
||||||
type: "intro-segment",
|
type: "sub-chapter",
|
||||||
chapter: null,
|
parentChapter: ch.number,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Chapters 1–4 (use meta.chapters for consistent title/ordering)
|
|
||||||
for (const ch of meta.chapters) {
|
|
||||||
items.push({
|
|
||||||
slug: String(ch.number),
|
|
||||||
label: `Part ${ch.number}: ${ch.title}`,
|
|
||||||
type: "chapter",
|
|
||||||
chapter: ch.number,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// Intro segments that come AFTER chapters (order >= 99)
|
// Intro segments that come AFTER chapters (order >= 99)
|
||||||
for (const doc of allDocs) {
|
if (allDocs.length > 0) {
|
||||||
if (doc.type === "intro-segment" && (doc.order ?? 0) >= 99) {
|
for (const doc of allDocs) {
|
||||||
items.push({
|
if (doc.type === "intro-segment" && (doc.order ?? 0) >= 99) {
|
||||||
slug: doc.slug,
|
items.push({
|
||||||
label: doc.title,
|
slug: doc.slug,
|
||||||
type: "intro-segment",
|
label: doc.title,
|
||||||
chapter: null,
|
type: "intro-segment",
|
||||||
});
|
parentChapter: null,
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return items;
|
return items;
|
||||||
}
|
}
|
||||||
|
|
||||||
function buildSidebarFromLocal(meta: SvaMeta | null): SvaSidebarItem[] {
|
function buildSidebarFromLocal(): SvaSidebarItem[] {
|
||||||
const items: SvaSidebarItem[] = [];
|
const items: SvaSidebarItem[] = [];
|
||||||
|
|
||||||
// Local intro segments (front matter)
|
// Intro segments
|
||||||
const introSegs = getLocalIntroSegments();
|
const introSegs = getLocalIntroSegments();
|
||||||
const frontMatter = introSegs.filter((s) => (s.order ?? 0) < 99);
|
for (const seg of introSegs) {
|
||||||
const backMatter = introSegs.filter((s) => (s.order ?? 0) >= 99);
|
if ((seg.order ?? 0) < 99) {
|
||||||
|
|
||||||
for (const seg of frontMatter) {
|
|
||||||
items.push({
|
|
||||||
slug: `intro-${seg.slug}`,
|
|
||||||
label: seg.title,
|
|
||||||
type: "intro-segment",
|
|
||||||
chapter: null,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// Chapters from meta or generate default 1-4
|
|
||||||
if (meta && meta.chapters) {
|
|
||||||
for (const ch of meta.chapters) {
|
|
||||||
items.push({
|
items.push({
|
||||||
slug: String(ch.number),
|
slug: `intro-${seg.slug}`,
|
||||||
label: `Part ${ch.number}: ${ch.title}`,
|
label: seg.title,
|
||||||
type: "chapter",
|
type: "intro-segment",
|
||||||
chapter: ch.number,
|
parentChapter: null,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
} else {
|
}
|
||||||
// Fallback: scan local chapter JSONs
|
|
||||||
const chaptersDir = path.join(SVA_JSON_DIR, "chapters");
|
// Load meta for chapter structure
|
||||||
if (fs.existsSync(chaptersDir)) {
|
const introPath = path.join(SVA_JSON_DIR, "intro.json");
|
||||||
const files = fs
|
const meta = readLocalJson<SvaMeta>(introPath);
|
||||||
.readdirSync(chaptersDir)
|
|
||||||
.filter((f) => f.endsWith(".json"))
|
if (meta?.chapters) {
|
||||||
.sort();
|
for (const ch of meta.chapters) {
|
||||||
for (const file of files) {
|
items.push({
|
||||||
const ch = readLocalJson<SvaChapter>(path.join(chaptersDir, file));
|
slug: `parent-${ch.number}`,
|
||||||
if (ch) {
|
label: `Part ${ch.number}: ${ch.title}`,
|
||||||
const num = ch.chapter ?? parseInt(file.replace(".json", ""), 10);
|
type: "parent",
|
||||||
items.push({
|
parentChapter: ch.number,
|
||||||
slug: String(num),
|
});
|
||||||
label: `Part ${num}: ${ch.title}`,
|
|
||||||
type: "chapter",
|
for (const sub of ch.subChapters) {
|
||||||
chapter: num,
|
items.push({
|
||||||
});
|
slug: sub.slug,
|
||||||
}
|
label: sub.title,
|
||||||
|
type: "sub-chapter",
|
||||||
|
parentChapter: ch.number,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Back matter
|
// Back matter
|
||||||
for (const seg of backMatter) {
|
for (const seg of introSegs) {
|
||||||
items.push({
|
if ((seg.order ?? 0) >= 99) {
|
||||||
slug: `intro-${seg.slug}`,
|
items.push({
|
||||||
label: seg.title,
|
slug: `intro-${seg.slug}`,
|
||||||
type: "intro-segment",
|
label: seg.title,
|
||||||
chapter: null,
|
type: "intro-segment",
|
||||||
});
|
parentChapter: null,
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return items;
|
return items;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the flat list of navigable sidebar items (excludes "parent" headings).
|
||||||
|
* Used for prev/next navigation.
|
||||||
|
*/
|
||||||
|
export function getNavigableItems(items: SvaSidebarItem[]): SvaSidebarItem[] {
|
||||||
|
return items.filter((item) => item.type !== "parent");
|
||||||
|
}
|
||||||
|
|||||||
Binary file not shown.
+139
-42
@@ -1,15 +1,19 @@
|
|||||||
/**
|
/**
|
||||||
* seed-sva.ts — Push Songs of the Vaiṣṇava Ācāryas JSON data into MongoDB.
|
* seed-sva.ts — Push Songs of the Vaiṣṇava Ācāryas JSON data into MongoDB.
|
||||||
*
|
*
|
||||||
* Uses the native MongoDB driver (same pattern as seed-lcl.ts) to avoid
|
* Uses the native MongoDB driver (same pattern as seed-ssr.ts) to avoid
|
||||||
* Prisma replica-set write requirements.
|
* Prisma replica-set write requirements.
|
||||||
*
|
*
|
||||||
* SVA is a prose/song book with 4 parts (chapters) plus intro segments.
|
* SVA is a song book with 4 parent parts, each containing multiple songs
|
||||||
* We use volume=0 as the sentinel for "no volumes".
|
* (sub-chapters). Each song is stored as its own prose_chapters document.
|
||||||
|
*
|
||||||
|
* Sub-chapters are stored with slugs like "1-1", "1-2", "2-1", etc.
|
||||||
|
* Each sub-chapter JSON has parentChapter and subChapter fields used to
|
||||||
|
* compute a stable ordering: parentChapter * 100 + subChapter.
|
||||||
*
|
*
|
||||||
* Collections written:
|
* Collections written:
|
||||||
* - book_meta → one doc { book: "sva", data: <intro.json> }
|
* - book_meta → one doc { book: "sva", data: <intro.json> }
|
||||||
* - prose_chapters → intro segments + 4 chapters
|
* - prose_chapters → 3 intro segments + N sub-chapters (songs)
|
||||||
*
|
*
|
||||||
* Usage:
|
* Usage:
|
||||||
* npx tsx scripts/seed-sva.ts
|
* npx tsx scripts/seed-sva.ts
|
||||||
@@ -44,6 +48,40 @@ function readJson<T>(filePath: string): T {
|
|||||||
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
|
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Recursively sanitize all strings in a document to prevent BSON encoding errors.
|
||||||
|
*
|
||||||
|
* Strategy: NFC-normalize every string, strip control characters and lone
|
||||||
|
* surrogates, then round-trip through Buffer to guarantee the bytes are
|
||||||
|
* well-formed UTF-8 that the MongoDB C driver can handle.
|
||||||
|
*/
|
||||||
|
function sanitizeDoc(obj: unknown): unknown {
|
||||||
|
if (typeof obj === "string") {
|
||||||
|
// 1. NFC normalize (composes combining characters)
|
||||||
|
let s = obj.normalize("NFC");
|
||||||
|
// 2. Remove null bytes and C0 control chars (keep \t \n \r)
|
||||||
|
s = s.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, "");
|
||||||
|
// 3. Remove lone surrogates (U+D800–U+DFFF) that are invalid in BSON
|
||||||
|
s = s.replace(/[\uD800-\uDFFF]/g, "");
|
||||||
|
// 4. Remove zero-width characters that can confuse BSON length calculations
|
||||||
|
s = s.replace(/[\u200B-\u200F\u2028-\u202F\uFEFF]/g, "");
|
||||||
|
// 5. Round-trip through Buffer to guarantee valid UTF-8 byte sequence
|
||||||
|
s = Buffer.from(s, "utf-8").toString("utf-8");
|
||||||
|
return s;
|
||||||
|
}
|
||||||
|
if (Array.isArray(obj)) {
|
||||||
|
return obj.map(sanitizeDoc);
|
||||||
|
}
|
||||||
|
if (obj !== null && typeof obj === "object") {
|
||||||
|
const result: Record<string, unknown> = {};
|
||||||
|
for (const [key, value] of Object.entries(obj as Record<string, unknown>)) {
|
||||||
|
result[key] = sanitizeDoc(value);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
return obj;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Insert documents one at a time with a small delay every N docs.
|
* Insert documents one at a time with a small delay every N docs.
|
||||||
* Uses insertOne to avoid BSON wire-protocol corruption.
|
* Uses insertOne to avoid BSON wire-protocol corruption.
|
||||||
@@ -53,7 +91,36 @@ async function insertBatched(
|
|||||||
docs: Record<string, unknown>[],
|
docs: Record<string, unknown>[],
|
||||||
) {
|
) {
|
||||||
for (let i = 0; i < docs.length; i++) {
|
for (let i = 0; i < docs.length; i++) {
|
||||||
await col.insertOne(docs[i]);
|
const sanitized = sanitizeDoc(docs[i]) as Record<string, unknown>;
|
||||||
|
|
||||||
|
// Work around BSON driver bug with large string arrays:
|
||||||
|
// Store paragraphs as a single JSON-encoded string instead of a native
|
||||||
|
// array. The data layer (lib/sva.ts) already handles stringified JSON
|
||||||
|
// via normalizeParagraphs().
|
||||||
|
if (Array.isArray(sanitized.paragraphs)) {
|
||||||
|
// Stringify, then sanitize the resulting string to ensure clean UTF-8
|
||||||
|
let jsonStr = JSON.stringify(sanitized.paragraphs);
|
||||||
|
// NFC normalize the whole JSON string
|
||||||
|
jsonStr = jsonStr.normalize("NFC");
|
||||||
|
// Remove any stray control chars or surrogates
|
||||||
|
jsonStr = jsonStr.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, "");
|
||||||
|
jsonStr = jsonStr.replace(/[\uD800-\uDFFF]/g, "");
|
||||||
|
jsonStr = jsonStr.replace(/[\u200B-\u200F\u2028-\u202F\uFEFF]/g, "");
|
||||||
|
// Round-trip through Buffer to guarantee valid UTF-8 byte sequence
|
||||||
|
jsonStr = Buffer.from(jsonStr, "utf-8").toString("utf-8");
|
||||||
|
sanitized.paragraphs = jsonStr;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
await col.insertOne(sanitized);
|
||||||
|
} catch (err) {
|
||||||
|
const slug = (sanitized.slug as string) || "unknown";
|
||||||
|
const title = (sanitized.title as string) || "unknown";
|
||||||
|
console.error(
|
||||||
|
`\n ❌ Failed inserting doc ${i} (slug="${slug}", title="${title}")`,
|
||||||
|
);
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
if ((i + 1) % 100 === 0) {
|
if ((i + 1) % 100 === 0) {
|
||||||
await new Promise((r) => setTimeout(r, 100));
|
await new Promise((r) => setTimeout(r, 100));
|
||||||
}
|
}
|
||||||
@@ -71,7 +138,11 @@ type SvaIntroJson = {
|
|||||||
year: number;
|
year: number;
|
||||||
description: string;
|
description: string;
|
||||||
totalChapters: number;
|
totalChapters: number;
|
||||||
chapters: { number: number; title: string }[];
|
chapters: {
|
||||||
|
number: number;
|
||||||
|
title: string;
|
||||||
|
subChapters: { slug: string; title: string }[];
|
||||||
|
}[];
|
||||||
};
|
};
|
||||||
|
|
||||||
type SvaIntroSegmentJson = {
|
type SvaIntroSegmentJson = {
|
||||||
@@ -83,10 +154,13 @@ type SvaIntroSegmentJson = {
|
|||||||
paragraphs: string[];
|
paragraphs: string[];
|
||||||
};
|
};
|
||||||
|
|
||||||
type SvaChapterJson = {
|
type SvaSubChapterJson = {
|
||||||
book: string;
|
book: string;
|
||||||
chapter: number | null;
|
slug: string;
|
||||||
|
parentChapter: number;
|
||||||
|
subChapter: number;
|
||||||
title: string;
|
title: string;
|
||||||
|
parentTitle: string;
|
||||||
type: string;
|
type: string;
|
||||||
paragraphs: string[];
|
paragraphs: string[];
|
||||||
};
|
};
|
||||||
@@ -104,65 +178,88 @@ async function seedSva(db: ReturnType<MongoClient["db"]>) {
|
|||||||
|
|
||||||
const intro = readJson<SvaIntroJson>(introPath);
|
const intro = readJson<SvaIntroJson>(introPath);
|
||||||
|
|
||||||
// ---- Book meta ----
|
// ---- 1. Book meta ----
|
||||||
const metaCol = db.collection("book_meta");
|
const metaCol = db.collection("book_meta");
|
||||||
await metaCol.deleteMany({ book: "sva" });
|
await metaCol.deleteMany({ book: "sva" });
|
||||||
await metaCol.insertOne({ book: "sva", data: intro });
|
await metaCol.insertOne({ book: "sva", data: intro });
|
||||||
console.log(" 📖 SVA book meta inserted");
|
console.log(" 📖 SVA book meta inserted");
|
||||||
|
|
||||||
// ---- Prose chapters collection ----
|
// ---- 2. Prose chapters collection ----
|
||||||
const proseCol = db.collection("prose_chapters");
|
const proseCol = db.collection("prose_chapters");
|
||||||
await proseCol.deleteMany({ book: "sva" });
|
await proseCol.deleteMany({ book: "sva" });
|
||||||
|
|
||||||
const docs: Record<string, unknown>[] = [];
|
const docs: Record<string, unknown>[] = [];
|
||||||
|
|
||||||
// Intro segments (front-pages, foreword, introduction)
|
// Intro segments (front-pages, foreword, introduction)
|
||||||
|
const introSegmentFiles = [
|
||||||
|
"front-pages.json",
|
||||||
|
"foreword.json",
|
||||||
|
"introduction.json",
|
||||||
|
];
|
||||||
|
|
||||||
const introDir = path.join(SVA_DIR, "intro");
|
const introDir = path.join(SVA_DIR, "intro");
|
||||||
if (fs.existsSync(introDir)) {
|
for (const file of introSegmentFiles) {
|
||||||
const introFiles = fs
|
const filePath = path.join(introDir, file);
|
||||||
.readdirSync(introDir)
|
if (!fs.existsSync(filePath)) {
|
||||||
.filter((f) => f.endsWith(".json"))
|
console.warn(` ⚠️ Intro segment not found, skipping: ${file}`);
|
||||||
.sort();
|
continue;
|
||||||
|
|
||||||
for (const file of introFiles) {
|
|
||||||
const seg = readJson<SvaIntroSegmentJson>(path.join(introDir, file));
|
|
||||||
docs.push({
|
|
||||||
book: "sva",
|
|
||||||
volume: NO_VOLUME,
|
|
||||||
chapter: null,
|
|
||||||
title: seg.title,
|
|
||||||
type: seg.type || "intro-segment",
|
|
||||||
slug: `intro-${seg.slug || file.replace(".json", "")}`,
|
|
||||||
order: seg.order ?? 0,
|
|
||||||
paragraphs: seg.paragraphs,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
console.log(` 📄 Intro segments: ${introFiles.length}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Chapters (01.json – 04.json)
|
const seg = readJson<SvaIntroSegmentJson>(filePath);
|
||||||
|
|
||||||
|
docs.push({
|
||||||
|
book: "sva",
|
||||||
|
volume: NO_VOLUME,
|
||||||
|
chapter: null,
|
||||||
|
title: seg.title,
|
||||||
|
type: "intro-segment",
|
||||||
|
slug: `intro-${seg.slug}`,
|
||||||
|
order: seg.order,
|
||||||
|
paragraphs: seg.paragraphs,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
console.log(` 📄 Intro segments: ${docs.length}`);
|
||||||
|
|
||||||
|
// Sub-chapters (songs): 1-1.json, 1-2.json, 2-1.json, etc.
|
||||||
const chaptersDir = path.join(SVA_DIR, "chapters");
|
const chaptersDir = path.join(SVA_DIR, "chapters");
|
||||||
|
let songCount = 0;
|
||||||
if (fs.existsSync(chaptersDir)) {
|
if (fs.existsSync(chaptersDir)) {
|
||||||
const chapterFiles = fs
|
const chapterFiles = fs
|
||||||
.readdirSync(chaptersDir)
|
.readdirSync(chaptersDir)
|
||||||
.filter((f) => f.endsWith(".json"))
|
.filter((f) => f.endsWith(".json"))
|
||||||
.sort();
|
.sort((a, b) => {
|
||||||
|
// Sort by parentChapter then subChapter numerically
|
||||||
|
const parseSlug = (name: string) => {
|
||||||
|
const slug = name.replace(".json", "");
|
||||||
|
const parts = slug.split("-");
|
||||||
|
return {
|
||||||
|
parent: parseInt(parts[0], 10),
|
||||||
|
sub: parseInt(parts[1], 10),
|
||||||
|
};
|
||||||
|
};
|
||||||
|
const aa = parseSlug(a);
|
||||||
|
const bb = parseSlug(b);
|
||||||
|
if (aa.parent !== bb.parent) return aa.parent - bb.parent;
|
||||||
|
return aa.sub - bb.sub;
|
||||||
|
});
|
||||||
|
|
||||||
for (const file of chapterFiles) {
|
for (const file of chapterFiles) {
|
||||||
const ch = readJson<SvaChapterJson>(path.join(chaptersDir, file));
|
const sc = readJson<SvaSubChapterJson>(path.join(chaptersDir, file));
|
||||||
const chNum = ch.chapter ?? parseInt(file.replace(".json", ""), 10);
|
const order = sc.parentChapter * 100 + sc.subChapter;
|
||||||
|
|
||||||
docs.push({
|
docs.push({
|
||||||
book: "sva",
|
book: "sva",
|
||||||
volume: NO_VOLUME,
|
volume: NO_VOLUME,
|
||||||
chapter: chNum,
|
chapter: null, // null since slug is a string like "1-1"
|
||||||
title: ch.title,
|
title: sc.title,
|
||||||
type: ch.type || "chapter",
|
type: "sub-chapter",
|
||||||
slug: String(chNum),
|
slug: sc.slug,
|
||||||
order: 100 + chNum,
|
order,
|
||||||
paragraphs: ch.paragraphs,
|
paragraphs: sc.paragraphs,
|
||||||
});
|
});
|
||||||
|
songCount++;
|
||||||
}
|
}
|
||||||
console.log(` 📖 Chapters: ${chapterFiles.length}`);
|
console.log(` 🎵 Songs (sub-chapters): ${songCount}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Insert all docs
|
// Insert all docs
|
||||||
|
|||||||
+386
-169
@@ -1,13 +1,21 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""Extract Songs of the Vaiṣṇava Ācāryas (1972) from other.html.
|
"""Extract Songs of the Vaiṣṇava Ācāryas (1972) from other.html.
|
||||||
|
|
||||||
The book has 3 intro segments (front-pages, foreword, introduction),
|
Hierarchical sub-chapter extraction: each song within a part becomes its OWN
|
||||||
and 4 chapters (Parts 1–4).
|
chapter JSON file with a parent-child relationship.
|
||||||
|
|
||||||
|
Parent chapters (Parts 1–4) are containers only — no content of their own.
|
||||||
|
|
||||||
|
Numbering: Part 1, Song 1 → slug "1-1", Part 2, Song 3 → slug "2-3", etc.
|
||||||
|
|
||||||
|
Song boundaries are delimited by p49 bookmarks with text like:
|
||||||
|
"SVA 1: Śrī Guru Praṇāma"
|
||||||
|
"SVA 2: Aruṇodaya-kīrtana"
|
||||||
|
|
||||||
Output:
|
Output:
|
||||||
lib/jsons/books/sva/intro.json — Book metadata
|
lib/jsons/books/sva/intro.json — Book metadata with hierarchical chapters
|
||||||
lib/jsons/books/sva/intro/*.json — Intro segments
|
lib/jsons/books/sva/intro/*.json — Intro segments (front-pages, foreword, introduction)
|
||||||
lib/jsons/books/sva/chapters/*.json — Chapters (01–04)
|
lib/jsons/books/sva/chapters/*.json — Sub-chapter files (1-1.json, 1-2.json, 2-1.json, ...)
|
||||||
|
|
||||||
Usage:
|
Usage:
|
||||||
python3 scripts/sva_extract.py
|
python3 scripts/sva_extract.py
|
||||||
@@ -17,6 +25,7 @@ import json
|
|||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import sys
|
import sys
|
||||||
|
from collections import defaultdict
|
||||||
|
|
||||||
from bs4 import BeautifulSoup, NavigableString, Tag
|
from bs4 import BeautifulSoup, NavigableString, Tag
|
||||||
|
|
||||||
@@ -39,13 +48,17 @@ SCAGOUDY_TO_IAST = {
|
|||||||
"ç": "ś",
|
"ç": "ś",
|
||||||
"Ç": "Ś",
|
"Ç": "Ś",
|
||||||
"ì": "ṅ",
|
"ì": "ṅ",
|
||||||
|
"Ì": "Ṅ",
|
||||||
"ï": "ñ",
|
"ï": "ñ",
|
||||||
|
"Ï": "Ñ",
|
||||||
"ü": "ū",
|
"ü": "ū",
|
||||||
"Ü": "Ū",
|
"Ü": "Ū",
|
||||||
"é": "ī",
|
"é": "ī",
|
||||||
"É": "Ī",
|
"É": "Ī",
|
||||||
"à": "ṁ",
|
"à": "ṁ",
|
||||||
|
"À": "Ṁ",
|
||||||
"ù": "ḥ",
|
"ù": "ḥ",
|
||||||
|
"Ù": "Ḥ",
|
||||||
"è": "ṝ",
|
"è": "ṝ",
|
||||||
"û": "ṁl",
|
"û": "ṁl",
|
||||||
"ÿ": "ḷ",
|
"ÿ": "ḷ",
|
||||||
@@ -69,6 +82,18 @@ def decode_html_content(tag: Tag) -> str:
|
|||||||
)
|
)
|
||||||
# Clean up Apple-tab-span leftovers (tab characters)
|
# Clean up Apple-tab-span leftovers (tab characters)
|
||||||
inner = inner.replace("\t", " ")
|
inner = inner.replace("\t", " ")
|
||||||
|
# Force valid UTF-8 by round-tripping through encode/decode with replacement
|
||||||
|
inner = inner.encode("utf-8", errors="replace").decode("utf-8", errors="replace")
|
||||||
|
# Strip null bytes and other control characters that break BSON
|
||||||
|
inner = re.sub(r"[\x00-\x08\x0b\x0c\x0e-\x1f]", "", inner)
|
||||||
|
# Remove lone surrogates (U+D800–U+DFFF) that are invalid in BSON/MongoDB
|
||||||
|
inner = re.sub(r"[\ud800-\udfff]", "", inner)
|
||||||
|
# Merge adjacent bold tags: </b><b> → (nothing)
|
||||||
|
for _ in range(5):
|
||||||
|
inner = re.sub(r"</b>\s*<b>", "", inner)
|
||||||
|
# Remove empty bold and italic tags
|
||||||
|
inner = re.sub(r"<b>\s*</b>", "", inner)
|
||||||
|
inner = re.sub(r"<i>\s*</i>", "", inner)
|
||||||
# Normalize whitespace
|
# Normalize whitespace
|
||||||
inner = re.sub(r"\s+", " ", inner).strip()
|
inner = re.sub(r"\s+", " ", inner).strip()
|
||||||
return inner
|
return inner
|
||||||
@@ -138,10 +163,16 @@ SONG_TITLE_CLASS = "p27"
|
|||||||
SONG_TITLE_VARIANT_CLASS = "p169"
|
SONG_TITLE_VARIANT_CLASS = "p169"
|
||||||
|
|
||||||
# Verse line classes (italic, centered)
|
# Verse line classes (italic, centered)
|
||||||
VERSE_PADA_CLASSES = frozenset(["p5", "p53", "p24", "p66", "p255"])
|
VERSE_PADA_CLASSES = frozenset(["p5", "p53", "p24", "p66", "p255", "p134"])
|
||||||
|
|
||||||
# Verse number classes (e.g. "3)", "4)", "[1]") — centered but NOT verse text
|
# Verse number classes (e.g. "3)", "4)", "[1]") — centered but NOT verse text
|
||||||
VERSE_NUMBER_CLASS = "p248"
|
VERSE_NUMBER_CLASSES = frozenset(["p248", "p30"])
|
||||||
|
|
||||||
|
# Translation variant classes (bold, centered) — in addition to p56
|
||||||
|
TRANSLATION_VARIANT_CLASSES = frozenset(["p86", "p240", "p242"])
|
||||||
|
|
||||||
|
# Reference/purport note classes that should be centered
|
||||||
|
CENTERED_NOTE_CLASSES = frozenset(["p241", "p22"])
|
||||||
|
|
||||||
# Synonyms text
|
# Synonyms text
|
||||||
SYNONYMS_CLASS = "p54"
|
SYNONYMS_CLASS = "p54"
|
||||||
@@ -246,6 +277,18 @@ SKIP_CLASSES = frozenset(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Clean parent titles (pre-decoded IAST)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
CLEAN_PARENT_TITLES = {
|
||||||
|
1: "Prayers and Praṇāmas",
|
||||||
|
2: "Songs of Śrīla Bhaktivinoda Ṭhākura",
|
||||||
|
3: "Songs of Śrīla Narottama dāsa Ṭhākura",
|
||||||
|
4: "Songs by other Ācāryas",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Extraction helpers
|
# Extraction helpers
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -337,12 +380,12 @@ def process_table_t3(table: Tag) -> list:
|
|||||||
joined = "<br>".join(verse_lines)
|
joined = "<br>".join(verse_lines)
|
||||||
result.append(f'<div style="text-align:center"><i>{joined}</i></div>')
|
result.append(f'<div style="text-align:center"><i>{joined}</i></div>')
|
||||||
|
|
||||||
# Right cell: English translation
|
# Right cell: English translation — centered
|
||||||
right_ps = tds[1].find_all("p")
|
right_ps = tds[1].find_all("p")
|
||||||
for p in right_ps:
|
for p in right_ps:
|
||||||
html = decode_html_content(p)
|
html = decode_html_content(p)
|
||||||
if html and html.strip() and html.strip() != "\xa0":
|
if html and html.strip() and html.strip() != "\xa0":
|
||||||
result.append(html)
|
result.append(f'<div style="text-align:center">{html}</div>')
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
@@ -367,7 +410,7 @@ def extract_paragraphs(
|
|||||||
start_idx: int,
|
start_idx: int,
|
||||||
end_idx: int,
|
end_idx: int,
|
||||||
) -> list:
|
) -> list:
|
||||||
"""Extract paragraphs for a section (chapter, intro segment).
|
"""Extract paragraphs for a section (song, intro segment).
|
||||||
|
|
||||||
Handles all CSS classes within the SVA content.
|
Handles all CSS classes within the SVA content.
|
||||||
Handles <table class="t3"> and <table class="t2"> elements.
|
Handles <table class="t3"> and <table class="t2"> elements.
|
||||||
@@ -460,6 +503,19 @@ def extract_paragraphs(
|
|||||||
# Non-verse paragraph — flush any accumulated verse first
|
# Non-verse paragraph — flush any accumulated verse first
|
||||||
flush_verse()
|
flush_verse()
|
||||||
|
|
||||||
|
# Short p6 content (verse numbers like "1)", "2)") should be centered
|
||||||
|
# Long p6 content (body paragraphs) stays left-aligned
|
||||||
|
if cls == "p6":
|
||||||
|
html = decode_html_content(elem)
|
||||||
|
if html:
|
||||||
|
plain = re.sub(r"<[^>]+>", "", html).strip()
|
||||||
|
# Short content that looks like a verse number or bracketed ref
|
||||||
|
if len(plain) <= 10 and re.match(r"^[\[\(]?\d+[\]\)]?\.?$", plain):
|
||||||
|
result.append(f'<div style="text-align:center">{html}</div>')
|
||||||
|
else:
|
||||||
|
result.append(html)
|
||||||
|
continue
|
||||||
|
|
||||||
# Front pages text (p18)
|
# Front pages text (p18)
|
||||||
if cls == FRONT_PAGES_CLASS:
|
if cls == FRONT_PAGES_CLASS:
|
||||||
html = decode_html_content(elem)
|
html = decode_html_content(elem)
|
||||||
@@ -467,29 +523,29 @@ def extract_paragraphs(
|
|||||||
result.append(html)
|
result.append(html)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Section heading (p31) — bold
|
# Section heading (p31) — bold, centered
|
||||||
if cls == SECTION_HEADING_CLASS:
|
if cls == SECTION_HEADING_CLASS:
|
||||||
html = decode_html_content(elem)
|
html = decode_html_content(elem)
|
||||||
if html:
|
if html:
|
||||||
if not html.strip().startswith("<b>"):
|
if not html.strip().startswith("<b>"):
|
||||||
html = f"<b>{html}</b>"
|
html = f"<b>{html}</b>"
|
||||||
result.append(html)
|
result.append(f'<div style="text-align:center">{html}</div>')
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Song title (p27) — bold
|
# Song title (p27) — bold, centered
|
||||||
if cls == SONG_TITLE_CLASS:
|
if cls == SONG_TITLE_CLASS:
|
||||||
html = decode_html_content(elem)
|
html = decode_html_content(elem)
|
||||||
if html:
|
if html:
|
||||||
if not html.strip().startswith("<b>"):
|
if not html.strip().startswith("<b>"):
|
||||||
html = f"<b>{html}</b>"
|
html = f"<b>{html}</b>"
|
||||||
result.append(html)
|
result.append(f'<div style="text-align:center">{html}</div>')
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Song title variant (p169) — bold italic with author
|
# Song title variant (p169) — bold italic with author, centered
|
||||||
if cls == SONG_TITLE_VARIANT_CLASS:
|
if cls == SONG_TITLE_VARIANT_CLASS:
|
||||||
html = decode_html_content(elem)
|
html = decode_html_content(elem)
|
||||||
if html:
|
if html:
|
||||||
result.append(html)
|
result.append(f'<div style="text-align:center">{html}</div>')
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Synonyms (p54)
|
# Synonyms (p54)
|
||||||
@@ -499,8 +555,25 @@ def extract_paragraphs(
|
|||||||
result.append(html)
|
result.append(html)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Verse number heading (p248) — e.g. "3)", "4)", "[1]" — center it
|
# Verse number heading (p248, p30) — e.g. "3)", "4)", "[1]" — center it
|
||||||
if cls == VERSE_NUMBER_CLASS:
|
if cls in VERSE_NUMBER_CLASSES:
|
||||||
|
flush_verse()
|
||||||
|
html = decode_html_content(elem)
|
||||||
|
if html:
|
||||||
|
result.append(f'<div style="text-align:center">{html}</div>')
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Translation variant classes (p86, p240, p242) — bold, centered
|
||||||
|
if cls in TRANSLATION_VARIANT_CLASSES:
|
||||||
|
flush_verse()
|
||||||
|
html = decode_html_content(elem)
|
||||||
|
if html:
|
||||||
|
result.append(f'<div style="text-align:center">{html}</div>')
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Centered note classes (p241)
|
||||||
|
if cls in CENTERED_NOTE_CLASSES:
|
||||||
|
flush_verse()
|
||||||
html = decode_html_content(elem)
|
html = decode_html_content(elem)
|
||||||
if html:
|
if html:
|
||||||
result.append(f'<div style="text-align:center">{html}</div>')
|
result.append(f'<div style="text-align:center">{html}</div>')
|
||||||
@@ -513,18 +586,18 @@ def extract_paragraphs(
|
|||||||
result.append(f'<div style="text-align:center">{html}</div>')
|
result.append(f'<div style="text-align:center">{html}</div>')
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# SYNONYMS header (p165)
|
# SYNONYMS header (p165) — centered
|
||||||
if cls == SYNONYMS_HEADER_CLASS:
|
if cls == SYNONYMS_HEADER_CLASS:
|
||||||
html = decode_html_content(elem)
|
html = decode_html_content(elem)
|
||||||
if html:
|
if html:
|
||||||
result.append(html)
|
result.append(f'<div style="text-align:center">{html}</div>')
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# TRANSLATION header (p55)
|
# TRANSLATION header (p55) — centered
|
||||||
if cls == TRANSLATION_HEADER_CLASS:
|
if cls == TRANSLATION_HEADER_CLASS:
|
||||||
html = decode_html_content(elem)
|
html = decode_html_content(elem)
|
||||||
if html:
|
if html:
|
||||||
result.append(html)
|
result.append(f'<div style="text-align:center">{html}</div>')
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Mantra text (p19)
|
# Mantra text (p19)
|
||||||
@@ -578,11 +651,11 @@ def extract_paragraphs(
|
|||||||
result.append(html)
|
result.append(html)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Reference notes
|
# Reference notes (p187, p231, p233, p237, p32, p251) — centered
|
||||||
if cls in REFERENCE_CLASSES:
|
if cls in REFERENCE_CLASSES:
|
||||||
html = decode_html_content(elem)
|
html = decode_html_content(elem)
|
||||||
if html:
|
if html:
|
||||||
result.append(html)
|
result.append(f'<div style="text-align:center">{html}</div>')
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Bold text classes
|
# Bold text classes
|
||||||
@@ -592,11 +665,11 @@ def extract_paragraphs(
|
|||||||
result.append(html)
|
result.append(html)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Synonym notes (p247)
|
# Synonym note (p247) — centered
|
||||||
if cls == SYNONYM_NOTES_CLASS:
|
if cls == SYNONYM_NOTES_CLASS:
|
||||||
html = decode_html_content(elem)
|
html = decode_html_content(elem)
|
||||||
if html:
|
if html:
|
||||||
result.append(html)
|
result.append(f'<div style="text-align:center">{html}</div>')
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Bold headings (p224, p223)
|
# Bold headings (p224, p223)
|
||||||
@@ -686,8 +759,8 @@ def extract_sva(soup: BeautifulSoup) -> dict:
|
|||||||
|
|
||||||
Returns a dict with:
|
Returns a dict with:
|
||||||
"intro_segments": list of intro segment dicts
|
"intro_segments": list of intro segment dicts
|
||||||
"chapters": list of chapter dicts
|
"sub_chapters": list of sub-chapter dicts (one per song)
|
||||||
"chapter_listing": list of {number, title} for intro.json
|
"parent_chapters": list of parent chapter info for intro.json
|
||||||
"""
|
"""
|
||||||
# Get all top-level <p> and <table> elements
|
# Get all top-level <p> and <table> elements
|
||||||
all_elements = list(soup.find_all(["p", "table"]))
|
all_elements = list(soup.find_all(["p", "table"]))
|
||||||
@@ -700,18 +773,22 @@ def extract_sva(soup: BeautifulSoup) -> dict:
|
|||||||
# Step 2: Find all SVA bookmarks
|
# Step 2: Find all SVA bookmarks
|
||||||
bookmarks = find_sva_bookmarks(all_elements, sva_start, sva_end)
|
bookmarks = find_sva_bookmarks(all_elements, sva_start, sva_end)
|
||||||
print(f" Found {len(bookmarks)} SVA bookmarks")
|
print(f" Found {len(bookmarks)} SVA bookmarks")
|
||||||
for pos, text, cls in bookmarks[:20]:
|
for pos, text, cls in bookmarks[:30]:
|
||||||
print(f" Elem ~{pos} [{cls}]: {text[:80]}")
|
print(f" Elem ~{pos} [{cls}]: {text[:80]}")
|
||||||
if len(bookmarks) > 20:
|
if len(bookmarks) > 30:
|
||||||
print(f" ... and {len(bookmarks) - 20} more")
|
print(f" ... and {len(bookmarks) - 30} more")
|
||||||
|
|
||||||
# Step 3: Categorize bookmarks into sections
|
# Step 3: Categorize bookmarks
|
||||||
# p13 bookmarks define major sections:
|
# p13 bookmarks define major sections:
|
||||||
# "SVA: Foreword" → foreword
|
# "SVA: Foreword" → foreword
|
||||||
# "SVA: Introduction" → introduction
|
# "SVA: Introduction" → introduction
|
||||||
# "SVA 2: ..." → Part 2 start
|
# "SVA 2: ..." → Part 2 start
|
||||||
# "SVA 3: ..." → Part 3 start
|
# "SVA 3: ..." → Part 3 start
|
||||||
# "SVA 4: ..." → Part 4 start
|
# "SVA 4: ..." → Part 4 start
|
||||||
|
# p49 bookmarks define songs within parts:
|
||||||
|
# "SVA 1: ..." → songs in Part 1
|
||||||
|
# "SVA 2: ..." → songs in Part 2
|
||||||
|
# etc.
|
||||||
|
|
||||||
foreword_pos = None
|
foreword_pos = None
|
||||||
introduction_pos = None
|
introduction_pos = None
|
||||||
@@ -719,18 +796,8 @@ def extract_sva(soup: BeautifulSoup) -> dict:
|
|||||||
part3_pos = None
|
part3_pos = None
|
||||||
part4_pos = None
|
part4_pos = None
|
||||||
|
|
||||||
# p49 bookmarks define songs within parts:
|
|
||||||
# "SVA 1: ..." → songs in Part 1
|
|
||||||
# "SVA 2: ..." → songs in Part 2
|
|
||||||
# "SVA 3: ..." → songs in Part 3
|
|
||||||
# "SVA 4: ..." → songs in Part 4
|
|
||||||
|
|
||||||
# Also find the first SVA 1: bookmark (start of Part 1)
|
|
||||||
first_part1_pos = None
|
first_part1_pos = None
|
||||||
|
|
||||||
# Find "SVA: A Glimpse..." bookmark in intro
|
|
||||||
glimpse_pos = None
|
|
||||||
|
|
||||||
for pos, text, cls in bookmarks:
|
for pos, text, cls in bookmarks:
|
||||||
if cls == BOOKMARK_CLASS: # p13
|
if cls == BOOKMARK_CLASS: # p13
|
||||||
if text.strip() == "SVA: Foreword":
|
if text.strip() == "SVA: Foreword":
|
||||||
@@ -747,8 +814,6 @@ def extract_sva(soup: BeautifulSoup) -> dict:
|
|||||||
if cls == SONG_BOOKMARK_CLASS: # p49
|
if cls == SONG_BOOKMARK_CLASS: # p49
|
||||||
if text.strip().startswith("SVA 1:") and first_part1_pos is None:
|
if text.strip().startswith("SVA 1:") and first_part1_pos is None:
|
||||||
first_part1_pos = pos
|
first_part1_pos = pos
|
||||||
if "Glimpse" in text or "Life of" in text:
|
|
||||||
glimpse_pos = pos
|
|
||||||
|
|
||||||
print(f"\n Section boundaries:")
|
print(f"\n Section boundaries:")
|
||||||
print(f" Book start: elem {sva_start}")
|
print(f" Book start: elem {sva_start}")
|
||||||
@@ -821,63 +886,133 @@ def extract_sva(soup: BeautifulSoup) -> dict:
|
|||||||
"paragraphs": intro_paras,
|
"paragraphs": intro_paras,
|
||||||
}
|
}
|
||||||
|
|
||||||
# Step 5: Extract chapters (Parts 1–4)
|
|
||||||
chapter_boundaries = []
|
|
||||||
|
|
||||||
if first_part1_pos is not None:
|
|
||||||
ch1_end = part2_pos if part2_pos else sva_end
|
|
||||||
chapter_boundaries.append((1, "Prayers and Praṇāmas", first_part1_pos, ch1_end))
|
|
||||||
|
|
||||||
if part2_pos is not None:
|
|
||||||
ch2_end = part3_pos if part3_pos else sva_end
|
|
||||||
chapter_boundaries.append(
|
|
||||||
(2, "Songs of Śrīla Bhaktivinoda Ṭhākura", part2_pos, ch2_end)
|
|
||||||
)
|
|
||||||
|
|
||||||
if part3_pos is not None:
|
|
||||||
ch3_end = part4_pos if part4_pos else sva_end
|
|
||||||
chapter_boundaries.append(
|
|
||||||
(3, "Songs of Śrīla Narottama dāsa Ṭhākura", part3_pos, ch3_end)
|
|
||||||
)
|
|
||||||
|
|
||||||
if part4_pos is not None:
|
|
||||||
chapter_boundaries.append((4, "Songs by other Ācāryas", part4_pos, sva_end))
|
|
||||||
|
|
||||||
chapters = []
|
|
||||||
for ch_num, ch_title, ch_start, ch_end in chapter_boundaries:
|
|
||||||
# Start extraction from the element after the p13 bookmark (for parts 2-4)
|
|
||||||
# or from the p49 bookmark itself for part 1
|
|
||||||
content_start = ch_start
|
|
||||||
if ch_num >= 2:
|
|
||||||
# For parts 2-4, skip the p13 bookmark and the p31 heading that follows
|
|
||||||
content_start = ch_start + 1
|
|
||||||
|
|
||||||
paragraphs = extract_paragraphs(all_elements, content_start, ch_end)
|
|
||||||
|
|
||||||
ch = {
|
|
||||||
"book": "sva",
|
|
||||||
"chapter": ch_num,
|
|
||||||
"title": ch_title,
|
|
||||||
"type": "chapter",
|
|
||||||
"paragraphs": paragraphs,
|
|
||||||
}
|
|
||||||
chapters.append(ch)
|
|
||||||
|
|
||||||
# Step 6: Build results
|
|
||||||
intro_segments = [front_pages]
|
intro_segments = [front_pages]
|
||||||
if foreword:
|
if foreword:
|
||||||
intro_segments.append(foreword)
|
intro_segments.append(foreword)
|
||||||
if introduction:
|
if introduction:
|
||||||
intro_segments.append(introduction)
|
intro_segments.append(introduction)
|
||||||
|
|
||||||
chapter_listing = [
|
# Step 5: Collect all p49 song bookmarks grouped by part number
|
||||||
{"number": ch["chapter"], "title": ch["title"]} for ch in chapters
|
song_bookmarks_by_part = defaultdict(list)
|
||||||
]
|
|
||||||
|
for pos, text, cls in bookmarks:
|
||||||
|
if cls != SONG_BOOKMARK_CLASS:
|
||||||
|
continue
|
||||||
|
# Parse: "SVA 1: Śrī Guru Praṇāma" → part_num=1, title="Śrī Guru Praṇāma"
|
||||||
|
m = re.match(r"SVA\s+(\d+):\s+(.+)", text.strip())
|
||||||
|
if m:
|
||||||
|
part_num = int(m.group(1))
|
||||||
|
song_title = m.group(2).strip()
|
||||||
|
song_bookmarks_by_part[part_num].append(
|
||||||
|
{
|
||||||
|
"pos": pos,
|
||||||
|
"title": song_title,
|
||||||
|
"raw_text": text.strip(),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
# Step 6: Define part boundaries
|
||||||
|
part_boundaries = []
|
||||||
|
|
||||||
|
if first_part1_pos is not None:
|
||||||
|
ch1_end = part2_pos if part2_pos else sva_end
|
||||||
|
part_boundaries.append((1, first_part1_pos, ch1_end))
|
||||||
|
|
||||||
|
if part2_pos is not None:
|
||||||
|
ch2_end = part3_pos if part3_pos else sva_end
|
||||||
|
part_boundaries.append((2, part2_pos, ch2_end))
|
||||||
|
|
||||||
|
if part3_pos is not None:
|
||||||
|
ch3_end = part4_pos if part4_pos else sva_end
|
||||||
|
part_boundaries.append((3, part3_pos, ch3_end))
|
||||||
|
|
||||||
|
if part4_pos is not None:
|
||||||
|
part_boundaries.append((4, part4_pos, sva_end))
|
||||||
|
|
||||||
|
# Step 7: Extract sub-chapters (songs) for each part
|
||||||
|
sub_chapters = []
|
||||||
|
parent_chapters = []
|
||||||
|
|
||||||
|
for part_num, part_start, part_end in part_boundaries:
|
||||||
|
parent_title = CLEAN_PARENT_TITLES.get(part_num, f"Part {part_num}")
|
||||||
|
|
||||||
|
songs = song_bookmarks_by_part.get(part_num, [])
|
||||||
|
# Sort by position
|
||||||
|
songs.sort(key=lambda s: s["pos"])
|
||||||
|
|
||||||
|
sub_entries = []
|
||||||
|
song_counter = 0
|
||||||
|
|
||||||
|
for si, song in enumerate(songs):
|
||||||
|
song_title = decode_scagoudy(song["title"])
|
||||||
|
|
||||||
|
# Detect "Purport to..." bookmarks — merge into preceding song
|
||||||
|
is_purport = song_title.lower().startswith("purport to")
|
||||||
|
if (
|
||||||
|
is_purport
|
||||||
|
and sub_chapters
|
||||||
|
and sub_chapters[-1]["parentChapter"] == part_num
|
||||||
|
):
|
||||||
|
# Merge this purport's content into the last song
|
||||||
|
content_start = song["pos"] + 1
|
||||||
|
if si + 1 < len(songs):
|
||||||
|
content_end = songs[si + 1]["pos"]
|
||||||
|
else:
|
||||||
|
content_end = part_end
|
||||||
|
|
||||||
|
purport_paragraphs = extract_paragraphs(
|
||||||
|
all_elements, content_start, content_end
|
||||||
|
)
|
||||||
|
sub_chapters[-1]["paragraphs"].extend(purport_paragraphs)
|
||||||
|
continue
|
||||||
|
|
||||||
|
song_counter += 1
|
||||||
|
slug = f"{part_num}-{song_counter}"
|
||||||
|
|
||||||
|
# Content range: from this bookmark to next song bookmark (or part end)
|
||||||
|
# We start AFTER the bookmark element itself
|
||||||
|
content_start = song["pos"] + 1
|
||||||
|
if si + 1 < len(songs):
|
||||||
|
content_end = songs[si + 1]["pos"]
|
||||||
|
else:
|
||||||
|
content_end = part_end
|
||||||
|
|
||||||
|
paragraphs = extract_paragraphs(all_elements, content_start, content_end)
|
||||||
|
|
||||||
|
sub_chapters.append(
|
||||||
|
{
|
||||||
|
"book": "sva",
|
||||||
|
"slug": slug,
|
||||||
|
"parentChapter": part_num,
|
||||||
|
"subChapter": song_counter,
|
||||||
|
"title": song_title,
|
||||||
|
"parentTitle": parent_title,
|
||||||
|
"type": "sub-chapter",
|
||||||
|
"paragraphs": paragraphs,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
sub_entries.append({"slug": slug, "title": song_title})
|
||||||
|
|
||||||
|
parent_chapters.append(
|
||||||
|
{
|
||||||
|
"number": part_num,
|
||||||
|
"title": parent_title,
|
||||||
|
"subChapters": sub_entries,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
print(f"\n Part {part_num}: {parent_title} → {len(sub_entries)} songs")
|
||||||
|
for entry in sub_entries:
|
||||||
|
matching = [sc for sc in sub_chapters if sc["slug"] == entry["slug"]]
|
||||||
|
para_count = len(matching[0]["paragraphs"]) if matching else 0
|
||||||
|
print(
|
||||||
|
f" {entry['slug']:>5}: {para_count:>3} paragraphs — {entry['title']}"
|
||||||
|
)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"intro_segments": intro_segments,
|
"intro_segments": intro_segments,
|
||||||
"chapters": chapters,
|
"sub_chapters": sub_chapters,
|
||||||
"chapter_listing": chapter_listing,
|
"parent_chapters": parent_chapters,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -910,8 +1045,8 @@ def main():
|
|||||||
result = extract_sva(soup)
|
result = extract_sva(soup)
|
||||||
|
|
||||||
intro_segments = result["intro_segments"]
|
intro_segments = result["intro_segments"]
|
||||||
chapters = result["chapters"]
|
sub_chapters = result["sub_chapters"]
|
||||||
chapter_listing = result["chapter_listing"]
|
parent_chapters = result["parent_chapters"]
|
||||||
|
|
||||||
# --- Write intro.json ---
|
# --- Write intro.json ---
|
||||||
intro_meta = {
|
intro_meta = {
|
||||||
@@ -921,7 +1056,7 @@ def main():
|
|||||||
"year": 1972,
|
"year": 1972,
|
||||||
"description": "Bengali devotional songs by the great Vaiṣṇava teachers, with English translations and commentary.",
|
"description": "Bengali devotional songs by the great Vaiṣṇava teachers, with English translations and commentary.",
|
||||||
"totalChapters": 4,
|
"totalChapters": 4,
|
||||||
"chapters": chapter_listing,
|
"chapters": parent_chapters,
|
||||||
}
|
}
|
||||||
intro_path = os.path.join(output_base, "intro.json")
|
intro_path = os.path.join(output_base, "intro.json")
|
||||||
write_json(intro_meta, intro_path)
|
write_json(intro_meta, intro_path)
|
||||||
@@ -933,75 +1068,82 @@ def main():
|
|||||||
filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
|
filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
|
||||||
write_json(seg, filepath)
|
write_json(seg, filepath)
|
||||||
print(
|
print(
|
||||||
f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}"
|
f" {seg['order']:>2}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}"
|
||||||
)
|
)
|
||||||
|
|
||||||
# --- Write chapters ---
|
# --- Write sub-chapter files (songs) ---
|
||||||
print(f"\n--- Chapters ---")
|
print(f"\n--- Songs (Sub-Chapters) ---")
|
||||||
for ch in chapters:
|
for sc in sub_chapters:
|
||||||
ch_num_str = str(ch["chapter"]).zfill(2)
|
slug = sc["slug"]
|
||||||
filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
|
filepath = os.path.join(output_base, "chapters", f"{slug}.json")
|
||||||
write_json(ch, filepath)
|
write_json(sc, filepath)
|
||||||
print(
|
print(
|
||||||
f" Part {ch['chapter']}: {len(ch['paragraphs']):>3} paragraphs — "
|
f" {slug:>5}: {len(sc['paragraphs']):>3} paragraphs — "
|
||||||
f"{ch['title']}"
|
f"[Part {sc['parentChapter']}] {sc['title']}"
|
||||||
)
|
)
|
||||||
|
|
||||||
# --- Summary ---
|
# --- Summary ---
|
||||||
total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments)
|
total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments)
|
||||||
total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
|
total_chapter_paras = sum(len(sc["paragraphs"]) for sc in sub_chapters)
|
||||||
total_paras = total_intro_paras + total_chapter_paras
|
total_paras = total_intro_paras + total_chapter_paras
|
||||||
|
|
||||||
print(f"\n{'=' * 60}")
|
print(f"\n{'=' * 60}")
|
||||||
print(f" EXTRACTION SUMMARY")
|
print(f" EXTRACTION SUMMARY")
|
||||||
print(f"{'=' * 60}")
|
print(f"{'=' * 60}")
|
||||||
print(f" Intro segments: {len(intro_segments)}")
|
print(f" Intro segments: {len(intro_segments)}")
|
||||||
for seg in intro_segments:
|
print(f" Parent chapters: {len(parent_chapters)}")
|
||||||
print(
|
print(f" Songs (sub-ch): {len(sub_chapters)}")
|
||||||
f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs"
|
|
||||||
)
|
|
||||||
print(f" Chapters (Parts): {len(chapters)}")
|
|
||||||
for ch in chapters:
|
|
||||||
print(
|
|
||||||
f" Part {ch['chapter']}: {len(ch['paragraphs'])} paragraphs — {ch['title']}"
|
|
||||||
)
|
|
||||||
print(f" Total paragraphs: {total_paras}")
|
print(f" Total paragraphs: {total_paras}")
|
||||||
print(f" Intro: {total_intro_paras}")
|
print(f" Intro: {total_intro_paras}")
|
||||||
print(f" Chapters: {total_chapter_paras}")
|
print(f" Songs: {total_chapter_paras}")
|
||||||
if chapters:
|
if sub_chapters:
|
||||||
print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
|
print(f" Avg paras/song: {total_chapter_paras / len(sub_chapters):.1f}")
|
||||||
print(f" Output directory: {output_base}/")
|
print(f" Output directory: {output_base}/")
|
||||||
|
|
||||||
|
# --- Per-parent breakdown ---
|
||||||
|
print(f"\n--- Part Breakdown ---")
|
||||||
|
for pc in parent_chapters:
|
||||||
|
part_num = pc["number"]
|
||||||
|
sub_count = len(pc["subChapters"])
|
||||||
|
part_paras = sum(
|
||||||
|
len(sc["paragraphs"])
|
||||||
|
for sc in sub_chapters
|
||||||
|
if sc["parentChapter"] == part_num
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
f" Part {part_num}: {sub_count:>2} songs, {part_paras:>4} paragraphs — {pc['title']}"
|
||||||
|
)
|
||||||
|
|
||||||
# --- Samples ---
|
# --- Samples ---
|
||||||
print(f"\n--- Sample: Part 1 (Prayers and Praṇāmas) ---")
|
print(f"\n--- Sample: Front Pages ---")
|
||||||
if chapters:
|
if intro_segments:
|
||||||
ch1 = chapters[0]
|
front = intro_segments[0]
|
||||||
for i, p in enumerate(ch1["paragraphs"][:5]):
|
for i, p in enumerate(front["paragraphs"][:5]):
|
||||||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
||||||
print(f" [{i}] {preview}...")
|
print(f" [{i}] {preview}...")
|
||||||
|
|
||||||
if len(chapters) >= 2:
|
if sub_chapters:
|
||||||
print(f"\n--- Sample: Part 2 (Songs of Bhaktivinoda Ṭhākura) ---")
|
first_sc = sub_chapters[0]
|
||||||
ch2 = chapters[1]
|
print(f"\n--- Sample: {first_sc['slug']} ({first_sc['title']}) ---")
|
||||||
for i, p in enumerate(ch2["paragraphs"][:5]):
|
for i, p in enumerate(first_sc["paragraphs"][:5]):
|
||||||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
||||||
print(f" [{i}] {preview}...")
|
print(f" [{i}] {preview}...")
|
||||||
|
|
||||||
if len(chapters) >= 4:
|
if len(sub_chapters) > 1:
|
||||||
print(f"\n--- Sample: Part 4 (Songs by other Ācāryas) ---")
|
last_sc = sub_chapters[-1]
|
||||||
ch4 = chapters[3]
|
print(f"\n--- Sample: {last_sc['slug']} ({last_sc['title']}) ---")
|
||||||
for i, p in enumerate(ch4["paragraphs"][:5]):
|
for i, p in enumerate(last_sc["paragraphs"][:5]):
|
||||||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
||||||
print(f" [{i}] {preview}...")
|
print(f" [{i}] {preview}...")
|
||||||
|
|
||||||
# --- Quality Checks ---
|
# --- Quality Checks ---
|
||||||
print(f"\n--- Quality Checks ---")
|
print(f"\n--- Quality Checks ---")
|
||||||
|
|
||||||
# Check all 4 parts present
|
# Check all 4 parts present
|
||||||
expected_chapters = set(range(1, 5))
|
expected_parents = set(range(1, 5))
|
||||||
actual_chapters = set(ch["chapter"] for ch in chapters)
|
actual_parents = {pc["number"] for pc in parent_chapters}
|
||||||
missing = expected_chapters - actual_chapters
|
missing = expected_parents - actual_parents
|
||||||
extra = actual_chapters - expected_chapters
|
extra = actual_parents - expected_parents
|
||||||
if missing:
|
if missing:
|
||||||
print(f" ⚠ Missing parts: {sorted(missing)}")
|
print(f" ⚠ Missing parts: {sorted(missing)}")
|
||||||
if extra:
|
if extra:
|
||||||
@@ -1009,7 +1151,7 @@ def main():
|
|||||||
if not missing and not extra:
|
if not missing and not extra:
|
||||||
print(f" ✅ All 4 parts present")
|
print(f" ✅ All 4 parts present")
|
||||||
|
|
||||||
# Check intro segments
|
# Check all 3 intro segments present
|
||||||
expected_intros = {"front-pages", "foreword", "introduction"}
|
expected_intros = {"front-pages", "foreword", "introduction"}
|
||||||
actual_intros = {s["slug"] for s in intro_segments}
|
actual_intros = {s["slug"] for s in intro_segments}
|
||||||
missing_intros = expected_intros - actual_intros
|
missing_intros = expected_intros - actual_intros
|
||||||
@@ -1018,17 +1160,24 @@ def main():
|
|||||||
else:
|
else:
|
||||||
print(f" ✅ All 3 intro segments present")
|
print(f" ✅ All 3 intro segments present")
|
||||||
|
|
||||||
# Check empty chapters
|
# Check empty sub-chapters
|
||||||
empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
|
empty_scs = [sc["slug"] for sc in sub_chapters if len(sc["paragraphs"]) == 0]
|
||||||
if empty_chapters:
|
if empty_scs:
|
||||||
print(f" ⚠ Empty parts (0 paragraphs): {empty_chapters}")
|
print(f" ⚠ Empty songs (0 paragraphs): {empty_scs}")
|
||||||
else:
|
else:
|
||||||
print(f" ✅ No empty parts")
|
print(f" ✅ No empty songs")
|
||||||
|
|
||||||
|
# Check empty intro segments
|
||||||
|
empty_intros = [s["slug"] for s in intro_segments if len(s["paragraphs"]) == 0]
|
||||||
|
if empty_intros:
|
||||||
|
print(f" ⚠ Empty intro segments: {empty_intros}")
|
||||||
|
else:
|
||||||
|
print(f" ✅ No empty intro segments")
|
||||||
|
|
||||||
# Check ScaGoudy decoding
|
# Check ScaGoudy decoding
|
||||||
encoded_chars = set("äåöëòçüéàùè")
|
encoded_chars = set("äåöëòçüéàùè")
|
||||||
suspect_count = 0
|
suspect_count = 0
|
||||||
all_sections = chapters + intro_segments
|
all_sections = sub_chapters + intro_segments
|
||||||
for section in all_sections:
|
for section in all_sections:
|
||||||
for p in section["paragraphs"]:
|
for p in section["paragraphs"]:
|
||||||
plain = re.sub(r"<[^>]+>", "", p)
|
plain = re.sub(r"<[^>]+>", "", p)
|
||||||
@@ -1045,16 +1194,16 @@ def main():
|
|||||||
|
|
||||||
# Check verse blocks
|
# Check verse blocks
|
||||||
verse_block_count = 0
|
verse_block_count = 0
|
||||||
for ch in chapters:
|
for sc in sub_chapters:
|
||||||
for p in ch["paragraphs"]:
|
for p in sc["paragraphs"]:
|
||||||
if '<div style="text-align:center"><i>' in p:
|
if '<div style="text-align:center"><i>' in p:
|
||||||
verse_block_count += 1
|
verse_block_count += 1
|
||||||
print(f" ℹ {verse_block_count} verse blocks in chapters")
|
print(f" ℹ {verse_block_count} verse blocks in songs")
|
||||||
|
|
||||||
# Check table-derived content
|
# Check table-derived content
|
||||||
table_verse_count = 0
|
table_verse_count = 0
|
||||||
for ch in chapters:
|
for sc in sub_chapters:
|
||||||
for p in ch["paragraphs"]:
|
for p in sc["paragraphs"]:
|
||||||
if '<div style="text-align:center"><i>' in p and "<br>" in p:
|
if '<div style="text-align:center"><i>' in p and "<br>" in p:
|
||||||
table_verse_count += 1
|
table_verse_count += 1
|
||||||
print(f" ℹ {table_verse_count} multi-line verse blocks (likely from tables)")
|
print(f" ℹ {table_verse_count} multi-line verse blocks (likely from tables)")
|
||||||
@@ -1077,11 +1226,11 @@ def main():
|
|||||||
# Sample verse blocks
|
# Sample verse blocks
|
||||||
print(f"\n--- Sample Verse Blocks ---")
|
print(f"\n--- Sample Verse Blocks ---")
|
||||||
shown = 0
|
shown = 0
|
||||||
for ch in chapters:
|
for sc in sub_chapters:
|
||||||
for p in ch["paragraphs"]:
|
for p in sc["paragraphs"]:
|
||||||
if '<div style="text-align:center"><i>' in p:
|
if '<div style="text-align:center"><i>' in p:
|
||||||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
||||||
print(f" Part {ch['chapter']}: {preview}...")
|
print(f" {sc['slug']} ({sc['title']}): {preview}...")
|
||||||
shown += 1
|
shown += 1
|
||||||
if shown >= 5:
|
if shown >= 5:
|
||||||
break
|
break
|
||||||
@@ -1089,24 +1238,92 @@ def main():
|
|||||||
break
|
break
|
||||||
|
|
||||||
# Song title check
|
# Song title check
|
||||||
print(f"\n--- Song Titles in Chapters ---")
|
print(f"\n--- Song Titles ---")
|
||||||
song_count = 0
|
for pc in parent_chapters:
|
||||||
sample_songs = []
|
print(f" Part {pc['number']}: {pc['title']}")
|
||||||
for ch in chapters:
|
for sub in pc["subChapters"]:
|
||||||
ch_songs = 0
|
print(f" {sub['slug']:>5}: {sub['title']}")
|
||||||
for p in ch["paragraphs"]:
|
|
||||||
# Bold italic song titles
|
# Verify slug numbering continuity within each parent
|
||||||
if "<b><i>" in p and "</i></b>" in p:
|
print(f"\n--- Slug Numbering Check ---")
|
||||||
plain = re.sub(r"<[^>]+>", "", p).strip()
|
for pc in parent_chapters:
|
||||||
if len(plain) < 100:
|
part_num = pc["number"]
|
||||||
ch_songs += 1
|
slugs = [s["slug"] for s in pc["subChapters"]]
|
||||||
if len(sample_songs) < 8:
|
expected_slugs = [f"{part_num}-{i + 1}" for i in range(len(slugs))]
|
||||||
sample_songs.append(f" Part {ch['chapter']}: {plain}")
|
if slugs == expected_slugs:
|
||||||
song_count += ch_songs
|
print(
|
||||||
print(f" Part {ch['chapter']}: ~{ch_songs} song titles")
|
f" ✅ Part {part_num}: {len(slugs)} songs, slugs {slugs[0]}..{slugs[-1]}"
|
||||||
print(f" Total song titles: ~{song_count}")
|
)
|
||||||
for s in sample_songs:
|
else:
|
||||||
print(s)
|
print(
|
||||||
|
f" ⚠ Part {part_num}: slug mismatch! expected {expected_slugs}, got {slugs}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Sub-chapter size check
|
||||||
|
print(f"\n--- Song Size Check ---")
|
||||||
|
very_small = []
|
||||||
|
very_large = []
|
||||||
|
for sc in sub_chapters:
|
||||||
|
count = len(sc["paragraphs"])
|
||||||
|
if count < 1:
|
||||||
|
very_small.append((sc["slug"], count, sc["title"]))
|
||||||
|
elif count > 200:
|
||||||
|
very_large.append((sc["slug"], count, sc["title"]))
|
||||||
|
if very_small:
|
||||||
|
print(f" ⚠ Very small songs (0 paragraphs):")
|
||||||
|
for slug, count, title in very_small:
|
||||||
|
print(f" {slug}: {count} paragraphs — {title}")
|
||||||
|
else:
|
||||||
|
print(f" ✅ All songs have ≥ 1 paragraph")
|
||||||
|
if very_large:
|
||||||
|
print(f" ⚠ Very large songs (> 200 paragraphs):")
|
||||||
|
for slug, count, title in very_large:
|
||||||
|
print(f" {slug}: {count} paragraphs — {title}")
|
||||||
|
else:
|
||||||
|
print(f" ✅ All songs have ≤ 200 paragraphs")
|
||||||
|
|
||||||
|
# Sub-chapter title decoding check
|
||||||
|
print(f"\n--- Song Title Decoding Check ---")
|
||||||
|
bad_titles = 0
|
||||||
|
for sc in sub_chapters:
|
||||||
|
title = sc["title"]
|
||||||
|
if "å" in title or "ë" in title or "ñ" in title or "ä" in title:
|
||||||
|
print(f" ⚠ {sc['slug']} title has un-decoded ScaGoudy: {title}")
|
||||||
|
bad_titles += 1
|
||||||
|
if bad_titles == 0:
|
||||||
|
print(f" ✅ All {len(sub_chapters)} song titles properly decoded")
|
||||||
|
|
||||||
|
# Check for residual span tags
|
||||||
|
span_count = 0
|
||||||
|
for sc in sub_chapters:
|
||||||
|
for p in sc["paragraphs"]:
|
||||||
|
if 'class="' in p and "<span" in p:
|
||||||
|
span_count += 1
|
||||||
|
for seg in intro_segments:
|
||||||
|
for p in seg["paragraphs"]:
|
||||||
|
if 'class="' in p and "<span" in p:
|
||||||
|
span_count += 1
|
||||||
|
if span_count == 0:
|
||||||
|
print(f" ✅ No residual span class artifacts found")
|
||||||
|
else:
|
||||||
|
print(f" ⚠ {span_count} paragraphs still contain span class artifacts")
|
||||||
|
|
||||||
|
# Check for dash-line paragraphs
|
||||||
|
dash_count = 0
|
||||||
|
for sc in sub_chapters:
|
||||||
|
for p in sc["paragraphs"]:
|
||||||
|
plain = re.sub(r"<[^>]+>", "", p)
|
||||||
|
if is_dash_line(plain):
|
||||||
|
dash_count += 1
|
||||||
|
for seg in intro_segments:
|
||||||
|
for p in seg["paragraphs"]:
|
||||||
|
plain = re.sub(r"<[^>]+>", "", p)
|
||||||
|
if is_dash_line(plain):
|
||||||
|
dash_count += 1
|
||||||
|
if dash_count == 0:
|
||||||
|
print(f" ✅ No dashed-line paragraphs in output")
|
||||||
|
else:
|
||||||
|
print(f" ⚠ {dash_count} dashed-line paragraphs found in output")
|
||||||
|
|
||||||
print(f"\nDone! ✨")
|
print(f"\nDone! ✨")
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user