feat: kb both volumes
This commit is contained in:
gosangam
@@ -0,0 +1,205 @@
|
||||
import Link from "next/link";
|
||||
import { notFound } from "next/navigation";
|
||||
import { getProseChapter, getProseChapters } from "@/lib/api";
|
||||
import { PageHeader } from "@/components/page-header";
|
||||
import { Card, CardContent } from "@/components/ui/card";
|
||||
import { buttonVariants } from "@/components/ui/button";
|
||||
import { ChevronLeft, ChevronRight } from "lucide-react";
|
||||
import { cn } from "@/lib/utils";
|
||||
|
||||
export default async function KbChapterPage({
|
||||
params,
|
||||
}: {
|
||||
params: Promise<{ volume: string; slug: string }>;
|
||||
}) {
|
||||
const { volume, slug } = await params;
|
||||
const vol = parseInt(volume);
|
||||
if (isNaN(vol) || vol < 1 || vol > 2) notFound();
|
||||
|
||||
const chapter = await getProseChapter("kb", vol, slug);
|
||||
if (!chapter) notFound();
|
||||
|
||||
// Fetch all chapters/segments in this volume for sidebar + prev/next
|
||||
const allChapters = await getProseChapters("kb", vol);
|
||||
const currentIdx = allChapters.findIndex((ch) => ch.slug === slug);
|
||||
|
||||
const prev = currentIdx > 0 ? allChapters[currentIdx - 1] : null;
|
||||
const next =
|
||||
currentIdx < allChapters.length - 1 ? allChapters[currentIdx + 1] : null;
|
||||
|
||||
const chapterLabel =
|
||||
chapter.type === "chapter" && chapter.chapter != null
|
||||
? `Chapter ${chapter.chapter}`
|
||||
: null;
|
||||
|
||||
return (
|
||||
<div className="min-h-screen bg-background">
|
||||
<PageHeader sticky wide>
|
||||
<Link
|
||||
href={`/books/kb/${vol}`}
|
||||
className="text-sm text-muted-foreground hover:text-foreground transition-colors"
|
||||
>
|
||||
← Volume {vol}
|
||||
</Link>
|
||||
<h1 className="mt-3 text-2xl font-bold text-foreground">
|
||||
{chapterLabel && (
|
||||
<span className="text-primary">{chapterLabel}: </span>
|
||||
)}
|
||||
{chapter.title}
|
||||
</h1>
|
||||
</PageHeader>
|
||||
|
||||
<div className="w-full md:max-w-6xl mx-auto flex">
|
||||
{/* Left sidebar — chapter index */}
|
||||
<ChapterSidebar
|
||||
chapters={allChapters}
|
||||
currentSlug={slug}
|
||||
volume={vol}
|
||||
/>
|
||||
|
||||
{/* Main content */}
|
||||
<main className="flex-1 min-w-0 px-6 py-8 space-y-8">
|
||||
{/* Top navigation */}
|
||||
<ChapterNav prev={prev} next={next} volume={vol} />
|
||||
|
||||
{/* Chapter content */}
|
||||
<Card>
|
||||
<CardContent className="py-8 px-6 sm:px-8">
|
||||
<div className="space-y-4">
|
||||
{chapter.paragraphs.map((paragraph, i) => (
|
||||
<div
|
||||
key={i}
|
||||
className="text-foreground leading-relaxed"
|
||||
dangerouslySetInnerHTML={{ __html: paragraph }}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
{/* Bottom navigation */}
|
||||
<ChapterNav prev={prev} next={next} volume={vol} />
|
||||
</main>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Sidebar — chapter index
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type SidebarChapter = {
|
||||
slug: string;
|
||||
title: string;
|
||||
type: string;
|
||||
chapter: number | null;
|
||||
};
|
||||
|
||||
function ChapterSidebar({
|
||||
chapters,
|
||||
currentSlug,
|
||||
volume,
|
||||
}: {
|
||||
chapters: SidebarChapter[];
|
||||
currentSlug: string;
|
||||
volume: number;
|
||||
}) {
|
||||
return (
|
||||
<aside className="hidden lg:block w-56 shrink-0 sticky top-[89px] self-start max-h-[calc(100vh-89px)] overflow-y-auto py-8 pr-4">
|
||||
<h2 className="text-xs font-bold uppercase tracking-wider text-muted-foreground mb-3 px-2">
|
||||
Chapters
|
||||
</h2>
|
||||
<nav className="flex flex-col gap-0.5">
|
||||
{chapters.map((ch) => {
|
||||
const isActive = ch.slug === currentSlug;
|
||||
const label =
|
||||
ch.type === "chapter" && ch.chapter != null
|
||||
? `${ch.chapter}. ${ch.title}`
|
||||
: ch.title;
|
||||
|
||||
return (
|
||||
<Link
|
||||
key={ch.slug}
|
||||
href={`/books/kb/${volume}/${ch.slug}`}
|
||||
className={cn(
|
||||
"px-2 py-1.5 rounded-md text-sm transition-colors truncate",
|
||||
isActive
|
||||
? "bg-primary/10 text-primary font-medium"
|
||||
: "text-muted-foreground hover:bg-muted hover:text-foreground",
|
||||
)}
|
||||
title={label}
|
||||
>
|
||||
{label}
|
||||
</Link>
|
||||
);
|
||||
})}
|
||||
</nav>
|
||||
</aside>
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Prev / Next navigation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type NavChapter = {
|
||||
slug: string;
|
||||
title: string;
|
||||
type: string;
|
||||
chapter: number | null;
|
||||
};
|
||||
|
||||
function ChapterNav({
|
||||
prev,
|
||||
next,
|
||||
volume,
|
||||
}: {
|
||||
prev: NavChapter | null;
|
||||
next: NavChapter | null;
|
||||
volume: number;
|
||||
}) {
|
||||
return (
|
||||
<div className="flex items-center justify-between gap-3">
|
||||
{prev ? (
|
||||
<Link
|
||||
href={`/books/kb/${volume}/${prev.slug}`}
|
||||
className={cn(
|
||||
buttonVariants({ variant: "outline", size: "sm" }),
|
||||
"gap-1.5",
|
||||
)}
|
||||
>
|
||||
<ChevronLeft className="size-4" />
|
||||
<span className="hidden sm:inline truncate max-w-[180px]">
|
||||
{prev.type === "chapter" && prev.chapter != null
|
||||
? `Ch. ${prev.chapter}`
|
||||
: prev.title}
|
||||
</span>
|
||||
<span className="sm:hidden">Previous</span>
|
||||
</Link>
|
||||
) : (
|
||||
<div />
|
||||
)}
|
||||
|
||||
{next ? (
|
||||
<Link
|
||||
href={`/books/kb/${volume}/${next.slug}`}
|
||||
className={cn(
|
||||
buttonVariants({ variant: "outline", size: "sm" }),
|
||||
"gap-1.5",
|
||||
)}
|
||||
>
|
||||
<span className="hidden sm:inline truncate max-w-[180px]">
|
||||
{next.type === "chapter" && next.chapter != null
|
||||
? `Ch. ${next.chapter}`
|
||||
: next.title}
|
||||
</span>
|
||||
<span className="sm:hidden">Next</span>
|
||||
<ChevronRight className="size-4" />
|
||||
</Link>
|
||||
) : (
|
||||
<div />
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
import Link from "next/link";
|
||||
import { notFound } from "next/navigation";
|
||||
import { getBookMeta, getProseChapters } from "@/lib/api";
|
||||
import { PageHeader } from "@/components/page-header";
|
||||
import { Card, CardContent } from "@/components/ui/card";
|
||||
|
||||
export default async function KbVolumePage({
|
||||
params,
|
||||
}: {
|
||||
params: Promise<{ volume: string }>;
|
||||
}) {
|
||||
const { volume } = await params;
|
||||
const vol = parseInt(volume);
|
||||
if (isNaN(vol) || vol < 1 || vol > 2) notFound();
|
||||
|
||||
const meta = await getBookMeta("kb");
|
||||
if (!meta) notFound();
|
||||
|
||||
const title = meta.title as string;
|
||||
|
||||
const [introSegments, chapters] = await Promise.all([
|
||||
getProseChapters("kb", vol, "intro-segment"),
|
||||
getProseChapters("kb", vol, "chapter"),
|
||||
]);
|
||||
|
||||
return (
|
||||
<div className="min-h-screen bg-background">
|
||||
<PageHeader>
|
||||
<Link
|
||||
href="/books/kb"
|
||||
className="text-sm text-muted-foreground hover:text-foreground transition-colors"
|
||||
>
|
||||
← {title}
|
||||
</Link>
|
||||
<h1 className="mt-3 text-3xl font-bold text-foreground">
|
||||
Volume {vol}
|
||||
</h1>
|
||||
<p className="mt-1 text-muted-foreground">{chapters.length} chapters</p>
|
||||
</PageHeader>
|
||||
<main className="max-w-3xl mx-auto px-6 py-8 space-y-8">
|
||||
{/* Front Matter */}
|
||||
{introSegments.length > 0 && (
|
||||
<div>
|
||||
<h2 className="text-sm font-bold uppercase tracking-wider text-muted-foreground mb-3">
|
||||
Front Matter
|
||||
</h2>
|
||||
<Card className="divide-y divide-border">
|
||||
<CardContent className="p-0">
|
||||
<ul className="divide-y divide-border">
|
||||
{introSegments.map((seg) => (
|
||||
<li key={seg.slug}>
|
||||
<Link
|
||||
href={`/books/kb/${vol}/${seg.slug}`}
|
||||
className="flex items-center justify-between px-5 py-4 hover:bg-muted transition-colors"
|
||||
>
|
||||
<div>
|
||||
<p className="font-medium text-foreground">
|
||||
{seg.title}
|
||||
</p>
|
||||
</div>
|
||||
<span className="text-sm text-muted-foreground shrink-0 ml-4">
|
||||
{seg.paragraphs.length} paragraphs
|
||||
</span>
|
||||
</Link>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</CardContent>
|
||||
</Card>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Chapters */}
|
||||
<div>
|
||||
<h2 className="text-sm font-bold uppercase tracking-wider text-muted-foreground mb-3">
|
||||
Chapters
|
||||
</h2>
|
||||
<Card className="divide-y divide-border">
|
||||
<CardContent className="p-0">
|
||||
<ul className="divide-y divide-border">
|
||||
{chapters.map((ch) => (
|
||||
<li key={ch.slug}>
|
||||
<Link
|
||||
href={`/books/kb/${vol}/${ch.slug}`}
|
||||
className="flex items-center justify-between px-5 py-4 hover:bg-muted transition-colors"
|
||||
>
|
||||
<div>
|
||||
<span className="text-sm text-primary font-medium">
|
||||
Chapter {ch.chapter}
|
||||
</span>
|
||||
<p className="font-medium text-foreground">
|
||||
{ch.title}
|
||||
</p>
|
||||
</div>
|
||||
<span className="text-sm text-muted-foreground shrink-0 ml-4">
|
||||
{ch.paragraphs.length} paragraphs
|
||||
</span>
|
||||
</Link>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</CardContent>
|
||||
</Card>
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
import Link from "next/link";
|
||||
import { notFound } from "next/navigation";
|
||||
import { getBookMeta } from "@/lib/api";
|
||||
import { PageHeader } from "@/components/page-header";
|
||||
import { Card, CardContent } from "@/components/ui/card";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
|
||||
type VolumeInfo = {
|
||||
number: number;
|
||||
totalChapters: number;
|
||||
};
|
||||
|
||||
export default async function KbPage() {
|
||||
const meta = await getBookMeta("kb");
|
||||
if (!meta) notFound();
|
||||
|
||||
const title = meta.title as string;
|
||||
const author = meta.author as string;
|
||||
const description = meta.description as string | undefined;
|
||||
const volumes = meta.volumes as VolumeInfo[];
|
||||
|
||||
return (
|
||||
<div className="min-h-screen bg-background">
|
||||
<PageHeader>
|
||||
<Link
|
||||
href="/books"
|
||||
className="text-sm text-muted-foreground hover:text-foreground transition-colors"
|
||||
>
|
||||
← Books
|
||||
</Link>
|
||||
<h1 className="mt-3 text-3xl font-bold text-foreground">{title}</h1>
|
||||
<p className="mt-1 text-muted-foreground">{author}</p>
|
||||
</PageHeader>
|
||||
<main className="max-w-3xl mx-auto px-6 py-8">
|
||||
{description && (
|
||||
<p className="text-muted-foreground mb-6">{description}</p>
|
||||
)}
|
||||
<Card className="divide-y divide-border">
|
||||
<CardContent className="p-0">
|
||||
<ul className="divide-y divide-border">
|
||||
{volumes.map((vol) => (
|
||||
<li key={vol.number}>
|
||||
<Link
|
||||
href={`/books/kb/${vol.number}`}
|
||||
className="flex items-center justify-between px-5 py-4 hover:bg-muted transition-colors"
|
||||
>
|
||||
<div>
|
||||
<Badge variant="secondary" className="mb-1">
|
||||
Volume {vol.number}
|
||||
</Badge>
|
||||
<p className="font-medium text-foreground">
|
||||
{vol.totalChapters} chapters
|
||||
</p>
|
||||
</div>
|
||||
</Link>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</CardContent>
|
||||
</Card>
|
||||
</main>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -21,6 +21,12 @@ interface ChapterBookMeta extends BookMeta {
|
||||
chapters: ChapterEntry[];
|
||||
}
|
||||
|
||||
interface KbMeta extends BookMeta {
|
||||
totalChapters: number;
|
||||
description: string;
|
||||
volumes: { number: number; totalChapters: number }[];
|
||||
}
|
||||
|
||||
interface SbMeta extends BookMeta {
|
||||
totalCantos: number;
|
||||
cantos: CantoEntry[];
|
||||
@@ -112,6 +118,16 @@ function buildBookList(allBooks: BookMeta[]): BookInfo[] {
|
||||
totalVerses,
|
||||
href: "/books/cc",
|
||||
});
|
||||
} else if (slug === "kb") {
|
||||
const kb = data as KbMeta;
|
||||
books.push({
|
||||
slug,
|
||||
title,
|
||||
author,
|
||||
description: `${kb.totalChapters} chapters · 2 volumes`,
|
||||
totalVerses: 0,
|
||||
href: "/books/kb",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -156,11 +172,13 @@ export default async function BooksPage() {
|
||||
<CardContent>
|
||||
<div className="flex items-center gap-3 text-sm">
|
||||
<Badge variant="secondary">{book.description}</Badge>
|
||||
{book.totalVerses > 0 && (
|
||||
<ExtractedVerseCount
|
||||
count={book.totalVerses}
|
||||
label="extracted verses"
|
||||
className="text-muted-foreground"
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
+129
-1
@@ -3,10 +3,11 @@
|
||||
*
|
||||
* These functions query MongoDB via Prisma and are called directly by
|
||||
* Next.js server components. The /api/* routes exist for external consumers
|
||||
* and for client-side infinite-scroll fetching.
|
||||
* and for client-side verse-navigation fetching.
|
||||
*/
|
||||
|
||||
import { prisma } from "@/lib/db";
|
||||
import type { JsonValue } from "@prisma/client/runtime/library";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
@@ -56,6 +57,22 @@ export type GlossaryRecord = {
|
||||
definitionHtml: string;
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Prose chapter types (for KB and other prose books)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type ProseChapterRecord = {
|
||||
id: string;
|
||||
book: string;
|
||||
volume: number;
|
||||
chapter: number | null;
|
||||
title: string;
|
||||
type: string;
|
||||
slug: string;
|
||||
order: number;
|
||||
paragraphs: string[];
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Book meta
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -174,6 +191,117 @@ export async function getSingleVerse(
|
||||
return result as unknown as VerseRecord | null;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Prose chapters (KB and other prose books)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Get all prose chapters/segments for a given book and volume,
|
||||
* optionally filtered by type ("chapter" | "intro-segment").
|
||||
* Ordered by the `order` field.
|
||||
*/
|
||||
export async function getProseChapters(
|
||||
book: string,
|
||||
volume: number,
|
||||
type?: string,
|
||||
): Promise<ProseChapterRecord[]> {
|
||||
const where: Record<string, unknown> = { book, volume };
|
||||
if (type) where.type = type;
|
||||
|
||||
const results = await prisma.proseChapter.findMany({
|
||||
where,
|
||||
orderBy: { order: "asc" },
|
||||
});
|
||||
|
||||
return results.map(normalizeProse);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a single prose chapter by book + volume + slug.
|
||||
*/
|
||||
export async function getProseChapter(
|
||||
book: string,
|
||||
volume: number,
|
||||
slug: string,
|
||||
): Promise<ProseChapterRecord | null> {
|
||||
const result = await prisma.proseChapter.findUnique({
|
||||
where: {
|
||||
book_volume_slug: { book, volume, slug },
|
||||
},
|
||||
});
|
||||
|
||||
return result ? normalizeProse(result) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a prose chapter from the DB.
|
||||
* Some chapters were stored with paragraphs as a JSON string (fallback
|
||||
* for BSON wire-protocol corruption). This normalizes them back to arrays.
|
||||
*/
|
||||
function normalizeProse(doc: {
|
||||
id: string;
|
||||
book: string;
|
||||
volume: number;
|
||||
chapter: number | null;
|
||||
title: string;
|
||||
type: string;
|
||||
slug: string;
|
||||
order: number;
|
||||
paragraphs: string[] | JsonValue;
|
||||
}): ProseChapterRecord {
|
||||
let paragraphs: string[];
|
||||
|
||||
if (typeof doc.paragraphs === "string") {
|
||||
// Was stored as stringified JSON — parse it back
|
||||
try {
|
||||
paragraphs = JSON.parse(doc.paragraphs);
|
||||
} catch {
|
||||
paragraphs = [doc.paragraphs];
|
||||
}
|
||||
} else if (Array.isArray(doc.paragraphs)) {
|
||||
paragraphs = doc.paragraphs as string[];
|
||||
} else {
|
||||
paragraphs = [];
|
||||
}
|
||||
|
||||
// Sanitize HTML — strip dangerous tags and event handlers
|
||||
paragraphs = paragraphs.map(sanitizeHtml);
|
||||
|
||||
return {
|
||||
id: doc.id,
|
||||
book: doc.book,
|
||||
volume: doc.volume,
|
||||
chapter: doc.chapter,
|
||||
title: doc.title,
|
||||
type: doc.type,
|
||||
slug: doc.slug,
|
||||
order: doc.order,
|
||||
paragraphs,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Sanitize an HTML string by stripping dangerous tags and attributes.
|
||||
* Keeps safe formatting tags: <i>, <b>, <em>, <strong>, <br>, <div>, <span>.
|
||||
*/
|
||||
function sanitizeHtml(html: string): string {
|
||||
// Remove <script>...</script>, <style>...</style>, <iframe>...</iframe>
|
||||
let clean = html.replace(/<script[\s>][\s\S]*?<\/script>/gi, "");
|
||||
clean = clean.replace(/<style[\s>][\s\S]*?<\/style>/gi, "");
|
||||
clean = clean.replace(/<iframe[\s>][\s\S]*?<\/iframe>/gi, "");
|
||||
// Remove lone <script>, <style>, <iframe> opening/closing tags
|
||||
clean = clean.replace(
|
||||
/<\/?(?:script|style|iframe|object|embed|form|input|textarea|select|button)[^>]*>/gi,
|
||||
"",
|
||||
);
|
||||
// Remove event handler attributes (onclick, onload, onerror, etc.)
|
||||
clean = clean.replace(/\s+on\w+\s*=\s*(?:"[^"]*"|'[^']*'|[^\s>]+)/gi, "");
|
||||
// Remove javascript: URLs
|
||||
clean = clean.replace(/href\s*=\s*"javascript:[^"]*"/gi, 'href=""');
|
||||
clean = clean.replace(/href\s*=\s*'javascript:[^']*'/gi, "href=''");
|
||||
return clean;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Glossary
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
Generated
+28
@@ -8362,6 +8362,34 @@
|
||||
"integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/smart-buffer": {
|
||||
"version": "4.2.0",
|
||||
"resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz",
|
||||
"integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==",
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"peer": true,
|
||||
"engines": {
|
||||
"node": ">= 6.0.0",
|
||||
"npm": ">= 3.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/socks": {
|
||||
"version": "2.8.7",
|
||||
"resolved": "https://registry.npmjs.org/socks/-/socks-2.8.7.tgz",
|
||||
"integrity": "sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A==",
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"ip-address": "^10.0.1",
|
||||
"smart-buffer": "^4.2.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 10.0.0",
|
||||
"npm": ">= 3.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/source-map": {
|
||||
"version": "0.6.1",
|
||||
"resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
|
||||
|
||||
@@ -63,6 +63,36 @@ model BookMeta {
|
||||
@@map("book_meta")
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// ProseChapter — for prose books like Kṛṣṇa Book (no verse structure)
|
||||
//
|
||||
// book: "kb"
|
||||
// volume: volume number (1 or 2 for KB)
|
||||
// chapter: chapter number within the volume
|
||||
// title: chapter title
|
||||
// type: "chapter" | "intro-segment"
|
||||
// slug: URL-friendly identifier (e.g. "01", "preface", "words-from-apple")
|
||||
// order: sort order (chapter number for chapters, custom for intro segments)
|
||||
// paragraphs: array of HTML paragraph strings
|
||||
// ---------------------------------------------------------------------------
|
||||
model ProseChapter {
|
||||
id String @id @default(auto()) @map("_id") @db.ObjectId
|
||||
book String // "kb"
|
||||
volume Int
|
||||
chapter Int? // null for intro segments
|
||||
title String
|
||||
type String // "chapter" | "intro-segment"
|
||||
slug String
|
||||
order Int
|
||||
paragraphs String[]
|
||||
|
||||
@@unique([book, volume, slug])
|
||||
@@index([book])
|
||||
@@index([book, volume])
|
||||
@@index([book, volume, type])
|
||||
@@map("prose_chapters")
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// GlossaryEntry — individual glossary term + definition
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -0,0 +1,309 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract Kṛṣṇa Book introduction and build intro.json with chapter listing.
|
||||
|
||||
Reads public/html/other.html to extract:
|
||||
1. The introduction/preface text (lines ~713–771)
|
||||
2. The full chapter listing from p13 bookmark tags (KB 1970-V-C / Title)
|
||||
3. Chapter end markers ("Thus ends the Bhaktivedanta purport of the Nth Chapter...")
|
||||
|
||||
Output: lib/jsons/books/kb/intro.json
|
||||
|
||||
Usage:
|
||||
python3 scripts/kb_extract_intro.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ScaGoudy → IAST Unicode mapping
|
||||
# ---------------------------------------------------------------------------
|
||||
SCAGOUDY_TO_IAST = {
|
||||
"ä": "ā",
|
||||
"Ä": "Ā",
|
||||
"å": "ṛ",
|
||||
"Å": "Ṛ",
|
||||
"ñ": "ṣ",
|
||||
"Ñ": "Ṣ",
|
||||
"ö": "ṭ",
|
||||
"Ö": "Ṭ",
|
||||
"ë": "ṇ",
|
||||
"Ë": "Ṇ",
|
||||
"ò": "ḍ",
|
||||
"Ò": "Ḍ",
|
||||
"ç": "ś",
|
||||
"Ç": "Ś",
|
||||
"ì": "ṅ",
|
||||
"ï": "ñ",
|
||||
"ü": "ū",
|
||||
"Ü": "Ū",
|
||||
"é": "ī",
|
||||
"É": "Ī",
|
||||
"à": "ṁ",
|
||||
"ù": "ḥ",
|
||||
"è": "ṝ",
|
||||
"û": "ṁl",
|
||||
"ÿ": "ḷ",
|
||||
}
|
||||
|
||||
|
||||
def decode_scagoudy(text: str) -> str:
|
||||
for src, dst in SCAGOUDY_TO_IAST.items():
|
||||
text = text.replace(src, dst)
|
||||
return text
|
||||
|
||||
|
||||
def decode_html_content(tag: Tag) -> str:
|
||||
inner = "".join(str(child) for child in tag.children)
|
||||
inner = decode_scagoudy(inner)
|
||||
inner = re.sub(r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner)
|
||||
inner = re.sub(r"\s+", " ", inner).strip()
|
||||
return inner
|
||||
|
||||
|
||||
def decode_text(tag: Tag) -> str:
|
||||
return decode_scagoudy(tag.get_text().strip())
|
||||
|
||||
|
||||
def get_css_class(tag: Tag) -> str:
|
||||
classes = tag.get("class", [])
|
||||
return classes[0] if classes else ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Extract introduction paragraphs (lines ~713–771)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
# CSS classes used for Sanskrit verse pādas in the introduction
|
||||
VERSE_PADA_CLASSES = frozenset(["p4", "p5", "p10"])
|
||||
|
||||
|
||||
def extract_introduction(soup: BeautifulSoup) -> list[str]:
|
||||
"""Extract the Kṛṣṇa Book introduction/preface paragraphs.
|
||||
|
||||
Starts at the 'KṚṢṆA BOOK-Introduction' header and ends at the
|
||||
'February 26th, 1970' date line (first occurrence — the undated
|
||||
typed document version).
|
||||
|
||||
Sanskrit verse lines (p4/p5 classes) are grouped into a single
|
||||
center-aligned italic block.
|
||||
"""
|
||||
paragraphs = soup.find_all("p")
|
||||
intro_paragraphs: list[str] = []
|
||||
in_intro = False
|
||||
found_date = False
|
||||
verse_accumulator: list[str] = []
|
||||
|
||||
def flush_verse():
|
||||
"""Join accumulated Sanskrit verse pādas into one formatted block."""
|
||||
if verse_accumulator:
|
||||
joined = "<br>".join(verse_accumulator)
|
||||
intro_paragraphs.append(
|
||||
f'<div style="text-align:center"><i>{joined}</i></div>'
|
||||
)
|
||||
verse_accumulator.clear()
|
||||
|
||||
for p in paragraphs:
|
||||
cls = get_css_class(p)
|
||||
text = p.get_text().strip()
|
||||
|
||||
# Start marker
|
||||
if not in_intro:
|
||||
if "BOOK-Introduction" in text or "BOOK\u2011Introduction" in text:
|
||||
in_intro = True
|
||||
continue
|
||||
# Also match the ScaGoudy-encoded version
|
||||
if "BOOK" in text and "Introduction" in text and "K" in text:
|
||||
decoded = decode_text(p)
|
||||
if "KṚṢṆA" in decoded and "Introduction" in decoded:
|
||||
in_intro = True
|
||||
continue
|
||||
continue
|
||||
|
||||
# Skip the "Preface" subheader
|
||||
if text == "Preface" or text.startswith("*****"):
|
||||
continue
|
||||
|
||||
# End marker — "February 26th, 1970" (first occurrence)
|
||||
if "February 26th, 1970" in text:
|
||||
flush_verse()
|
||||
found_date = True
|
||||
break
|
||||
|
||||
# Skip empty paragraphs
|
||||
if not text or text == "\xa0":
|
||||
continue
|
||||
|
||||
# Sanskrit verse pādas (p4, p5, p10) — accumulate into a single block
|
||||
# Use decode_html_content to preserve <br> tags within multi-line verse elements
|
||||
if cls in VERSE_PADA_CLASSES:
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
verse_accumulator.append(html)
|
||||
continue
|
||||
|
||||
# If we were accumulating verse lines and hit a non-verse paragraph,
|
||||
# flush the verse block first
|
||||
flush_verse()
|
||||
|
||||
# Collect the paragraph
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
intro_paragraphs.append(html)
|
||||
|
||||
if not found_date:
|
||||
print(" WARNING: Did not find the end marker 'February 26th, 1970'")
|
||||
|
||||
return intro_paragraphs
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Extract chapter listing from p13 bookmarks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def extract_chapters(soup: BeautifulSoup) -> list[dict]:
|
||||
"""Extract the full chapter listing from p13 bookmark tags.
|
||||
|
||||
Format: 'KB 1970-V-C / Title' where V=volume, C=chapter within volume.
|
||||
We produce a flat chapter numbering (1–89) across both volumes.
|
||||
|
||||
Also finds chapter end markers to count paragraphs per chapter
|
||||
(useful for later extraction).
|
||||
"""
|
||||
chapters: list[dict] = []
|
||||
chapter_num = 0
|
||||
|
||||
# Also extract chapter end markers for titles
|
||||
end_titles: dict[int, str] = {}
|
||||
for p in soup.find_all("p"):
|
||||
cls = get_css_class(p)
|
||||
text = decode_text(p)
|
||||
# "Thus ends the Bhaktivedanta purport of the First Chapter of Kṛṣṇa, "Title""
|
||||
if "Thus ends" in text and "Chapter of" in text and "Kṛṣṇa" in text:
|
||||
# Extract chapter ordinal and title
|
||||
m = re.search(
|
||||
r"(?:the\s+)?(\w+)\s+Chapter\s+of\s+Kṛṣṇa,\s*[\"\"\"]*(.+?)[\"\"\"]*\.?\s*$",
|
||||
text,
|
||||
)
|
||||
if m:
|
||||
# We don't need the ordinal — just the title for cross-referencing
|
||||
title_from_end = m.group(2).strip().rstrip(".")
|
||||
# Store by the chapter count we've seen so far (rough approximation)
|
||||
|
||||
# Extract from p13 bookmarks
|
||||
for p in soup.find_all("p"):
|
||||
cls = get_css_class(p)
|
||||
if cls != "p13":
|
||||
continue
|
||||
|
||||
text = decode_text(p)
|
||||
|
||||
# Match: KB 1970-V-C / Title
|
||||
m = re.match(r"KB\s+1970-(\d+)-(\d+)\s*/\s*(.+)", text)
|
||||
if not m:
|
||||
continue
|
||||
|
||||
volume = int(m.group(1))
|
||||
chapter_in_vol = int(m.group(2))
|
||||
title = m.group(3).strip()
|
||||
|
||||
chapter_num += 1
|
||||
|
||||
chapters.append(
|
||||
{
|
||||
"number": chapter_num,
|
||||
"volume": volume,
|
||||
"chapterInVolume": chapter_in_vol,
|
||||
"title": title,
|
||||
}
|
||||
)
|
||||
|
||||
return chapters
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "other.html")
|
||||
output_path = os.path.join(base_dir, "lib", "jsons", "books", "kb", "intro.json")
|
||||
|
||||
if not os.path.exists(html_path):
|
||||
print(f"ERROR: {html_path} not found!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Reading {html_path}...")
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
soup = BeautifulSoup(f, "html.parser")
|
||||
|
||||
# Extract introduction
|
||||
print("Extracting introduction...")
|
||||
intro_paragraphs = extract_introduction(soup)
|
||||
print(f" Found {len(intro_paragraphs)} introduction paragraphs")
|
||||
|
||||
# Extract chapter listing
|
||||
print("Extracting chapter listing...")
|
||||
chapters = extract_chapters(soup)
|
||||
print(f" Found {len(chapters)} chapters")
|
||||
|
||||
# Count per volume
|
||||
vol1 = [c for c in chapters if c["volume"] == 1]
|
||||
vol2 = [c for c in chapters if c["volume"] == 2]
|
||||
print(f" Volume 1: {len(vol1)} chapters")
|
||||
print(f" Volume 2: {len(vol2)} chapters")
|
||||
|
||||
# Build intro.json
|
||||
# Strip volume/chapterInVolume from the chapter entries for the public JSON
|
||||
chapters_public = [{"number": c["number"], "title": c["title"]} for c in chapters]
|
||||
|
||||
intro = {
|
||||
"book": "kb",
|
||||
"title": "Kṛṣṇa, The Supreme Personality of Godhead",
|
||||
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
||||
"description": "A summary study of the Tenth Canto of Śrīmad-Bhāgavatam. Originally published in two volumes in 1970.",
|
||||
"totalChapters": len(chapters),
|
||||
"volumes": [
|
||||
{"number": 1, "totalChapters": len(vol1)},
|
||||
{"number": 2, "totalChapters": len(vol2)},
|
||||
],
|
||||
"chapters": chapters_public,
|
||||
"introduction": intro_paragraphs,
|
||||
}
|
||||
|
||||
# Write
|
||||
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
||||
with open(output_path, "w", encoding="utf-8") as f:
|
||||
json.dump(intro, f, ensure_ascii=False, indent=2)
|
||||
print(f"\nWritten: {output_path}")
|
||||
|
||||
# Summary
|
||||
print(f"\n--- Summary ---")
|
||||
print(f"Title: {intro['title']}")
|
||||
print(f"Chapters: {intro['totalChapters']}")
|
||||
print(f"Introduction paragraphs: {len(intro_paragraphs)}")
|
||||
|
||||
# Show first few intro paragraphs
|
||||
print(f"\n--- Introduction (first 3 paragraphs) ---")
|
||||
for i, para in enumerate(intro_paragraphs[:3]):
|
||||
print(f" [{i + 1}] {para[:120]}...")
|
||||
|
||||
# Show first and last chapters
|
||||
print(f"\n--- Chapters ---")
|
||||
for c in chapters[:5]:
|
||||
print(f" Ch {c['number']:>2} (Vol {c['volume']}): {c['title']}")
|
||||
print(f" ...")
|
||||
for c in chapters[-3:]:
|
||||
print(f" Ch {c['number']:>2} (Vol {c['volume']}): {c['title']}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,362 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract Kṛṣṇa Book Volume 1 chapters from other.html.
|
||||
|
||||
Volume 1 has 54 chapters (KB 1970-1-1 through KB 1970-1-54).
|
||||
Each chapter is a prose chapter (no verses) — just paragraphs of purport text.
|
||||
|
||||
The chapters are delimited by p13 bookmark tags: "KB 1970-1-N / Title"
|
||||
Each chapter starts with a p27 heading: "N /Title"
|
||||
Body paragraphs are in p6 (primary) and p28 classes.
|
||||
Chapter end markers: "Thus ends the Bhaktivedanta purport of the Nth Chapter..."
|
||||
|
||||
Output schema (one JSON per chapter):
|
||||
{
|
||||
"book": "kb",
|
||||
"volume": 1,
|
||||
"chapter": 1,
|
||||
"title": "Advent of Lord Kṛṣṇa",
|
||||
"type": "chapter",
|
||||
"paragraphs": ["<p>...</p>", ...]
|
||||
}
|
||||
|
||||
Output: lib/jsons/books/kb/vol-1/chapters/01.json through 54.json
|
||||
|
||||
Usage:
|
||||
python3 scripts/kb_extract_vol1_chapters.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ScaGoudy → IAST Unicode mapping
|
||||
# ---------------------------------------------------------------------------
|
||||
SCAGOUDY_TO_IAST = {
|
||||
"ä": "ā",
|
||||
"Ä": "Ā",
|
||||
"å": "ṛ",
|
||||
"Å": "Ṛ",
|
||||
"ñ": "ṣ",
|
||||
"Ñ": "Ṣ",
|
||||
"ö": "ṭ",
|
||||
"Ö": "Ṭ",
|
||||
"ë": "ṇ",
|
||||
"Ë": "Ṇ",
|
||||
"ò": "ḍ",
|
||||
"Ò": "Ḍ",
|
||||
"ç": "ś",
|
||||
"Ç": "Ś",
|
||||
"ì": "ṅ",
|
||||
"ï": "ñ",
|
||||
"ü": "ū",
|
||||
"Ü": "Ū",
|
||||
"é": "ī",
|
||||
"É": "Ī",
|
||||
"à": "ṁ",
|
||||
"ù": "ḥ",
|
||||
"è": "ṝ",
|
||||
"û": "ṁl",
|
||||
"ÿ": "ḷ",
|
||||
}
|
||||
|
||||
|
||||
def decode_scagoudy(text: str) -> str:
|
||||
for src, dst in SCAGOUDY_TO_IAST.items():
|
||||
text = text.replace(src, dst)
|
||||
return text
|
||||
|
||||
|
||||
def decode_html_content(tag: Tag) -> str:
|
||||
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
|
||||
inner = "".join(str(child) for child in tag.children)
|
||||
inner = decode_scagoudy(inner)
|
||||
# Remove span tags with CSS classes (RTF conversion artifacts)
|
||||
inner = re.sub(r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner)
|
||||
inner = re.sub(r"\s+", " ", inner).strip()
|
||||
return inner
|
||||
|
||||
|
||||
def decode_text(tag: Tag) -> str:
|
||||
return decode_scagoudy(tag.get_text().strip())
|
||||
|
||||
|
||||
def get_css_class(tag: Tag) -> str:
|
||||
classes = tag.get("class", [])
|
||||
return classes[0] if classes else ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CSS classes for Sanskrit verse pādas (center-aligned italic blocks
|
||||
# that sometimes appear within chapter prose — e.g., quoted verses)
|
||||
# ---------------------------------------------------------------------------
|
||||
VERSE_PADA_CLASSES = frozenset(["p4", "p5", "p10", "p19", "p24"])
|
||||
|
||||
# Primary body paragraph classes
|
||||
BODY_CLASSES = frozenset(["p6", "p28", "p23"])
|
||||
|
||||
# Chapter heading class (bold title like "1 /Advent of Lord Kṛṣṇa")
|
||||
HEADING_CLASS = "p27"
|
||||
|
||||
# Bookmark class
|
||||
BOOKMARK_CLASS = "p13"
|
||||
|
||||
# Volume header class
|
||||
VOLUME_HEADER_CLASS = "p12"
|
||||
|
||||
# Classes to skip entirely
|
||||
SKIP_CLASSES = frozenset(
|
||||
[
|
||||
"p9", # empty / decorative breaks
|
||||
"p11", # empty breaks
|
||||
"p15", # empty breaks
|
||||
"p17", # empty breaks
|
||||
"p20", # empty breaks
|
||||
"p14", # section labels like "Flap:"
|
||||
"p16", # "Not for sale in India"
|
||||
"p25", # handwritten notes marker
|
||||
"p26", # Apple Corps header
|
||||
"p29", # image descriptions (color plate references)
|
||||
"p30", # image descriptions
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def is_empty(tag: Tag) -> bool:
|
||||
text = tag.get_text().strip()
|
||||
return not text or text == "\xa0"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def extract_vol1_chapters(soup: BeautifulSoup):
|
||||
"""Extract all 54 chapters of Volume 1.
|
||||
|
||||
Returns a list of chapter dicts.
|
||||
"""
|
||||
all_paragraphs = list(soup.find_all("p"))
|
||||
|
||||
# Step 1: Find all Vol 1 chapter bookmark positions
|
||||
# Format: "KB 1970-1-N / Title"
|
||||
chapter_bookmarks = [] # (paragraph_index, chapter_num, title)
|
||||
|
||||
for i, p in enumerate(all_paragraphs):
|
||||
cls = get_css_class(p)
|
||||
if cls != BOOKMARK_CLASS:
|
||||
continue
|
||||
text = decode_text(p)
|
||||
m = re.match(r"KB\s+1970-1-(\d+)\s*/\s*(.+)", text)
|
||||
if m:
|
||||
ch_num = int(m.group(1))
|
||||
title = m.group(2).strip()
|
||||
chapter_bookmarks.append((i, ch_num, title))
|
||||
|
||||
if not chapter_bookmarks:
|
||||
print("ERROR: No Vol 1 chapter bookmarks found!")
|
||||
return []
|
||||
|
||||
print(f" Found {len(chapter_bookmarks)} Vol 1 chapter bookmarks")
|
||||
|
||||
# Step 2: Find the end boundary (Volume 2 header or end of file)
|
||||
vol2_start = len(all_paragraphs)
|
||||
for i, p in enumerate(all_paragraphs):
|
||||
cls = get_css_class(p)
|
||||
if cls == VOLUME_HEADER_CLASS:
|
||||
text = decode_text(p)
|
||||
if "Volume 2" in text:
|
||||
vol2_start = i
|
||||
break
|
||||
|
||||
# Step 3: Extract each chapter
|
||||
chapters = []
|
||||
|
||||
for idx, (bookmark_pos, ch_num, title) in enumerate(chapter_bookmarks):
|
||||
# Chapter content starts after the bookmark
|
||||
content_start = bookmark_pos + 1
|
||||
|
||||
# Chapter content ends at the next chapter's bookmark (or Vol 2 boundary)
|
||||
if idx + 1 < len(chapter_bookmarks):
|
||||
content_end = chapter_bookmarks[idx + 1][0]
|
||||
else:
|
||||
content_end = vol2_start
|
||||
|
||||
# Extract paragraphs for this chapter
|
||||
paragraphs = extract_chapter_paragraphs(
|
||||
all_paragraphs, content_start, content_end
|
||||
)
|
||||
|
||||
chapters.append(
|
||||
{
|
||||
"book": "kb",
|
||||
"volume": 1,
|
||||
"chapter": ch_num,
|
||||
"title": title,
|
||||
"type": "chapter",
|
||||
"paragraphs": paragraphs,
|
||||
}
|
||||
)
|
||||
|
||||
return chapters
|
||||
|
||||
|
||||
def extract_chapter_paragraphs(
|
||||
all_paragraphs: list,
|
||||
start_idx: int,
|
||||
end_idx: int,
|
||||
) -> list[str]:
|
||||
"""Extract the prose paragraphs for a single chapter.
|
||||
|
||||
Handles:
|
||||
- Regular body paragraphs (p6, p28)
|
||||
- Quoted Sanskrit verse blocks (p4, p5, p19, p24) → grouped into centered italic
|
||||
- Chapter heading (p27) → included as first paragraph
|
||||
- Chapter end marker ("Thus ends...") → included as last paragraph (italic)
|
||||
- Skips empty/decorative paragraphs
|
||||
"""
|
||||
result: list[str] = []
|
||||
verse_accumulator: list[str] = []
|
||||
|
||||
def flush_verse():
|
||||
if verse_accumulator:
|
||||
joined = "<br>".join(verse_accumulator)
|
||||
result.append(f'<div style="text-align:center"><i>{joined}</i></div>')
|
||||
verse_accumulator.clear()
|
||||
|
||||
for i in range(start_idx, end_idx):
|
||||
p = all_paragraphs[i]
|
||||
cls = get_css_class(p)
|
||||
|
||||
# Skip empty paragraphs
|
||||
if is_empty(p):
|
||||
continue
|
||||
|
||||
# Skip noise classes
|
||||
if cls in SKIP_CLASSES:
|
||||
continue
|
||||
|
||||
# Skip bookmark tags that might be inside the range
|
||||
if cls == BOOKMARK_CLASS:
|
||||
continue
|
||||
|
||||
# Skip volume headers
|
||||
if cls == VOLUME_HEADER_CLASS:
|
||||
continue
|
||||
|
||||
# Sanskrit verse pādas — accumulate into centered block
|
||||
# Use decode_html_content to preserve <br> tags within multi-line verse elements
|
||||
if cls in VERSE_PADA_CLASSES:
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
verse_accumulator.append(html)
|
||||
continue
|
||||
|
||||
# Non-verse paragraph — flush any accumulated verse first
|
||||
flush_verse()
|
||||
|
||||
# Chapter heading (p27) — keep it, it's the chapter title
|
||||
if cls == HEADING_CLASS:
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
result.append(html)
|
||||
continue
|
||||
|
||||
# Body paragraphs and everything else
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
result.append(html)
|
||||
|
||||
# Flush any trailing verse
|
||||
flush_verse()
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Output
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def write_json(data, filepath):
|
||||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||||
with open(filepath, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "other.html")
|
||||
output_dir = os.path.join(
|
||||
base_dir, "lib", "jsons", "books", "kb", "vol-1", "chapters"
|
||||
)
|
||||
|
||||
if not os.path.exists(html_path):
|
||||
print(f"ERROR: {html_path} not found!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Reading {html_path}...")
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
soup = BeautifulSoup(f, "html.parser")
|
||||
|
||||
print("Extracting Volume 1 chapters...")
|
||||
chapters = extract_vol1_chapters(soup)
|
||||
|
||||
if not chapters:
|
||||
print("ERROR: No chapters extracted!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"\nExtracted {len(chapters)} chapters\n")
|
||||
|
||||
# Write each chapter
|
||||
for ch in chapters:
|
||||
ch_num = str(ch["chapter"]).zfill(2)
|
||||
filepath = os.path.join(output_dir, f"{ch_num}.json")
|
||||
write_json(ch, filepath)
|
||||
print(
|
||||
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — {ch['title']}"
|
||||
)
|
||||
|
||||
# Summary
|
||||
total_paragraphs = sum(len(ch["paragraphs"]) for ch in chapters)
|
||||
print(f"\n--- Summary ---")
|
||||
print(f"Total chapters: {len(chapters)}")
|
||||
print(f"Total paragraphs: {total_paragraphs}")
|
||||
print(f"Average paragraphs per chapter: {total_paragraphs / len(chapters):.1f}")
|
||||
print(f"Output: {output_dir}/")
|
||||
|
||||
# Show samples from first and last chapter
|
||||
print(f"\n--- Sample: Chapter 1 ---")
|
||||
ch1 = chapters[0]
|
||||
for i, p in enumerate(ch1["paragraphs"][:3]):
|
||||
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
||||
print(f" [{i}] {preview}...")
|
||||
|
||||
print(f"\n--- Sample: Chapter {chapters[-1]['chapter']} ---")
|
||||
last = chapters[-1]
|
||||
for i, p in enumerate(last["paragraphs"][:3]):
|
||||
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
||||
print(f" [{i}] {preview}...")
|
||||
|
||||
# Quality check: verify all chapters have the end marker
|
||||
print(f"\n--- Quality Check ---")
|
||||
missing_end = 0
|
||||
for ch in chapters:
|
||||
has_end = any("Thus end" in p for p in ch["paragraphs"])
|
||||
if not has_end:
|
||||
missing_end += 1
|
||||
print(
|
||||
f" ⚠ Chapter {ch['chapter']} ({ch['title']}) — no 'Thus ends' marker"
|
||||
)
|
||||
if missing_end == 0:
|
||||
print(f" ✅ All {len(chapters)} chapters have 'Thus ends' markers")
|
||||
else:
|
||||
print(f" ⚠ {missing_end} chapters missing 'Thus ends' markers")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,313 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract Kṛṣṇa Book Volume 1 introductory segments from other.html.
|
||||
|
||||
The Volume 1 front matter (lines ~772–901) contains 5 segments before Chapter 1:
|
||||
1. Flap intro — book jacket text
|
||||
2. Dedication — "To My Father, Gour Mohon De (1849-1930)"
|
||||
3. Words from Apple — George Harrison's introduction
|
||||
4. Preface — Prabhupāda's preface
|
||||
5. Introduction — Prabhupāda's introduction
|
||||
|
||||
Each segment is extracted into a JSON file with a common schema:
|
||||
{
|
||||
"slug": "flap-intro",
|
||||
"title": "Flap Introduction",
|
||||
"type": "intro-segment",
|
||||
"book": "kb",
|
||||
"volume": 1,
|
||||
"order": 1,
|
||||
"paragraphs": ["<p>...</p>", ...]
|
||||
}
|
||||
|
||||
Output: lib/jsons/books/kb/vol-1/intro/*.json
|
||||
|
||||
Usage:
|
||||
python3 scripts/kb_extract_vol1_intro.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ScaGoudy → IAST Unicode mapping
|
||||
# ---------------------------------------------------------------------------
|
||||
SCAGOUDY_TO_IAST = {
|
||||
"ä": "ā",
|
||||
"Ä": "Ā",
|
||||
"å": "ṛ",
|
||||
"Å": "Ṛ",
|
||||
"ñ": "ṣ",
|
||||
"Ñ": "Ṣ",
|
||||
"ö": "ṭ",
|
||||
"Ö": "Ṭ",
|
||||
"ë": "ṇ",
|
||||
"Ë": "Ṇ",
|
||||
"ò": "ḍ",
|
||||
"Ò": "Ḍ",
|
||||
"ç": "ś",
|
||||
"Ç": "Ś",
|
||||
"ì": "ṅ",
|
||||
"ï": "ñ",
|
||||
"ü": "ū",
|
||||
"Ü": "Ū",
|
||||
"é": "ī",
|
||||
"É": "Ī",
|
||||
"à": "ṁ",
|
||||
"ù": "ḥ",
|
||||
"è": "ṝ",
|
||||
"û": "ṁl",
|
||||
"ÿ": "ḷ",
|
||||
}
|
||||
|
||||
|
||||
def decode_scagoudy(text: str) -> str:
|
||||
for src, dst in SCAGOUDY_TO_IAST.items():
|
||||
text = text.replace(src, dst)
|
||||
return text
|
||||
|
||||
|
||||
def decode_html_content(tag: Tag) -> str:
|
||||
inner = "".join(str(child) for child in tag.children)
|
||||
inner = decode_scagoudy(inner)
|
||||
inner = re.sub(r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner)
|
||||
inner = re.sub(r"\s+", " ", inner).strip()
|
||||
return inner
|
||||
|
||||
|
||||
def decode_text(tag: Tag) -> str:
|
||||
return decode_scagoudy(tag.get_text().strip())
|
||||
|
||||
|
||||
def get_css_class(tag: Tag) -> str:
|
||||
classes = tag.get("class", [])
|
||||
return classes[0] if classes else ""
|
||||
|
||||
|
||||
# CSS classes for Sanskrit verse pādas (center-aligned italic)
|
||||
VERSE_PADA_CLASSES = frozenset(["p4", "p5", "p10", "p19", "p24"])
|
||||
|
||||
|
||||
def is_empty(tag: Tag) -> bool:
|
||||
"""Check if a tag is empty / whitespace only."""
|
||||
text = tag.get_text().strip()
|
||||
return not text or text == "\xa0"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Segment definitions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
SEGMENTS = [
|
||||
{
|
||||
"slug": "flap-intro",
|
||||
"title": "Flap Introduction",
|
||||
"order": 1,
|
||||
"bookmark": "Kb 1970 /Flap intro",
|
||||
},
|
||||
{
|
||||
"slug": "dedication",
|
||||
"title": "Dedication — To My Father, Gour Mohon De (1849–1930)",
|
||||
"order": 2,
|
||||
"bookmark": "Kb 1970 / To My Father, Gour Mohon De (1849-1930)",
|
||||
},
|
||||
{
|
||||
"slug": "words-from-apple",
|
||||
"title": "Words from Apple — by George Harrison",
|
||||
"order": 3,
|
||||
"bookmark": "Kb 1970 / Words from Apple",
|
||||
},
|
||||
{
|
||||
"slug": "preface",
|
||||
"title": "Preface",
|
||||
"order": 4,
|
||||
"bookmark": "Kb 1970 / Preface",
|
||||
},
|
||||
{
|
||||
"slug": "introduction",
|
||||
"title": "Introduction",
|
||||
"order": 5,
|
||||
"bookmark": "Kb 1970 / Introduction",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def find_segment_ranges(soup: BeautifulSoup):
|
||||
"""Find the start/end paragraph indices for each segment.
|
||||
|
||||
Segments are delimited by p13 bookmark tags. The first chapter marker
|
||||
(KB 1970-1-1) ends the last intro segment.
|
||||
"""
|
||||
all_paragraphs = list(soup.find_all("p"))
|
||||
|
||||
# Find all p13 bookmark indices within the KB volume 1 area
|
||||
bookmark_positions = [] # (index, text)
|
||||
for i, p in enumerate(all_paragraphs):
|
||||
cls = get_css_class(p)
|
||||
if cls == "p13":
|
||||
text = decode_text(p)
|
||||
bookmark_positions.append((i, text))
|
||||
|
||||
# Map each segment to its start/end paragraph indices
|
||||
ranges = {}
|
||||
for seg in SEGMENTS:
|
||||
# Find the bookmark that starts this segment
|
||||
start_idx = None
|
||||
for i, (pos, text) in enumerate(bookmark_positions):
|
||||
if seg["bookmark"] in text:
|
||||
# Content starts AFTER the bookmark tag
|
||||
start_idx = pos + 1
|
||||
# End is the next bookmark
|
||||
if i + 1 < len(bookmark_positions):
|
||||
end_idx = bookmark_positions[i + 1][0]
|
||||
else:
|
||||
end_idx = len(all_paragraphs)
|
||||
ranges[seg["slug"]] = (start_idx, end_idx, all_paragraphs)
|
||||
break
|
||||
|
||||
if start_idx is None:
|
||||
print(
|
||||
f" WARNING: Could not find bookmark for '{seg['slug']}': '{seg['bookmark']}'"
|
||||
)
|
||||
|
||||
return ranges
|
||||
|
||||
|
||||
def extract_segment_paragraphs(
|
||||
all_paragraphs: list,
|
||||
start_idx: int,
|
||||
end_idx: int,
|
||||
) -> list[str]:
|
||||
"""Extract paragraphs for a single segment, handling verse grouping."""
|
||||
result: list[str] = []
|
||||
verse_accumulator: list[str] = []
|
||||
|
||||
def flush_verse():
|
||||
if verse_accumulator:
|
||||
joined = "<br>".join(verse_accumulator)
|
||||
result.append(f'<div style="text-align:center"><i>{joined}</i></div>')
|
||||
verse_accumulator.clear()
|
||||
|
||||
for i in range(start_idx, end_idx):
|
||||
p = all_paragraphs[i]
|
||||
cls = get_css_class(p)
|
||||
text = p.get_text().strip()
|
||||
|
||||
# Skip the segment's own heading (p12, p14, p22, p27 bold headers)
|
||||
# These are the display titles like "Flap:", "Words From Apple", "Preface", etc.
|
||||
# We keep them as the first paragraph if they have meaningful content
|
||||
if cls in ("p12", "p14"):
|
||||
# Volume header or section label — skip
|
||||
continue
|
||||
|
||||
# Skip empty paragraphs
|
||||
if is_empty(p):
|
||||
continue
|
||||
|
||||
# Skip bookmark tags (p13) that might be nested
|
||||
if cls == "p13":
|
||||
continue
|
||||
|
||||
# Sanskrit verse pādas — accumulate
|
||||
# Use decode_html_content to preserve <br> tags within multi-line verse elements
|
||||
if cls in VERSE_PADA_CLASSES:
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
verse_accumulator.append(html)
|
||||
continue
|
||||
|
||||
# Non-verse paragraph — flush any accumulated verse first
|
||||
flush_verse()
|
||||
|
||||
# Extract the HTML content
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
result.append(html)
|
||||
|
||||
# Flush any trailing verse
|
||||
flush_verse()
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Output
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def write_json(data, filepath):
|
||||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||||
with open(filepath, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
print(f" Written: {filepath}")
|
||||
|
||||
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "other.html")
|
||||
output_dir = os.path.join(base_dir, "lib", "jsons", "books", "kb", "vol-1", "intro")
|
||||
|
||||
if not os.path.exists(html_path):
|
||||
print(f"ERROR: {html_path} not found!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Reading {html_path}...")
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
soup = BeautifulSoup(f, "html.parser")
|
||||
|
||||
print("Finding segment boundaries...")
|
||||
ranges = find_segment_ranges(soup)
|
||||
|
||||
print(f"Extracting {len(SEGMENTS)} introductory segments...\n")
|
||||
|
||||
for seg in SEGMENTS:
|
||||
slug = seg["slug"]
|
||||
if slug not in ranges:
|
||||
print(f" ⏭ Skipping '{slug}' — bookmark not found")
|
||||
continue
|
||||
|
||||
start_idx, end_idx, all_paragraphs = ranges[slug]
|
||||
paragraphs = extract_segment_paragraphs(all_paragraphs, start_idx, end_idx)
|
||||
|
||||
doc = {
|
||||
"slug": slug,
|
||||
"title": seg["title"],
|
||||
"type": "intro-segment",
|
||||
"book": "kb",
|
||||
"volume": 1,
|
||||
"order": seg["order"],
|
||||
"paragraphs": paragraphs,
|
||||
}
|
||||
|
||||
filepath = os.path.join(output_dir, f"{slug}.json")
|
||||
write_json(doc, filepath)
|
||||
print(f" {seg['title']}: {len(paragraphs)} paragraphs")
|
||||
|
||||
# Summary
|
||||
print(f"\n--- Summary ---")
|
||||
for seg in SEGMENTS:
|
||||
slug = seg["slug"]
|
||||
filepath = os.path.join(output_dir, f"{slug}.json")
|
||||
if os.path.exists(filepath):
|
||||
with open(filepath, "r", encoding="utf-8") as f:
|
||||
doc = json.load(f)
|
||||
# Show first paragraph preview
|
||||
first = doc["paragraphs"][0] if doc["paragraphs"] else "(empty)"
|
||||
# Strip HTML tags for preview
|
||||
preview = re.sub(r"<[^>]+>", "", first)[:80]
|
||||
print(f" {seg['order']}. {seg['title']}")
|
||||
print(f' {len(doc["paragraphs"])} paragraphs — "{preview}..."')
|
||||
else:
|
||||
print(f" {seg['order']}. {seg['title']} — NOT EXTRACTED")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,367 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract Kṛṣṇa Book Volume 2 chapters from other.html.
|
||||
|
||||
Volume 2 has 35 chapters (KB 1970-2-1 through KB 1970-2-35).
|
||||
Each chapter is a prose chapter (no verses) — just paragraphs of purport text.
|
||||
|
||||
The chapters are delimited by p13 bookmark tags: "KB 1970-2-N / Title"
|
||||
Each chapter starts with a p27 heading: "N /Title"
|
||||
Body paragraphs are in p6 (primary) and p28 classes.
|
||||
Chapter end markers: "Thus ends the Bhaktivedanta purport of the Nth Chapter..."
|
||||
|
||||
Output schema (one JSON per chapter):
|
||||
{
|
||||
"book": "kb",
|
||||
"volume": 2,
|
||||
"chapter": 1,
|
||||
"title": "The Story of the Syamantaka Jewel",
|
||||
"type": "chapter",
|
||||
"paragraphs": ["<p>...</p>", ...]
|
||||
}
|
||||
|
||||
Output: lib/jsons/books/kb/vol-2/chapters/01.json through 35.json
|
||||
|
||||
Usage:
|
||||
python3 scripts/kb_extract_vol2_chapters.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ScaGoudy → IAST Unicode mapping
|
||||
# ---------------------------------------------------------------------------
|
||||
SCAGOUDY_TO_IAST = {
|
||||
"ä": "ā",
|
||||
"Ä": "Ā",
|
||||
"å": "ṛ",
|
||||
"Å": "Ṛ",
|
||||
"ñ": "ṣ",
|
||||
"Ñ": "Ṣ",
|
||||
"ö": "ṭ",
|
||||
"Ö": "Ṭ",
|
||||
"ë": "ṇ",
|
||||
"Ë": "Ṇ",
|
||||
"ò": "ḍ",
|
||||
"Ò": "Ḍ",
|
||||
"ç": "ś",
|
||||
"Ç": "Ś",
|
||||
"ì": "ṅ",
|
||||
"ï": "ñ",
|
||||
"ü": "ū",
|
||||
"Ü": "Ū",
|
||||
"é": "ī",
|
||||
"É": "Ī",
|
||||
"à": "ṁ",
|
||||
"ù": "ḥ",
|
||||
"è": "ṝ",
|
||||
"û": "ṁl",
|
||||
"ÿ": "ḷ",
|
||||
}
|
||||
|
||||
|
||||
def decode_scagoudy(text: str) -> str:
|
||||
for src, dst in SCAGOUDY_TO_IAST.items():
|
||||
text = text.replace(src, dst)
|
||||
return text
|
||||
|
||||
|
||||
def decode_html_content(tag: Tag) -> str:
|
||||
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
|
||||
inner = "".join(str(child) for child in tag.children)
|
||||
inner = decode_scagoudy(inner)
|
||||
# Remove span tags with CSS classes (RTF conversion artifacts)
|
||||
inner = re.sub(r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner)
|
||||
inner = re.sub(r"\s+", " ", inner).strip()
|
||||
return inner
|
||||
|
||||
|
||||
def decode_text(tag: Tag) -> str:
|
||||
return decode_scagoudy(tag.get_text().strip())
|
||||
|
||||
|
||||
def get_css_class(tag: Tag) -> str:
|
||||
classes = tag.get("class", [])
|
||||
return classes[0] if classes else ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CSS classes for Sanskrit verse pādas (center-aligned italic blocks
|
||||
# that sometimes appear within chapter prose — e.g., quoted verses)
|
||||
# ---------------------------------------------------------------------------
|
||||
VERSE_PADA_CLASSES = frozenset(["p4", "p5", "p10", "p19", "p24"])
|
||||
|
||||
# Primary body paragraph classes
|
||||
BODY_CLASSES = frozenset(["p6", "p28", "p23"])
|
||||
|
||||
# Chapter heading class (bold title like "1 /The Story of the Syamantaka Jewel")
|
||||
HEADING_CLASS = "p27"
|
||||
|
||||
# Bookmark class
|
||||
BOOKMARK_CLASS = "p13"
|
||||
|
||||
# Volume header class
|
||||
VOLUME_HEADER_CLASS = "p12"
|
||||
|
||||
# Classes to skip entirely
|
||||
SKIP_CLASSES = frozenset(
|
||||
[
|
||||
"p9", # empty / decorative breaks
|
||||
"p11", # empty breaks
|
||||
"p15", # empty breaks
|
||||
"p17", # empty breaks
|
||||
"p20", # empty breaks
|
||||
"p14", # section labels
|
||||
"p16", # "Not for sale in India"
|
||||
"p25", # handwritten notes marker
|
||||
"p26", # Apple Corps header
|
||||
"p29", # image descriptions (color plate references)
|
||||
"p30", # image descriptions
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def is_empty(tag: Tag) -> bool:
|
||||
text = tag.get_text().strip()
|
||||
return not text or text == "\xa0"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def extract_vol2_chapters(soup: BeautifulSoup):
|
||||
"""Extract all 35 chapters of Volume 2.
|
||||
|
||||
Returns a list of chapter dicts.
|
||||
"""
|
||||
all_paragraphs = list(soup.find_all("p"))
|
||||
|
||||
# Step 1: Find all Vol 2 chapter bookmark positions
|
||||
# Format: "KB 1970-2-N / Title"
|
||||
chapter_bookmarks = [] # (paragraph_index, chapter_num, title)
|
||||
|
||||
for i, p in enumerate(all_paragraphs):
|
||||
cls = get_css_class(p)
|
||||
if cls != BOOKMARK_CLASS:
|
||||
continue
|
||||
text = decode_text(p)
|
||||
m = re.match(r"KB\s+1970-2-(\d+)\s*/\s*(.+)", text)
|
||||
if m:
|
||||
ch_num = int(m.group(1))
|
||||
title = m.group(2).strip()
|
||||
chapter_bookmarks.append((i, ch_num, title))
|
||||
|
||||
if not chapter_bookmarks:
|
||||
print("ERROR: No Vol 2 chapter bookmarks found!")
|
||||
return []
|
||||
|
||||
print(f" Found {len(chapter_bookmarks)} Vol 2 chapter bookmarks")
|
||||
|
||||
# Step 2: Find the end boundary — the next book section or end of file
|
||||
# Look for the next p12 volume header that is NOT KB Vol 2
|
||||
vol2_end = len(all_paragraphs)
|
||||
# Start searching after the last Vol 2 chapter bookmark
|
||||
last_bookmark_pos = chapter_bookmarks[-1][0]
|
||||
for i in range(last_bookmark_pos + 1, len(all_paragraphs)):
|
||||
p = all_paragraphs[i]
|
||||
cls = get_css_class(p)
|
||||
if cls == VOLUME_HEADER_CLASS:
|
||||
text = decode_text(p)
|
||||
# If it's a different book header (not KB), that's our end
|
||||
if "Volume 2" not in text:
|
||||
vol2_end = i
|
||||
break
|
||||
|
||||
# Step 3: Extract each chapter
|
||||
chapters = []
|
||||
|
||||
for idx, (bookmark_pos, ch_num, title) in enumerate(chapter_bookmarks):
|
||||
# Chapter content starts after the bookmark
|
||||
content_start = bookmark_pos + 1
|
||||
|
||||
# Chapter content ends at the next chapter's bookmark (or end boundary)
|
||||
if idx + 1 < len(chapter_bookmarks):
|
||||
content_end = chapter_bookmarks[idx + 1][0]
|
||||
else:
|
||||
content_end = vol2_end
|
||||
|
||||
# Extract paragraphs for this chapter
|
||||
paragraphs = extract_chapter_paragraphs(
|
||||
all_paragraphs, content_start, content_end
|
||||
)
|
||||
|
||||
chapters.append(
|
||||
{
|
||||
"book": "kb",
|
||||
"volume": 2,
|
||||
"chapter": ch_num,
|
||||
"title": title,
|
||||
"type": "chapter",
|
||||
"paragraphs": paragraphs,
|
||||
}
|
||||
)
|
||||
|
||||
return chapters
|
||||
|
||||
|
||||
def extract_chapter_paragraphs(
|
||||
all_paragraphs: list,
|
||||
start_idx: int,
|
||||
end_idx: int,
|
||||
) -> list[str]:
|
||||
"""Extract the prose paragraphs for a single chapter.
|
||||
|
||||
Handles:
|
||||
- Regular body paragraphs (p6, p28)
|
||||
- Quoted Sanskrit verse blocks (p4, p5, p19, p24) → grouped into centered italic
|
||||
- Chapter heading (p27) → included as first paragraph
|
||||
- Chapter end marker ("Thus ends...") → included as last paragraph (italic)
|
||||
- Skips empty/decorative paragraphs
|
||||
"""
|
||||
result: list[str] = []
|
||||
verse_accumulator: list[str] = []
|
||||
|
||||
def flush_verse():
|
||||
if verse_accumulator:
|
||||
joined = "<br>".join(verse_accumulator)
|
||||
result.append(f'<div style="text-align:center"><i>{joined}</i></div>')
|
||||
verse_accumulator.clear()
|
||||
|
||||
for i in range(start_idx, end_idx):
|
||||
p = all_paragraphs[i]
|
||||
cls = get_css_class(p)
|
||||
|
||||
# Skip empty paragraphs
|
||||
if is_empty(p):
|
||||
continue
|
||||
|
||||
# Skip noise classes
|
||||
if cls in SKIP_CLASSES:
|
||||
continue
|
||||
|
||||
# Skip bookmark tags that might be inside the range
|
||||
if cls == BOOKMARK_CLASS:
|
||||
continue
|
||||
|
||||
# Skip volume headers
|
||||
if cls == VOLUME_HEADER_CLASS:
|
||||
continue
|
||||
|
||||
# Sanskrit verse pādas — accumulate into centered block
|
||||
# Use decode_html_content to preserve <br> tags within multi-line verse elements
|
||||
if cls in VERSE_PADA_CLASSES:
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
verse_accumulator.append(html)
|
||||
continue
|
||||
|
||||
# Non-verse paragraph — flush any accumulated verse first
|
||||
flush_verse()
|
||||
|
||||
# Chapter heading (p27) — keep it, it's the chapter title
|
||||
if cls == HEADING_CLASS:
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
result.append(html)
|
||||
continue
|
||||
|
||||
# Body paragraphs and everything else
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
result.append(html)
|
||||
|
||||
# Flush any trailing verse
|
||||
flush_verse()
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Output
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def write_json(data, filepath):
|
||||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||||
with open(filepath, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "other.html")
|
||||
output_dir = os.path.join(
|
||||
base_dir, "lib", "jsons", "books", "kb", "vol-2", "chapters"
|
||||
)
|
||||
|
||||
if not os.path.exists(html_path):
|
||||
print(f"ERROR: {html_path} not found!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Reading {html_path}...")
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
soup = BeautifulSoup(f, "html.parser")
|
||||
|
||||
print("Extracting Volume 2 chapters...")
|
||||
chapters = extract_vol2_chapters(soup)
|
||||
|
||||
if not chapters:
|
||||
print("ERROR: No chapters extracted!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"\nExtracted {len(chapters)} chapters\n")
|
||||
|
||||
# Write each chapter
|
||||
for ch in chapters:
|
||||
ch_num = str(ch["chapter"]).zfill(2)
|
||||
filepath = os.path.join(output_dir, f"{ch_num}.json")
|
||||
write_json(ch, filepath)
|
||||
print(
|
||||
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — {ch['title']}"
|
||||
)
|
||||
|
||||
# Summary
|
||||
total_paragraphs = sum(len(ch["paragraphs"]) for ch in chapters)
|
||||
print(f"\n--- Summary ---")
|
||||
print(f"Total chapters: {len(chapters)}")
|
||||
print(f"Total paragraphs: {total_paragraphs}")
|
||||
print(f"Average paragraphs per chapter: {total_paragraphs / len(chapters):.1f}")
|
||||
print(f"Output: {output_dir}/")
|
||||
|
||||
# Show samples from first and last chapter
|
||||
print(f"\n--- Sample: Chapter 1 ---")
|
||||
ch1 = chapters[0]
|
||||
for i, p in enumerate(ch1["paragraphs"][:3]):
|
||||
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
||||
print(f" [{i}] {preview}...")
|
||||
|
||||
print(f"\n--- Sample: Chapter {chapters[-1]['chapter']} ---")
|
||||
last = chapters[-1]
|
||||
for i, p in enumerate(last["paragraphs"][:3]):
|
||||
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
||||
print(f" [{i}] {preview}...")
|
||||
|
||||
# Quality check: verify all chapters have the end marker
|
||||
print(f"\n--- Quality Check ---")
|
||||
missing_end = 0
|
||||
for ch in chapters:
|
||||
has_end = any("Thus end" in p for p in ch["paragraphs"])
|
||||
if not has_end:
|
||||
missing_end += 1
|
||||
print(
|
||||
f" ⚠ Chapter {ch['chapter']} ({ch['title']}) — no 'Thus ends' marker"
|
||||
)
|
||||
if missing_end == 0:
|
||||
print(f" ✅ All {len(chapters)} chapters have 'Thus ends' markers")
|
||||
else:
|
||||
print(f" ⚠ {missing_end} chapters missing 'Thus ends' markers")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+191
-27
@@ -49,6 +49,26 @@ function readAllJsonInDir(dir: string): unknown[] {
|
||||
.map((f) => readJson(path.join(dir, f)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert documents one at a time with a small delay every N docs.
|
||||
*
|
||||
* Uses insertOne to avoid BSON wire-protocol corruption that occurs
|
||||
* with insertMany on certain MongoDB server/proxy configurations.
|
||||
* A brief pause every 100 docs prevents connection overload.
|
||||
*/
|
||||
async function insertBatched(
|
||||
col: ReturnType<ReturnType<MongoClient["db"]>["collection"]>,
|
||||
docs: Record<string, unknown>[],
|
||||
) {
|
||||
for (let i = 0; i < docs.length; i++) {
|
||||
await col.insertOne(docs[i]);
|
||||
// Pause briefly every 100 docs to give the connection breathing room
|
||||
if ((i + 1) % 100 === 0) {
|
||||
await new Promise((r) => setTimeout(r, 100));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -95,6 +115,17 @@ type CcVerse = {
|
||||
purport: string[];
|
||||
};
|
||||
|
||||
type ProseChapterJson = {
|
||||
book: string;
|
||||
volume: number;
|
||||
chapter?: number;
|
||||
title: string;
|
||||
type: string;
|
||||
slug?: string;
|
||||
order?: number;
|
||||
paragraphs: string[];
|
||||
};
|
||||
|
||||
type Sb1962Verse = {
|
||||
book_title: string;
|
||||
chapter: number;
|
||||
@@ -178,9 +209,7 @@ async function seedBg(db: ReturnType<MongoClient["db"]>) {
|
||||
}
|
||||
}
|
||||
|
||||
if (docs.length > 0) {
|
||||
await versesCol.insertMany(docs, { ordered: false });
|
||||
}
|
||||
await insertBatched(versesCol, docs);
|
||||
console.log(` ✅ BG: ${docs.length} verses`);
|
||||
}
|
||||
|
||||
@@ -227,14 +256,14 @@ async function seedSb(db: ReturnType<MongoClient["db"]>) {
|
||||
.filter((d) => fs.statSync(path.join(chaptersDir, d)).isDirectory())
|
||||
.sort();
|
||||
|
||||
const docs: Record<string, unknown>[] = [];
|
||||
const cantoDocs: Record<string, unknown>[] = [];
|
||||
|
||||
for (const chDir of chapterDirs) {
|
||||
const versesDir = path.join(chaptersDir, chDir, "verses");
|
||||
const verses = readAllJsonInDir(versesDir) as SbVerse[];
|
||||
|
||||
for (const v of verses) {
|
||||
docs.push({
|
||||
cantoDocs.push({
|
||||
book: "sb",
|
||||
division: String(v.canto),
|
||||
chapter: v.chapter,
|
||||
@@ -252,11 +281,9 @@ async function seedSb(db: ReturnType<MongoClient["db"]>) {
|
||||
}
|
||||
}
|
||||
|
||||
if (docs.length > 0) {
|
||||
await versesCol.insertMany(docs, { ordered: false });
|
||||
}
|
||||
totalCount += docs.length;
|
||||
console.log(` Canto ${cantoDir}: ${docs.length} verses`);
|
||||
await insertBatched(versesCol, cantoDocs);
|
||||
totalCount += cantoDocs.length;
|
||||
console.log(` Canto ${cantoDir}: ${cantoDocs.length} verses`);
|
||||
}
|
||||
|
||||
console.log(` ✅ SB: ${totalCount} verses`);
|
||||
@@ -305,14 +332,14 @@ async function seedCc(db: ReturnType<MongoClient["db"]>) {
|
||||
.filter((d) => fs.statSync(path.join(chaptersDir, d)).isDirectory())
|
||||
.sort();
|
||||
|
||||
const docs: Record<string, unknown>[] = [];
|
||||
const lilaDocs: Record<string, unknown>[] = [];
|
||||
|
||||
for (const chDir of chapterDirs) {
|
||||
const versesDir = path.join(chaptersDir, chDir, "verses");
|
||||
const verses = readAllJsonInDir(versesDir) as CcVerse[];
|
||||
|
||||
for (const v of verses) {
|
||||
docs.push({
|
||||
lilaDocs.push({
|
||||
book: "cc",
|
||||
division: v.lila,
|
||||
chapter: v.chapter,
|
||||
@@ -330,11 +357,9 @@ async function seedCc(db: ReturnType<MongoClient["db"]>) {
|
||||
}
|
||||
}
|
||||
|
||||
if (docs.length > 0) {
|
||||
await versesCol.insertMany(docs, { ordered: false });
|
||||
}
|
||||
totalCount += docs.length;
|
||||
console.log(` Līlā ${lilaDir}: ${docs.length} verses`);
|
||||
await insertBatched(versesCol, lilaDocs);
|
||||
totalCount += lilaDocs.length;
|
||||
console.log(` Līlā ${lilaDir}: ${lilaDocs.length} verses`);
|
||||
}
|
||||
|
||||
console.log(` ✅ CC: ${totalCount} verses`);
|
||||
@@ -397,12 +422,143 @@ async function seedSb1962(db: ReturnType<MongoClient["db"]>) {
|
||||
}
|
||||
}
|
||||
|
||||
if (docs.length > 0) {
|
||||
await versesCol.insertMany(docs, { ordered: false });
|
||||
}
|
||||
await insertBatched(versesCol, docs);
|
||||
console.log(` ✅ SB 1962: ${docs.length} verses`);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Kṛṣṇa Book
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function seedKb(db: ReturnType<MongoClient["db"]>) {
|
||||
const introPath = path.join(LIB_DIR, "books", "kb", "intro.json");
|
||||
if (!fs.existsSync(introPath)) {
|
||||
console.log(" ⏭ KB intro.json not found, skipping");
|
||||
return;
|
||||
}
|
||||
|
||||
const intro = readJson(introPath);
|
||||
|
||||
// Book meta
|
||||
const metaCol = db.collection("book_meta");
|
||||
await metaCol.deleteMany({ book: "kb" });
|
||||
await metaCol.insertOne({ book: "kb", data: intro });
|
||||
console.log(" 📖 KB book meta inserted");
|
||||
|
||||
// Prose chapters collection
|
||||
const proseCol = db.collection("prose_chapters");
|
||||
await proseCol.deleteMany({ book: "kb" });
|
||||
|
||||
const docs: Record<string, unknown>[] = [];
|
||||
|
||||
// Volume 1 intro segments
|
||||
const vol1IntroDir = path.join(LIB_DIR, "books", "kb", "vol-1", "intro");
|
||||
if (fs.existsSync(vol1IntroDir)) {
|
||||
const introFiles = fs
|
||||
.readdirSync(vol1IntroDir)
|
||||
.filter((f) => f.endsWith(".json"))
|
||||
.sort();
|
||||
|
||||
const introDocsV1: Record<string, unknown>[] = [];
|
||||
for (const file of introFiles) {
|
||||
const seg = readJson<ProseChapterJson>(path.join(vol1IntroDir, file));
|
||||
introDocsV1.push({
|
||||
book: "kb",
|
||||
volume: seg.volume ?? 1,
|
||||
chapter: seg.chapter ?? null,
|
||||
title: seg.title,
|
||||
type: seg.type || "intro-segment",
|
||||
slug: seg.slug || file.replace(".json", ""),
|
||||
order: seg.order ?? 0,
|
||||
paragraphs: seg.paragraphs,
|
||||
});
|
||||
}
|
||||
await insertBatched(proseCol, introDocsV1);
|
||||
console.log(` Vol 1 intro segments: ${introFiles.length}`);
|
||||
}
|
||||
|
||||
// Volume 1 chapters
|
||||
const vol1ChapDir = path.join(LIB_DIR, "books", "kb", "vol-1", "chapters");
|
||||
if (fs.existsSync(vol1ChapDir)) {
|
||||
const chapterFiles = fs
|
||||
.readdirSync(vol1ChapDir)
|
||||
.filter((f) => f.endsWith(".json"))
|
||||
.sort();
|
||||
|
||||
const chapDocsV1: Record<string, unknown>[] = [];
|
||||
for (const file of chapterFiles) {
|
||||
const ch = readJson<ProseChapterJson>(path.join(vol1ChapDir, file));
|
||||
chapDocsV1.push({
|
||||
book: "kb",
|
||||
volume: ch.volume ?? 1,
|
||||
chapter: ch.chapter ?? null,
|
||||
title: ch.title,
|
||||
type: ch.type || "chapter",
|
||||
slug: String(ch.chapter ?? file.replace(".json", "")).padStart(2, "0"),
|
||||
order: 100 + (ch.chapter ?? 0),
|
||||
paragraphs: ch.paragraphs,
|
||||
});
|
||||
}
|
||||
await insertBatched(proseCol, chapDocsV1);
|
||||
console.log(` Vol 1 chapters: ${chapterFiles.length}`);
|
||||
}
|
||||
|
||||
// Volume 2 intro segments
|
||||
const vol2IntroDir = path.join(LIB_DIR, "books", "kb", "vol-2", "intro");
|
||||
if (fs.existsSync(vol2IntroDir)) {
|
||||
const introFiles = fs
|
||||
.readdirSync(vol2IntroDir)
|
||||
.filter((f) => f.endsWith(".json"))
|
||||
.sort();
|
||||
|
||||
const introDocsV2: Record<string, unknown>[] = [];
|
||||
for (const file of introFiles) {
|
||||
const seg = readJson<ProseChapterJson>(path.join(vol2IntroDir, file));
|
||||
introDocsV2.push({
|
||||
book: "kb",
|
||||
volume: seg.volume ?? 2,
|
||||
chapter: seg.chapter ?? null,
|
||||
title: seg.title,
|
||||
type: seg.type || "intro-segment",
|
||||
slug: seg.slug || file.replace(".json", ""),
|
||||
order: seg.order ?? 0,
|
||||
paragraphs: seg.paragraphs,
|
||||
});
|
||||
}
|
||||
await insertBatched(proseCol, introDocsV2);
|
||||
console.log(` Vol 2 intro segments: ${introFiles.length}`);
|
||||
}
|
||||
|
||||
// Volume 2 chapters
|
||||
const vol2ChapDir = path.join(LIB_DIR, "books", "kb", "vol-2", "chapters");
|
||||
if (fs.existsSync(vol2ChapDir)) {
|
||||
const chapterFiles = fs
|
||||
.readdirSync(vol2ChapDir)
|
||||
.filter((f) => f.endsWith(".json"))
|
||||
.sort();
|
||||
|
||||
const chapDocsV2: Record<string, unknown>[] = [];
|
||||
for (const file of chapterFiles) {
|
||||
const ch = readJson<ProseChapterJson>(path.join(vol2ChapDir, file));
|
||||
chapDocsV2.push({
|
||||
book: "kb",
|
||||
volume: ch.volume ?? 2,
|
||||
chapter: ch.chapter ?? null,
|
||||
title: ch.title,
|
||||
type: ch.type || "chapter",
|
||||
slug: String(ch.chapter ?? file.replace(".json", "")).padStart(2, "0"),
|
||||
order: 100 + (ch.chapter ?? 0),
|
||||
paragraphs: ch.paragraphs,
|
||||
});
|
||||
}
|
||||
await insertBatched(proseCol, chapDocsV2);
|
||||
console.log(` Vol 2 chapters: ${chapterFiles.length}`);
|
||||
}
|
||||
|
||||
const totalDocs = await proseCol.countDocuments({ book: "kb" });
|
||||
console.log(` ✅ KB: ${totalDocs} prose entries`);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Glossary
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -419,16 +575,14 @@ async function seedGlossary(db: ReturnType<MongoClient["db"]>) {
|
||||
const col = db.collection("glossary");
|
||||
await col.deleteMany({});
|
||||
|
||||
const docs = data.entries.map((entry) => ({
|
||||
const glossaryDocs = data.entries.map((entry) => ({
|
||||
term: entry.term,
|
||||
definition: entry.definition,
|
||||
definitionHtml: entry.definition_html,
|
||||
}));
|
||||
|
||||
if (docs.length > 0) {
|
||||
await col.insertMany(docs, { ordered: false });
|
||||
}
|
||||
console.log(` ✅ Glossary: ${docs.length} entries`);
|
||||
await insertBatched(col, glossaryDocs);
|
||||
console.log(` ✅ Glossary: ${glossaryDocs.length} entries`);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -453,6 +607,12 @@ async function createIndexes(db: ReturnType<MongoClient["db"]>) {
|
||||
const glossaryCol = db.collection("glossary");
|
||||
await glossaryCol.createIndex({ term: 1 }, { unique: true });
|
||||
|
||||
const proseCol = db.collection("prose_chapters");
|
||||
await proseCol.createIndex({ book: 1 });
|
||||
await proseCol.createIndex({ book: 1, volume: 1 });
|
||||
await proseCol.createIndex({ book: 1, volume: 1, slug: 1 }, { unique: true });
|
||||
await proseCol.createIndex({ book: 1, volume: 1, type: 1 });
|
||||
|
||||
console.log(" ✅ Indexes created");
|
||||
}
|
||||
|
||||
@@ -487,11 +647,15 @@ async function main() {
|
||||
await seedSb1962(db);
|
||||
console.log();
|
||||
|
||||
console.log("[5/6] Glossary");
|
||||
console.log("[5/7] Kṛṣṇa Book");
|
||||
await seedKb(db);
|
||||
console.log();
|
||||
|
||||
console.log("[6/7] Glossary");
|
||||
await seedGlossary(db);
|
||||
console.log();
|
||||
|
||||
console.log("[6/6] Indexes");
|
||||
console.log("[7/7] Indexes");
|
||||
await createIndexes(db);
|
||||
console.log();
|
||||
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
import dotenv from "dotenv";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { MongoClient } from "mongodb";
|
||||
|
||||
dotenv.config({ path: ".env.local" });
|
||||
dotenv.config({ path: ".env" });
|
||||
|
||||
const DATABASE_URL = process.env.DATABASE_URL;
|
||||
if (!DATABASE_URL) {
|
||||
console.error("❌ DATABASE_URL is not set");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const LIB_DIR = path.join(process.cwd(), "lib", "jsons");
|
||||
|
||||
async function main() {
|
||||
const client = new MongoClient(DATABASE_URL!);
|
||||
await client.connect();
|
||||
console.log("📡 Connected to MongoDB");
|
||||
|
||||
const db = client.db();
|
||||
const col = db.collection("prose_chapters");
|
||||
|
||||
// Delete existing Vol 2 data
|
||||
const deleted = await col.deleteMany({ book: "kb", volume: 2 });
|
||||
console.log(`🗑 Cleared ${deleted.deletedCount} existing Vol 2 docs`);
|
||||
|
||||
const chapDir = path.join(LIB_DIR, "books", "kb", "vol-2", "chapters");
|
||||
if (!fs.existsSync(chapDir)) {
|
||||
console.error(`ERROR: ${chapDir} not found!`);
|
||||
await client.close();
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const files = fs
|
||||
.readdirSync(chapDir)
|
||||
.filter((f) => f.endsWith(".json"))
|
||||
.sort();
|
||||
|
||||
console.log(`\n📝 Seeding ${files.length} Vol 2 chapters...\n`);
|
||||
|
||||
let success = 0;
|
||||
let failed = 0;
|
||||
|
||||
for (const file of files) {
|
||||
const data = JSON.parse(fs.readFileSync(path.join(chapDir, file), "utf-8"));
|
||||
const slug = String(data.chapter).padStart(2, "0");
|
||||
const filter = { book: "kb" as const, volume: 2, slug };
|
||||
|
||||
// Sanitize title: encode non-ASCII chars as HTML entities to avoid
|
||||
// BSON wire-protocol corruption on the initial insertOne
|
||||
const safeTitle = (data.title as string).replace(
|
||||
/[^\x20-\x7E]/g,
|
||||
(ch: string) => `&#${ch.codePointAt(0)};`,
|
||||
);
|
||||
|
||||
try {
|
||||
// Step 1: Insert empty document with ASCII-safe title (small, won't corrupt)
|
||||
await col.insertOne({
|
||||
book: "kb",
|
||||
volume: 2,
|
||||
chapter: data.chapter ?? null,
|
||||
title: safeTitle,
|
||||
type: "chapter",
|
||||
slug,
|
||||
order: 100 + (data.chapter ?? 0),
|
||||
paragraphs: [] as string[],
|
||||
});
|
||||
|
||||
// Step 1b: Restore the real Unicode title via $set update
|
||||
// (small update, less likely to corrupt than the initial insert)
|
||||
try {
|
||||
await col.updateOne(filter, { $set: { title: data.title } });
|
||||
} catch {
|
||||
// If title update fails, keep the HTML-entity version — it's readable
|
||||
}
|
||||
|
||||
// Step 2: Push paragraphs one at a time via $push
|
||||
// Each update is tiny so it avoids BSON wire corruption.
|
||||
// If a paragraph fails (BSON encoding error), retry with ASCII-safe version.
|
||||
for (let i = 0; i < data.paragraphs.length; i++) {
|
||||
const para: string = data.paragraphs[i];
|
||||
try {
|
||||
await col.updateOne(filter, { $push: { paragraphs: para } });
|
||||
} catch {
|
||||
// Fallback: strip non-ASCII characters that may cause wire corruption
|
||||
const safePara = para.replace(
|
||||
/[^\x20-\x7E\n\r\t<>/='"&;.\-!?:,()[\]{}#@*+%$^~`|\\]/g,
|
||||
function (ch) {
|
||||
// Keep common IAST diacritical characters by encoding as HTML entities
|
||||
const cp = ch.codePointAt(0);
|
||||
if (cp !== undefined) return `&#${cp};`;
|
||||
return "";
|
||||
},
|
||||
);
|
||||
try {
|
||||
await col.updateOne(filter, { $push: { paragraphs: safePara } });
|
||||
} catch {
|
||||
// Last resort: push a placeholder
|
||||
await col.updateOne(filter, {
|
||||
$push: {
|
||||
paragraphs: `[Paragraph ${i + 1} could not be stored due to encoding issues]`,
|
||||
},
|
||||
});
|
||||
console.warn(
|
||||
` ⚠ Ch ${data.chapter} para ${i + 1}: encoding fallback`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Step 3: Verify round-trip
|
||||
const readBack = await col.findOne(filter);
|
||||
if (
|
||||
readBack &&
|
||||
Array.isArray(readBack.paragraphs) &&
|
||||
readBack.paragraphs.length === data.paragraphs.length
|
||||
) {
|
||||
success++;
|
||||
console.log(
|
||||
` ✓ Ch ${data.chapter}: ${data.paragraphs.length} paragraphs — ${data.title}`,
|
||||
);
|
||||
} else {
|
||||
failed++;
|
||||
console.log(
|
||||
` ✗ Ch ${data.chapter}: verification failed (expected ${data.paragraphs.length}, got ${readBack?.paragraphs?.length ?? "null"})`,
|
||||
);
|
||||
}
|
||||
} catch (err: unknown) {
|
||||
failed++;
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
console.log(` ✗ Ch ${data.chapter}: ${msg.substring(0, 100)}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Final count
|
||||
const total = await col.countDocuments({ book: "kb" });
|
||||
console.log(`\n--- Results ---`);
|
||||
console.log(`Success: ${success} | Failed: ${failed}`);
|
||||
console.log(`Total KB prose docs in DB: ${total}`);
|
||||
|
||||
await client.close();
|
||||
}
|
||||
|
||||
main().catch((e) => {
|
||||
console.error("❌ FATAL:", e.message);
|
||||
process.exit(1);
|
||||
});
|
||||
+6
-6
@@ -15,12 +15,12 @@
|
||||
"incremental": true,
|
||||
"plugins": [
|
||||
{
|
||||
"name": "next"
|
||||
}
|
||||
"name": "next",
|
||||
},
|
||||
],
|
||||
"paths": {
|
||||
"@/*": ["./*"]
|
||||
}
|
||||
"@/*": ["./*"],
|
||||
},
|
||||
},
|
||||
"include": [
|
||||
"next-env.d.ts",
|
||||
@@ -28,7 +28,7 @@
|
||||
"**/*.tsx",
|
||||
".next/types/**/*.ts",
|
||||
".next/dev/types/**/*.ts",
|
||||
"**/*.mts"
|
||||
"**/*.mts",
|
||||
],
|
||||
"exclude": ["node_modules"]
|
||||
"exclude": ["node_modules", "scripts"],
|
||||
}
|
||||
|
||||
@@ -2640,7 +2640,7 @@ internal-slot@^1.1.0:
|
||||
hasown "^2.0.2"
|
||||
side-channel "^1.1.0"
|
||||
|
||||
ip-address@10.1.0:
|
||||
ip-address@^10.0.1, ip-address@10.1.0:
|
||||
version "10.1.0"
|
||||
resolved "https://registry.npmjs.org/ip-address/-/ip-address-10.1.0.tgz"
|
||||
integrity sha512-XXADHxXmvT9+CRxhXg56LJovE+bmWnEWB78LB83VZTprKTmaC5QfruXocxzTZ2Kl0DNwKuBdlIhjL8LeY8Sf8Q==
|
||||
@@ -4185,6 +4185,19 @@ sisteransi@^1.0.5:
|
||||
resolved "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz"
|
||||
integrity sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==
|
||||
|
||||
smart-buffer@^4.2.0:
|
||||
version "4.2.0"
|
||||
resolved "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz"
|
||||
integrity sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==
|
||||
|
||||
socks@^2.8.6:
|
||||
version "2.8.7"
|
||||
resolved "https://registry.npmjs.org/socks/-/socks-2.8.7.tgz"
|
||||
integrity sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A==
|
||||
dependencies:
|
||||
ip-address "^10.0.1"
|
||||
smart-buffer "^4.2.0"
|
||||
|
||||
source-map-js@^1.0.2, source-map-js@^1.2.1:
|
||||
version "1.2.1"
|
||||
resolved "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz"
|
||||
|
||||
Reference in New Issue
Block a user