From 35d1d14f37767a30c586af0df753fefccb7a2c70 Mon Sep 17 00:00:00 2001 From: gosangam Date: Sun, 3 May 2026 21:02:41 +0530 Subject: [PATCH] feat: add journals --- PROJECT_CONTEXT.md | 59 +- app/books/bk/[chapter]/page.tsx | 249 ++++ app/books/bk/page.tsx | 66 ++ app/books/cbhc/[chapter]/page.tsx | 249 ++++ app/books/cbhc/page.tsx | 66 ++ app/books/mog/[chapter]/page.tsx | 276 +++++ app/books/mog/page.tsx | 100 ++ app/books/pk/[chapter]/page.tsx | 239 ++++ app/books/pk/page.tsx | 66 ++ app/books/va/[chapter]/page.tsx | 192 ++++ app/books/va/page.tsx | 61 + app/essays/essays-grid.tsx | 144 +++ app/essays/page.tsx | 39 +- app/journals/[slug]/[chapter]/page.tsx | 223 ++++ app/journals/[slug]/page.tsx | 138 +++ app/journals/page.tsx | 52 + app/other-documents/[slug]/[chapter]/page.tsx | 229 ++++ app/other-documents/[slug]/page.tsx | 75 ++ app/other-documents/other-documents-grid.tsx | 141 +++ app/other-documents/page.tsx | 42 + app/page.tsx | 67 +- components/mobile-nav.tsx | 6 +- lib/bk.ts | 299 +++++ lib/cbhc.ts | 299 +++++ lib/journals.ts | 289 +++++ lib/mog.ts | 430 +++++++ lib/other-documents.ts | 327 ++++++ lib/pk.ts | 306 +++++ lib/va.ts | 167 +++ prisma/schema.prisma | 80 ++ scripts/bk_extract.py | 590 ++++++++++ scripts/cbhc_extract.py | 444 +++++++ scripts/cc_essays_extract.py | 675 +++++++++++ scripts/jaladuta_diary_extract.py | 688 +++++++++++ scripts/mog_extract.py | 539 +++++++++ scripts/od_analysis_bg_extract.py | 303 +++++ scripts/pk_extract.py | 533 +++++++++ scripts/seed-bk.ts | 169 +++ scripts/seed-cbhc.ts | 169 +++ scripts/seed-cc-essays.ts | 163 +++ scripts/seed-journals.ts | 171 +++ scripts/seed-mog.ts | 204 ++++ scripts/seed-other-documents.ts | 274 +++++ scripts/seed-pk.ts | 169 +++ scripts/seed-pre-1977-essays.ts | 179 +++ scripts/seed-va.ts | 139 +++ scripts/the_beginning_extract.py | 1017 +++++++++++++++++ 47 files changed, 11365 insertions(+), 37 deletions(-) create mode 100644 app/books/bk/[chapter]/page.tsx create mode 100644 app/books/bk/page.tsx create mode 100644 app/books/cbhc/[chapter]/page.tsx create mode 100644 app/books/cbhc/page.tsx create mode 100644 app/books/mog/[chapter]/page.tsx create mode 100644 app/books/mog/page.tsx create mode 100644 app/books/pk/[chapter]/page.tsx create mode 100644 app/books/pk/page.tsx create mode 100644 app/books/va/[chapter]/page.tsx create mode 100644 app/books/va/page.tsx create mode 100644 app/essays/essays-grid.tsx create mode 100644 app/journals/[slug]/[chapter]/page.tsx create mode 100644 app/journals/[slug]/page.tsx create mode 100644 app/journals/page.tsx create mode 100644 app/other-documents/[slug]/[chapter]/page.tsx create mode 100644 app/other-documents/[slug]/page.tsx create mode 100644 app/other-documents/other-documents-grid.tsx create mode 100644 app/other-documents/page.tsx create mode 100644 lib/bk.ts create mode 100644 lib/cbhc.ts create mode 100644 lib/journals.ts create mode 100644 lib/mog.ts create mode 100644 lib/other-documents.ts create mode 100644 lib/pk.ts create mode 100644 lib/va.ts create mode 100644 scripts/bk_extract.py create mode 100644 scripts/cbhc_extract.py create mode 100644 scripts/cc_essays_extract.py create mode 100644 scripts/jaladuta_diary_extract.py create mode 100644 scripts/mog_extract.py create mode 100644 scripts/od_analysis_bg_extract.py create mode 100644 scripts/pk_extract.py create mode 100644 scripts/seed-bk.ts create mode 100644 scripts/seed-cbhc.ts create mode 100644 scripts/seed-cc-essays.ts create mode 100644 scripts/seed-journals.ts create mode 100644 scripts/seed-mog.ts create mode 100644 scripts/seed-other-documents.ts create mode 100644 scripts/seed-pk.ts create mode 100644 scripts/seed-pre-1977-essays.ts create mode 100644 scripts/seed-va.ts create mode 100644 scripts/the_beginning_extract.py diff --git a/PROJECT_CONTEXT.md b/PROJECT_CONTEXT.md index 154dda83..2731b12f 100644 --- a/PROJECT_CONTEXT.md +++ b/PROJECT_CONTEXT.md @@ -251,6 +251,9 @@ model GlossaryEntry { | SB First Canto (1962 Edition) | `sb1962` | 762 | 19 | — | | Śrī Īśopaniṣad | `iso` | 19 | — | Invocation + 18 mantras | | The Nectar of Instruction | `noi` | 11 | — | 11 texts | +| Śrī Brahma-saṁhitā | `bs` | 62 | 1 (Ch. 5) | — | +| Mukunda-mālā-stotra | `mm` | 6 | 1 | — | +| Nārada-bhakti-sūtra | `nbs` | 8 | 1 | — | ### Prose Books (stored in `prose_chapters` collection) @@ -272,13 +275,40 @@ model GlossaryEntry { | The Kṛṣṇa Consciousness Handbook | `hkh` | 19 | other.html | | The Science of Self-Realization | `ssr` | 50 sub-chapters (8 parents) | other.html | | Dialectic Spiritualism | `ds` | 36 sub-chapters (13 parents) | other.html | +| Life Comes from Life | `lcl` | 16 | other.html | +| Light of the Bhāgavata | `lob` | 48 | other.html | +| Teachings of Lord Kapila | `tlk` | 18 | other.html | +| Teachings of Queen Kuntī | `tqk` | 26 | other.html | +| Gītār Gān | `gg` | 18 | other.html (Bengali poetic BG) | +| Songs of the Vaiṣṇava Ācāryas | `sva` | 67 songs (4 parts, hierarchical) | other.html | +| Message of Godhead | `mog` | 3 | others-before-1977.html | +| Śrī Caitanya-bhāgavata Commentary | `cbhc` | 4 | others-before-1977.html | +| Bhagavāner Kathā | `bk` | 2 | others-before-1977.html (ScaGoudy) | +| Prayers of King Kulaśekhara (1958) | `pk` | 6 | others-before-1977.html | ### Essays (stored in `essays` collection) -| Essay | Slug | Group | -|-------|------|-------| -| Krishna The Reservoir Of Pleasure | `krishna-reservoir-of-pleasure` | `two-essays` | -| Who Is Crazy? | `who-is-crazy` | `two-essays` | +| Essay | Slug | Group | Year | +|-------|------|-------|------| +| Krishna The Reservoir Of Pleasure | `krishna-reservoir-of-pleasure` | `two-essays` | 1966 | +| Who Is Crazy? | `who-is-crazy` | `two-essays` | 1966 | +| The Spiritual Masters | `the-spiritual-masters` | `cc-essays` | 1960 | +| The Absolute Truth | `the-absolute-truth` | `cc-essays` | 1957 | +| The Causes of Lord Caitanya's Incarnation | `causes-of-lord-caitanyas-incarnation` | `cc-essays` | 1957 | +| The Confidential Reasons for Lord Caitanya's Appearance | `confidential-reasons-for-lord-caitanyas-appearance` | `cc-essays` | 1957 | +| The Glories of Lord Nityānanda | `glories-of-lord-nityananda` | `cc-essays` | 1957 | +| Vedānta-sūtra (Typed Transcript) | `vedanta-sutra-typed` | `typed-transcripts` | 1969 | +| Shaktipujah | `shaktipujah` | `reviews` | — | +| Devotee and the Divinity | `devotee-and-the-divinity` | `standalone` | — | + +### Other Documents (stored in `book_meta` + `prose_chapters` with `od-` prefix) + +| Document | Code | Chapters | +|----------|------|----------| +| An Analysis of Bhagwat Geeta | `od-analysis-bg` | 3 | +| Solution of Present Crisis by Bhagwat Geeta | `od-solution-crisis` | 4 | +| Śrī Caitanya Mahāprabhu | `od-sri-caitanya` | 3 | +| Conception of Geeta Nagari on the Principles of Gandhi | `od-geeta-nagari` | 1 | ### Other @@ -315,6 +345,11 @@ IAST transliteration → Devanagari via `indic-transliteration` library. Key fix - **TLC (1968 Edition)** — plain English transliterations ("Krishna" not "Kṛṣṇa") - **Two Essays (1966)** — plain English +- **GG (Gītār Gān)** — Bengali poetic text (ScaGoudy decoded for transliteration) +- **LCL (Life Comes from Life)** — dialogue book, ScaGoudy decoded +- **MOG, CBHC, PK** — pre-1977 handwritten documents, plain English +- **BK (Bhagavāner Kathā)** — pre-1977, uses ScaGoudy encoding +- **CC Essays, Vedānta-sūtra transcript** — plain English - All other books from other.html use ScaGoudy encoding --- @@ -331,13 +366,16 @@ IAST transliteration → Devanagari via `indic-transliteration` library. Key fix | `/books/sb1962/[chapter]/[verse]` | SB1962 | | `/books/iso/[mantra]` | Īśopaniṣad (verse + intro segments) | | `/books/noi/[text]` | Nectar of Instruction (verse + intro segments) | +| `/books/bs/[chapter]/[verse]` | Brahma-saṁhitā (chapter 5, 62 verses) | +| `/books/mm/[chapter]/[verse]` | Mukunda-mālā-stotra (6 mantras) | +| `/books/nbs/[chapter]/[verse]` | Nārada-bhakti-sūtra (8 sūtras) | ### Prose books (use per-book data layers + reading pages) | Route Pattern | Books | |---------------|-------| | `/books/kb/[volume]/[slug]` | Kṛṣṇa Book | -| `/books/{code}/[chapter]` | NoD, TLC, EJ, EK, MG, BBD, OWK, RV, POY, PQPA, TTP, OCHKM, HKH | +| `/books/{code}/[chapter]` | NoD, TLC, EJ, EK, MG, BBD, OWK, RV, POY, PQPA, TTP, OCHKM, HKH, LCL, LOB, TLK, TQK, GG, MOG, CBHC, BK, PK | ### Hierarchical prose books (sub-chapters as separate pages) @@ -345,14 +383,23 @@ IAST transliteration → Devanagari via `indic-transliteration` library. Key fix |---------------|-------| | `/books/ssr/[chapter]` | Science of Self-Realization (slugs: "1-1", "1-2", "intro-foreword", etc.) | | `/books/ds/[chapter]` | Dialectic Spiritualism (slugs: "1-1" = Socrates, "1-2" = Plato, etc.) | +| `/books/sva/[chapter]` | Songs of the Vaiṣṇava Ācāryas (slugs: "1-1" = Śrī Guru Praṇāma, etc.) | ### Essays | Route | Content | |-------|---------| -| `/essays` | All essays listing | +| `/essays` | All essays listing (10 essays across 4 groups) | | `/essays/[slug]` | Individual essay reader | +### Other Documents + +| Route | Content | +|-------|---------| +| `/other-documents` | All other documents listing | +| `/other-documents/[slug]` | Document index (chapter listing) | +| `/other-documents/[slug]/[chapter]` | Individual chapter reader | + ### Other routes | Route | Content | diff --git a/app/books/bk/[chapter]/page.tsx b/app/books/bk/[chapter]/page.tsx new file mode 100644 index 00000000..43ce392c --- /dev/null +++ b/app/books/bk/[chapter]/page.tsx @@ -0,0 +1,249 @@ +import Link from "next/link"; +import { notFound } from "next/navigation"; +import { + getBkContent, + getBkSidebarItems, + type BkChapter, + type BkSidebarItem, +} from "@/lib/bk"; +import { PageHeader } from "@/components/page-header"; +import { Card, CardContent } from "@/components/ui/card"; +import { buttonVariants } from "@/components/ui/button"; +import { ChevronLeft, ChevronRight } from "lucide-react"; +import { cn } from "@/lib/utils"; +import { MobileNav, type MobileNavGroup } from "@/components/mobile-nav"; + +export const dynamic = "force-dynamic"; + +// --------------------------------------------------------------------------- +// Page +// --------------------------------------------------------------------------- + +export default async function BkChapterPage({ + params, +}: { + params: Promise<{ chapter: string }>; +}) { + const { chapter: slug } = await params; + + const content = await getBkContent(slug); + if (!content) notFound(); + + const sidebarItems = await getBkSidebarItems(); + const currentIdx = sidebarItems.findIndex((item) => item.slug === slug); + + const prev = currentIdx > 0 ? sidebarItems[currentIdx - 1] : null; + const next = + currentIdx < sidebarItems.length - 1 ? sidebarItems[currentIdx + 1] : null; + + // Build the page title + const chapterNum = (content as BkChapter).chapter ?? null; + + return ( +
+ + + ← Bhagavāner Kathā + +

+ {chapterNum != null && ( + Chapter {chapterNum}: + )} + {content.title} +

+ + +
+ {/* Left sidebar */} + + + {/* Main content */} +
+ {/* Top navigation */} + + + {/* Content */} + + +
+ {content.paragraphs.map((paragraph, i) => ( +
+ ))} +
+ + + + {/* Bottom navigation */} + +
+
+
+ ); +} + +// --------------------------------------------------------------------------- +// Sidebar +// --------------------------------------------------------------------------- + +function BkSidebar({ + items, + currentSlug, +}: { + items: BkSidebarItem[]; + currentSlug: string; +}) { + const chapterItems = items.filter((item) => item.type === "chapter"); + + return ( + + ); +} + +function SidebarLink({ + item, + isActive, +}: { + item: BkSidebarItem; + isActive: boolean; +}) { + let label: string; + if (item.type === "chapter" && item.chapter != null) { + // Extract just the title portion after "Chapter N: " + const titlePart = item.label.replace(/^Chapter \d+:\s*/, ""); + label = `${item.chapter}. ${titlePart}`; + } else { + label = item.label; + } + + return ( + + {label} + + ); +} + +// --------------------------------------------------------------------------- +// Prev / Next navigation +// --------------------------------------------------------------------------- + +function BkNav({ + prev, + next, + sidebarItems, + currentSlug, +}: { + prev: BkSidebarItem | null; + next: BkSidebarItem | null; + sidebarItems: BkSidebarItem[]; + currentSlug: string; +}) { + const mobileNavGroups: MobileNavGroup[] = []; + + const chapterItems = sidebarItems.filter((i) => i.type === "chapter"); + if (chapterItems.length > 0) { + mobileNavGroups.push({ + title: "Chapters", + items: chapterItems.map((i) => ({ + slug: i.slug, + label: i.label, + href: `/books/bk/${i.slug}`, + type: i.type, + })), + }); + } + + return ( +
+
+ + {prev ? ( + + + + {prev.type === "chapter" && prev.chapter != null + ? `Ch. ${prev.chapter}` + : prev.label} + + Previous + + ) : ( +
+ )} +
+ + {next ? ( + + + {next.type === "chapter" && next.chapter != null + ? `Ch. ${next.chapter}` + : next.label} + + Next + + + ) : ( +
+ )} +
+ ); +} diff --git a/app/books/bk/page.tsx b/app/books/bk/page.tsx new file mode 100644 index 00000000..853d0eda --- /dev/null +++ b/app/books/bk/page.tsx @@ -0,0 +1,66 @@ +export const dynamic = "force-dynamic"; + +import Link from "next/link"; +import { notFound } from "next/navigation"; +import { getBkMeta } from "@/lib/bk"; +import { PageHeader } from "@/components/page-header"; +import { Card, CardContent } from "@/components/ui/card"; + +export default async function BkPage() { + const meta = await getBkMeta(); + if (!meta) notFound(); + + return ( +
+ + + ← Books + +

+ {meta.title} +

+

{meta.author}

+ + +
+ {/* Description */} + {meta.description && ( +

{meta.description}

+ )} + + {/* Chapters */} +
+

+ Chapters +

+ + +
    + {meta.chapters.map((ch) => ( +
  • + +
    + + Chapter {ch.number} + +

    + {ch.title} +

    +
    + +
    • + ))} +
+ + +
+
+
+ ); +} diff --git a/app/books/cbhc/[chapter]/page.tsx b/app/books/cbhc/[chapter]/page.tsx new file mode 100644 index 00000000..1e194a11 --- /dev/null +++ b/app/books/cbhc/[chapter]/page.tsx @@ -0,0 +1,249 @@ +import Link from "next/link"; +import { notFound } from "next/navigation"; +import { + getCbhcContent, + getCbhcSidebarItems, + type CbhcChapter, + type CbhcSidebarItem, +} from "@/lib/cbhc"; +import { PageHeader } from "@/components/page-header"; +import { Card, CardContent } from "@/components/ui/card"; +import { buttonVariants } from "@/components/ui/button"; +import { ChevronLeft, ChevronRight } from "lucide-react"; +import { cn } from "@/lib/utils"; +import { MobileNav, type MobileNavGroup } from "@/components/mobile-nav"; + +export const dynamic = "force-dynamic"; + +// --------------------------------------------------------------------------- +// Page +// --------------------------------------------------------------------------- + +export default async function CbhcChapterPage({ + params, +}: { + params: Promise<{ chapter: string }>; +}) { + const { chapter: slug } = await params; + + const content = await getCbhcContent(slug); + if (!content) notFound(); + + const sidebarItems = await getCbhcSidebarItems(); + const currentIdx = sidebarItems.findIndex((item) => item.slug === slug); + + const prev = currentIdx > 0 ? sidebarItems[currentIdx - 1] : null; + const next = + currentIdx < sidebarItems.length - 1 ? sidebarItems[currentIdx + 1] : null; + + // Build the page title + const chapterNum = (content as CbhcChapter).chapter ?? null; + + return ( +
+ + + ← Śrī Caitanya-bhāgavata Commentary + +

+ {chapterNum != null && ( + Chapter {chapterNum}: + )} + {content.title} +

+ + +
+ {/* Left sidebar */} + + + {/* Main content */} +
+ {/* Top navigation */} + + + {/* Content */} + + +
+ {content.paragraphs.map((paragraph, i) => ( +
+ ))} +
+ + + + {/* Bottom navigation */} + +
+
+
+ ); +} + +// --------------------------------------------------------------------------- +// Sidebar +// --------------------------------------------------------------------------- + +function CbhcSidebar({ + items, + currentSlug, +}: { + items: CbhcSidebarItem[]; + currentSlug: string; +}) { + const chapterItems = items.filter((item) => item.type === "chapter"); + + return ( + + ); +} + +function SidebarLink({ + item, + isActive, +}: { + item: CbhcSidebarItem; + isActive: boolean; +}) { + let label: string; + if (item.type === "chapter" && item.chapter != null) { + // Extract just the title portion after "Chapter N: " + const titlePart = item.label.replace(/^Chapter \d+:\s*/, ""); + label = `${item.chapter}. ${titlePart}`; + } else { + label = item.label; + } + + return ( + + {label} + + ); +} + +// --------------------------------------------------------------------------- +// Prev / Next navigation +// --------------------------------------------------------------------------- + +function CbhcNav({ + prev, + next, + sidebarItems, + currentSlug, +}: { + prev: CbhcSidebarItem | null; + next: CbhcSidebarItem | null; + sidebarItems: CbhcSidebarItem[]; + currentSlug: string; +}) { + const mobileNavGroups: MobileNavGroup[] = []; + + const chapterItems = sidebarItems.filter((i) => i.type === "chapter"); + if (chapterItems.length > 0) { + mobileNavGroups.push({ + title: "Chapters", + items: chapterItems.map((i) => ({ + slug: i.slug, + label: i.label, + href: `/books/cbhc/${i.slug}`, + type: i.type, + })), + }); + } + + return ( +
+
+ + {prev ? ( + + + + {prev.type === "chapter" && prev.chapter != null + ? `Ch. ${prev.chapter}` + : prev.label} + + Previous + + ) : ( +
+ )} +
+ + {next ? ( + + + {next.type === "chapter" && next.chapter != null + ? `Ch. ${next.chapter}` + : next.label} + + Next + + + ) : ( +
+ )} +
+ ); +} diff --git a/app/books/cbhc/page.tsx b/app/books/cbhc/page.tsx new file mode 100644 index 00000000..98b1a27f --- /dev/null +++ b/app/books/cbhc/page.tsx @@ -0,0 +1,66 @@ +export const dynamic = "force-dynamic"; + +import Link from "next/link"; +import { notFound } from "next/navigation"; +import { getCbhcMeta } from "@/lib/cbhc"; +import { PageHeader } from "@/components/page-header"; +import { Card, CardContent } from "@/components/ui/card"; + +export default async function CbhcPage() { + const meta = await getCbhcMeta(); + if (!meta) notFound(); + + return ( +
+ + + ← Books + +

+ {meta.title} +

+

{meta.author}

+ + +
+ {/* Description */} + {meta.description && ( +

{meta.description}

+ )} + + {/* Chapters */} +
+

+ Chapters +

+ + +
    + {meta.chapters.map((ch) => ( +
  • + +
    + + Chapter {ch.number} + +

    + {ch.title} +

    +
    + +
    • + ))} +
+ + +
+
+
+ ); +} diff --git a/app/books/mog/[chapter]/page.tsx b/app/books/mog/[chapter]/page.tsx new file mode 100644 index 00000000..0d15baa7 --- /dev/null +++ b/app/books/mog/[chapter]/page.tsx @@ -0,0 +1,276 @@ +import Link from "next/link"; +import { notFound } from "next/navigation"; +import { + getMogContent, + getMogSidebarItems, + type MogChapter, + type MogSidebarItem, +} from "@/lib/mog"; +import { PageHeader } from "@/components/page-header"; +import { Card, CardContent } from "@/components/ui/card"; +import { buttonVariants } from "@/components/ui/button"; +import { ChevronLeft, ChevronRight } from "lucide-react"; +import { cn } from "@/lib/utils"; +import { MobileNav, type MobileNavGroup } from "@/components/mobile-nav"; + +export const dynamic = "force-dynamic"; + +// --------------------------------------------------------------------------- +// Page +// --------------------------------------------------------------------------- + +export default async function MogChapterPage({ + params, +}: { + params: Promise<{ chapter: string }>; +}) { + const { chapter: slug } = await params; + + const content = await getMogContent(slug); + if (!content) notFound(); + + const sidebarItems = await getMogSidebarItems(); + const currentIdx = sidebarItems.findIndex((item) => item.slug === slug); + + const prev = currentIdx > 0 ? sidebarItems[currentIdx - 1] : null; + const next = + currentIdx < sidebarItems.length - 1 ? sidebarItems[currentIdx + 1] : null; + + // Build the page title + const isChapter = + "chapter" in content && + typeof (content as MogChapter).chapter === "number" && + (content as MogChapter).chapter !== null; + const chapterNum = isChapter ? (content as MogChapter).chapter : null; + + return ( +
+ + + ← Message of Godhead + +

+ {chapterNum != null && ( + Chapter {chapterNum}: + )} + {content.title} +

+ + +
+ {/* Left sidebar */} + + + {/* Main content */} +
+ {/* Top navigation */} + + + {/* Content */} + + +
+ {content.paragraphs.map((paragraph, i) => ( +
+ ))} +
+ + + + {/* Bottom navigation */} + +
+
+
+ ); +} + +// --------------------------------------------------------------------------- +// Sidebar +// --------------------------------------------------------------------------- + +function MogSidebar({ + items, + currentSlug, +}: { + items: MogSidebarItem[]; + currentSlug: string; +}) { + const introItems = items.filter((item) => item.type === "intro-segment"); + const chapterItems = items.filter((item) => item.type === "chapter"); + + return ( + + ); +} + +function SidebarLink({ + item, + isActive, +}: { + item: MogSidebarItem; + isActive: boolean; +}) { + return ( + + {item.label} + + ); +} + +// --------------------------------------------------------------------------- +// Prev / Next navigation +// --------------------------------------------------------------------------- + +function MogNav({ + prev, + next, + sidebarItems, + currentSlug, +}: { + prev: MogSidebarItem | null; + next: MogSidebarItem | null; + sidebarItems: MogSidebarItem[]; + currentSlug: string; +}) { + const mobileNavGroups: MobileNavGroup[] = []; + + const introItems = sidebarItems.filter((i) => i.type === "intro-segment"); + if (introItems.length > 0) { + mobileNavGroups.push({ + title: "Front Matter", + items: introItems.map((i) => ({ + slug: i.slug, + label: i.label, + href: `/books/mog/${i.slug}`, + type: i.type, + })), + }); + } + + const chapterItems = sidebarItems.filter((i) => i.type === "chapter"); + if (chapterItems.length > 0) { + mobileNavGroups.push({ + title: "Chapters", + items: chapterItems.map((i) => ({ + slug: i.slug, + label: i.label, + href: `/books/mog/${i.slug}`, + type: i.type, + })), + }); + } + + return ( +
+
+ + {prev ? ( + + + + {prev.type === "chapter" && prev.chapter != null + ? `Ch. ${prev.chapter}` + : prev.label} + + Previous + + ) : ( +
+ )} +
+ + {next ? ( + + + {next.type === "chapter" && next.chapter != null + ? `Ch. ${next.chapter}` + : next.label} + + Next + + + ) : ( +
+ )} +
+ ); +} diff --git a/app/books/mog/page.tsx b/app/books/mog/page.tsx new file mode 100644 index 00000000..11611c97 --- /dev/null +++ b/app/books/mog/page.tsx @@ -0,0 +1,100 @@ +export const dynamic = "force-dynamic"; + +import Link from "next/link"; +import { notFound } from "next/navigation"; +import { getMogMeta, getMogIntroSegments } from "@/lib/mog"; +import { PageHeader } from "@/components/page-header"; +import { Card, CardContent } from "@/components/ui/card"; + +export default async function MogPage() { + const meta = await getMogMeta(); + if (!meta) notFound(); + + const introSegments = await getMogIntroSegments(); + + return ( +
+ + + ← Books + +

+ {meta.title} +

+

{meta.author}

+ + +
+ {/* Description */} + {meta.description && ( +

{meta.description}

+ )} + + {/* Front Matter */} + {introSegments && introSegments.length > 0 && ( +
+

+ Front Matter +

+ + +
    + {introSegments.map((seg) => ( +
  • + +
    +

    + {seg.title} +

    +
    + + {seg.paragraphs.length} paragraphs + + +
    • + ))} +
+ + +
+ )} + + {/* Chapters */} +
+

+ Chapters +

+ + +
    + {meta.chapters.map((ch) => ( +
  • + +
    + + Chapter {ch.number} + +

    + {ch.title} +

    +
    + +
    • + ))} +
+ + +
+
+
+ ); +} diff --git a/app/books/pk/[chapter]/page.tsx b/app/books/pk/[chapter]/page.tsx new file mode 100644 index 00000000..cefed86e --- /dev/null +++ b/app/books/pk/[chapter]/page.tsx @@ -0,0 +1,239 @@ +import Link from "next/link"; +import { notFound } from "next/navigation"; +import { + getPkContent, + getPkSidebarItems, + type PkChapter, + type PkSidebarItem, +} from "@/lib/pk"; +import { PageHeader } from "@/components/page-header"; +import { Card, CardContent } from "@/components/ui/card"; +import { buttonVariants } from "@/components/ui/button"; +import { ChevronLeft, ChevronRight } from "lucide-react"; +import { cn } from "@/lib/utils"; +import { MobileNav, type MobileNavGroup } from "@/components/mobile-nav"; + +export const dynamic = "force-dynamic"; + +// --------------------------------------------------------------------------- +// Page +// --------------------------------------------------------------------------- + +export default async function PkChapterPage({ + params, +}: { + params: Promise<{ chapter: string }>; +}) { + const { chapter: slug } = await params; + + const content = await getPkContent(slug); + if (!content) notFound(); + + const sidebarItems = await getPkSidebarItems(); + const currentIdx = sidebarItems.findIndex((item) => item.slug === slug); + + const prev = currentIdx > 0 ? sidebarItems[currentIdx - 1] : null; + const next = + currentIdx < sidebarItems.length - 1 ? sidebarItems[currentIdx + 1] : null; + + const chapterNum = (content as PkChapter).chapter; + + return ( +
+ + + ← Prayers of King Kulaśekhara + +

+ {chapterNum != null && ( + Verse {chapterNum}: + )} + {content.title} +

+ + +
+ {/* Left sidebar */} + + + {/* Main content */} +
+ {/* Top navigation */} + + + {/* Content */} + + +
+ {content.paragraphs.map((paragraph, i) => ( +
+ ))} +
+ + + + {/* Bottom navigation */} + +
+
+
+ ); +} + +// --------------------------------------------------------------------------- +// Sidebar +// --------------------------------------------------------------------------- + +function PkSidebar({ + items, + currentSlug, +}: { + items: PkSidebarItem[]; + currentSlug: string; +}) { + const chapterItems = items.filter((item) => item.type === "chapter"); + + return ( + + ); +} + +function SidebarLink({ + item, + isActive, +}: { + item: PkSidebarItem; + isActive: boolean; +}) { + return ( + + {item.label} + + ); +} + +// --------------------------------------------------------------------------- +// Prev / Next navigation +// --------------------------------------------------------------------------- + +function PkNav({ + prev, + next, + sidebarItems, + currentSlug, +}: { + prev: PkSidebarItem | null; + next: PkSidebarItem | null; + sidebarItems: PkSidebarItem[]; + currentSlug: string; +}) { + const mobileNavGroups: MobileNavGroup[] = []; + + const chapterItems = sidebarItems.filter((i) => i.type === "chapter"); + if (chapterItems.length > 0) { + mobileNavGroups.push({ + title: "Prayers", + items: chapterItems.map((i) => ({ + slug: i.slug, + label: i.label, + href: `/books/pk/${i.slug}`, + type: i.type, + })), + }); + } + + return ( +
+
+ + {prev ? ( + + + + {prev.chapter != null + ? `Prayer ${prev.chapter}` + : prev.label} + + Previous + + ) : ( +
+ )} +
+ + {next ? ( + + + {next.chapter != null + ? `Prayer ${next.chapter}` + : next.label} + + Next + + + ) : ( +
+ )} +
+ ); +} diff --git a/app/books/pk/page.tsx b/app/books/pk/page.tsx new file mode 100644 index 00000000..8753b8eb --- /dev/null +++ b/app/books/pk/page.tsx @@ -0,0 +1,66 @@ +export const dynamic = "force-dynamic"; + +import Link from "next/link"; +import { notFound } from "next/navigation"; +import { getPkMeta } from "@/lib/pk"; +import { PageHeader } from "@/components/page-header"; +import { Card, CardContent } from "@/components/ui/card"; + +export default async function PkPage() { + const meta = await getPkMeta(); + if (!meta) notFound(); + + return ( +
+ + + ← Books + +

+ {meta.title} +

+

{meta.author}

+ + +
+ {/* Description */} + {meta.description && ( +

{meta.description}

+ )} + + {/* Chapters (Prayers) */} +
+

+ Prayers +

+ + +
    + {meta.chapters.map((ch) => ( +
  • + +
    + + Verse {ch.number} + +

    + {ch.title} +

    +
    + +
    • + ))} +
+ + +
+
+
+ ); +} diff --git a/app/books/va/[chapter]/page.tsx b/app/books/va/[chapter]/page.tsx new file mode 100644 index 00000000..32f5806e --- /dev/null +++ b/app/books/va/[chapter]/page.tsx @@ -0,0 +1,192 @@ +export const dynamic = "force-dynamic"; + +import Link from "next/link"; +import { notFound } from "next/navigation"; +import { + getVaContent, + getVaSidebarItems, + type VaChapter, + type VaSidebarItem, +} from "@/lib/va"; +import { PageHeader } from "@/components/page-header"; +import { Card, CardContent } from "@/components/ui/card"; +import { buttonVariants } from "@/components/ui/button"; +import { ChevronLeft, ChevronRight } from "lucide-react"; +import { cn } from "@/lib/utils"; +import { MobileNav, type MobileNavGroup } from "@/components/mobile-nav"; + +export default async function VaChapterPage({ + params, +}: { + params: Promise<{ chapter: string }>; +}) { + const { chapter: slug } = await params; + + const content = await getVaContent(slug); + if (!content) notFound(); + + const sidebarItems = await getVaSidebarItems(); + const currentIdx = sidebarItems.findIndex((item) => item.slug === slug); + + const prev = currentIdx > 0 ? sidebarItems[currentIdx - 1] : null; + const next = + currentIdx < sidebarItems.length - 1 ? sidebarItems[currentIdx + 1] : null; + + const chapterNum = (content as VaChapter).chapter; + + return ( +
+ + + ← Vaiśiṣṭyāṣṭakam + +

+ {content.title} +

+ + +
+ + +
+ + + + +
+ {content.paragraphs.map((paragraph, i) => ( +
+ ))} +
+ + + + +
+
+
+ ); +} + +function VaSidebar({ + items, + currentSlug, +}: { + items: VaSidebarItem[]; + currentSlug: string; +}) { + return ( + + ); +} + +function VaNav({ + prev, + next, + sidebarItems, + currentSlug, +}: { + prev: VaSidebarItem | null; + next: VaSidebarItem | null; + sidebarItems: VaSidebarItem[]; + currentSlug: string; +}) { + const mobileNavGroups: MobileNavGroup[] = []; + + if (sidebarItems.length > 0) { + mobileNavGroups.push({ + title: "Chapters", + items: sidebarItems.map((i) => ({ + slug: i.slug, + label: i.label, + href: `/books/va/${i.slug}`, + type: i.type, + })), + }); + } + + return ( +
+
+ + {prev ? ( + + + + {prev.label} + + Previous + + ) : ( +
+ )} +
+ + {next ? ( + + + {next.label} + + Next + + + ) : ( +
+ )} +
+ ); +} diff --git a/app/books/va/page.tsx b/app/books/va/page.tsx new file mode 100644 index 00000000..c4caec70 --- /dev/null +++ b/app/books/va/page.tsx @@ -0,0 +1,61 @@ +export const dynamic = "force-dynamic"; + +import Link from "next/link"; +import { notFound } from "next/navigation"; +import { getVaMeta } from "@/lib/va"; +import { PageHeader } from "@/components/page-header"; +import { Card, CardContent } from "@/components/ui/card"; + +export default async function VaPage() { + const meta = await getVaMeta(); + if (!meta) notFound(); + + return ( +
+ + + ← Books + +

+ {meta.title} +

+

{meta.author}

+ + +
+ {meta.description && ( +

{meta.description}

+ )} + +
+

+ Chapters +

+ + +
    + {meta.chapters.map((ch) => ( +
  • + +
    +

    + {ch.title} +

    +
    + +
    • + ))} +
+ + +
+
+
+ ); +} diff --git a/app/essays/essays-grid.tsx b/app/essays/essays-grid.tsx new file mode 100644 index 00000000..af6f5c26 --- /dev/null +++ b/app/essays/essays-grid.tsx @@ -0,0 +1,144 @@ +"use client"; + +import { useState, useMemo } from "react"; +import Link from "next/link"; +import { + Card, + CardHeader, + CardTitle, + CardContent, +} from "@/components/ui/card"; +import { Badge } from "@/components/ui/badge"; +import { Input } from "@/components/ui/input"; +import { cn } from "@/lib/utils"; +import { Search } from "lucide-react"; + +export type EssayInfo = { + slug: string; + title: string; + year: number | null; + group: string | null; + href: string; +}; + +/** + * Strip diacritics / accents so that "Krsna" matches "Kṛṣṇa", + * "Gita" matches "Gītā", "Srimad" matches "Śrīmad", etc. + */ +function normalize(text: string): string { + return ( + text + .toLowerCase() + .replace(/ṛ/g, "r") + .replace(/ṝ/g, "r") + .replace(/ṣ/g, "s") + .replace(/ś/g, "s") + .replace(/ṭ/g, "t") + .replace(/ḍ/g, "d") + .replace(/ṇ/g, "n") + .replace(/ñ/g, "n") + .replace(/ṅ/g, "n") + .replace(/ḥ/g, "h") + .replace(/ṁ/g, "m") + .replace(/ṃ/g, "m") + .replace(/ḷ/g, "l") + .normalize("NFD") + .replace(/[\u0300-\u036f]/g, "") + ); +} + +/** Human-readable group labels */ +const GROUP_LABELS: Record = { + "cc-essays": "Śrī Caitanya-caritāmṛta Essays", + "pre-1977-essays": "Pre-1977 Essays", + "two-essays": "Two Essays (1966)", + reviews: "Reviews", + standalone: "Standalone", + "typed-transcripts": "Typed Transcripts", +}; + +function groupLabel(group: string | null): string { + if (!group) return "Other"; + return GROUP_LABELS[group] ?? group; +} + +export function EssaysGrid({ essays }: { essays: EssayInfo[] }) { + const [query, setQuery] = useState(""); + + const normalizedEssays = useMemo( + () => + essays.map((essay) => ({ + essay, + searchText: normalize( + `${essay.title} ${essay.group ?? ""} ${essay.slug}`, + ), + })), + [essays], + ); + + const filtered = useMemo(() => { + if (!query.trim()) return essays; + const q = normalize(query.trim()); + return normalizedEssays + .filter((entry) => entry.searchText.includes(q)) + .map((entry) => entry.essay); + }, [essays, normalizedEssays, query]); + + return ( +
+ {/* Search input */} +
+ + setQuery(e.target.value)} + className="pl-9" + /> +
+ + {/* Results count when searching */} + {query.trim() && ( +

+ {filtered.length === 0 + ? "No essays found" + : `${filtered.length} essay${filtered.length === 1 ? "" : "s"} found`} +

+ )} + + {/* Grid */} +
+ {filtered.map((essay) => ( + + + + + {essay.title} + + + +
+ {essay.year && ( + + {essay.year} + + )} + {essay.group && ( + + {groupLabel(essay.group)} + + )} +
+ + + + ))} +
+
+ ); +} diff --git a/app/essays/page.tsx b/app/essays/page.tsx index ecaf9436..4e6eaf40 100644 --- a/app/essays/page.tsx +++ b/app/essays/page.tsx @@ -2,19 +2,27 @@ export const dynamic = "force-dynamic"; import Link from "next/link"; import { PageHeader } from "@/components/page-header"; -import { Card, CardContent } from "@/components/ui/card"; import { prisma } from "@/lib/db"; +import { EssaysGrid } from "./essays-grid"; async function getAllEssays() { return prisma.essay.findMany({ orderBy: [{ group: "asc" }, { order: "asc" }], - select: { slug: true, title: true, author: true, year: true, group: true }, + select: { slug: true, title: true, year: true, group: true }, }); } export default async function EssaysPage() { const essays = await getAllEssays(); + const items = essays.map((essay) => ({ + slug: essay.slug, + title: essay.title, + year: essay.year, + group: essay.group, + href: `/essays/${essay.slug}`, + })); + return (
@@ -30,31 +38,8 @@ export default async function EssaysPage() {

-
- - -
    - {essays.map((essay) => ( -
  • - -
    -

    - {essay.title} -

    -

    - {essay.author} - {essay.year ? ` · ${essay.year}` : ""} -

    -
    - -
    • - ))} -
- - +
+
); diff --git a/app/journals/[slug]/[chapter]/page.tsx b/app/journals/[slug]/[chapter]/page.tsx new file mode 100644 index 00000000..dfe6cf9b --- /dev/null +++ b/app/journals/[slug]/[chapter]/page.tsx @@ -0,0 +1,223 @@ +export const dynamic = "force-dynamic"; + +import Link from "next/link"; +import { notFound } from "next/navigation"; +import { + getJournalContent, + getJournalMeta, + getJournalSidebarItems, + slugToBookCode, + type JournalSidebarItem, +} from "@/lib/journals"; +import { PageHeader } from "@/components/page-header"; +import { Card, CardContent } from "@/components/ui/card"; +import { buttonVariants } from "@/components/ui/button"; +import { ChevronLeft, ChevronRight } from "lucide-react"; +import { cn } from "@/lib/utils"; +import { MobileNav, type MobileNavGroup } from "@/components/mobile-nav"; + +export default async function JournalChapterPage({ + params, +}: { + params: Promise<{ slug: string; chapter: string }>; +}) { + const { slug, chapter: chapterSlug } = await params; + const bookCode = slugToBookCode(slug); + + const [meta, content, sidebarItems] = await Promise.all([ + getJournalMeta(bookCode), + getJournalContent(bookCode, chapterSlug), + getJournalSidebarItems(bookCode), + ]); + + if (!meta || !content) notFound(); + + // Filter out heading items for prev/next navigation + const navItems = sidebarItems.filter((item) => item.type !== "heading"); + const currentIdx = navItems.findIndex((item) => item.slug === chapterSlug); + const prev = currentIdx > 0 ? navItems[currentIdx - 1] : null; + const next = + currentIdx < navItems.length - 1 ? navItems[currentIdx + 1] : null; + + return ( +
+ + + ← {meta.title} + +

+ {content.title} +

+ + +
+ + +
+ + + + +
+ {content.paragraphs.map((paragraph, i) => ( +
+ ))} +
+ + + + +
+
+
+ ); +} + +function ChapterSidebar({ + items, + currentSlug, + docSlug, +}: { + items: JournalSidebarItem[]; + currentSlug: string; + docSlug: string; +}) { + return ( + + ); +} + +function ChapterNav({ + prev, + next, + navItems, + currentSlug, + docSlug, + docTitle, + showMobileNav = false, +}: { + prev: JournalSidebarItem | null; + next: JournalSidebarItem | null; + navItems: JournalSidebarItem[]; + currentSlug: string; + docSlug: string; + docTitle: string; + showMobileNav?: boolean; +}) { + const mobileNavGroups: MobileNavGroup[] = []; + if (showMobileNav && navItems.length > 0) { + mobileNavGroups.push({ + title: "Chapters", + items: navItems.map((i) => ({ + slug: i.slug, + label: i.label, + href: `/journals/${docSlug}/${i.slug}`, + type: i.type, + })), + }); + } + + return ( +
+
+ {showMobileNav && ( + + )} + {prev ? ( + + + + {prev.label} + + Previous + + ) : ( +
+ )} +
+ {next ? ( + + + {next.label} + + Next + + + ) : ( +
+ )} +
+ ); +} diff --git a/app/journals/[slug]/page.tsx b/app/journals/[slug]/page.tsx new file mode 100644 index 00000000..3d224761 --- /dev/null +++ b/app/journals/[slug]/page.tsx @@ -0,0 +1,138 @@ +export const dynamic = "force-dynamic"; + +import Link from "next/link"; +import { notFound } from "next/navigation"; +import { + getJournalMeta, + slugToBookCode, + type JournalChapterEntry, +} from "@/lib/journals"; +import { PageHeader } from "@/components/page-header"; +import { Card, CardContent } from "@/components/ui/card"; + +export default async function JournalIndexPage({ + params, +}: { + params: Promise<{ slug: string }>; +}) { + const { slug } = await params; + const bookCode = slugToBookCode(slug); + const meta = await getJournalMeta(bookCode); + if (!meta) notFound(); + + return ( +
+ + + ← Journals + +

+ {meta.title} +

+

{meta.author}

+ + +
+ {meta.description && ( +

{meta.description}

+ )} + + {meta.chapters.map((ch: JournalChapterEntry, idx: number) => { + // Top-level chapter with a slug (Preface, Introduction, etc.) + if ("slug" in ch && !("subChapters" in ch)) { + return ( +
+ + + +

{ch.title}

+ + + +
+ ); + } + + // Group with sub-chapters (may be 2 or 3 levels deep) + if ("subChapters" in ch) { + const group = ch as { + title: string; + subChapters: { + slug?: string; + title: string; + subChapters?: { slug: string; title: string }[]; + }[]; + }; + return ( +
+

+ {group.title} +

+
+ {group.subChapters.map((sc, scIdx) => { + // Sub-chapter with its own nested sub-chapters (e.g., month → days) + if ("subChapters" in sc && sc.subChapters) { + return ( +
+

+ {sc.title} +

+ + +
    + {sc.subChapters.map((day) => ( +
  • + +

    + {day.title} +

    + +
    • + ))} +
+ + +
+ ); + } + + // Direct sub-chapter with a slug (e.g., Summary, Notice) + if ("slug" in sc && sc.slug) { + return ( + + + +

+ {sc.title} +

+ + + + ); + } + + return null; + })} +
+
+ ); + } + + return null; + })} +
+
+ ); +} diff --git a/app/journals/page.tsx b/app/journals/page.tsx new file mode 100644 index 00000000..cf00a129 --- /dev/null +++ b/app/journals/page.tsx @@ -0,0 +1,52 @@ +export const dynamic = "force-dynamic"; + +import Link from "next/link"; +import { PageHeader } from "@/components/page-header"; +import { Card, CardHeader, CardTitle, CardDescription, CardContent } from "@/components/ui/card"; +import { Badge } from "@/components/ui/badge"; +import { getAllJournals, bookCodeToSlug } from "@/lib/journals"; + +export default async function JournalsPage() { + const journals = await getAllJournals(); + + return ( +
+ + + ← Home + +

Journals

+

+ Personal diaries and journals of Śrīla Prabhupāda +

+ + +
+
+ {journals.map((journal) => ( + + + + {journal.title} + {journal.description && ( + {journal.description} + )} + + +
+ {journal.year && {journal.year}} + {journal.totalChapters > 0 && ( + + {journal.totalChapters} {journal.totalChapters === 1 ? "chapter" : "chapters"} + + )} +
+ + + + ))} +
+
+
+ ); +} diff --git a/app/other-documents/[slug]/[chapter]/page.tsx b/app/other-documents/[slug]/[chapter]/page.tsx new file mode 100644 index 00000000..35d11ec1 --- /dev/null +++ b/app/other-documents/[slug]/[chapter]/page.tsx @@ -0,0 +1,229 @@ +export const dynamic = "force-dynamic"; + +import Link from "next/link"; +import { notFound } from "next/navigation"; +import { + getOtherDocumentContent, + getOtherDocumentMeta, + getOtherDocumentSidebarItems, + slugToBookCode, + type OtherDocumentSidebarItem, +} from "@/lib/other-documents"; +import { PageHeader } from "@/components/page-header"; +import { Card, CardContent } from "@/components/ui/card"; +import { buttonVariants } from "@/components/ui/button"; +import { ChevronLeft, ChevronRight } from "lucide-react"; +import { cn } from "@/lib/utils"; +import { MobileNav, type MobileNavGroup } from "@/components/mobile-nav"; + +// --------------------------------------------------------------------------- +// Page +// --------------------------------------------------------------------------- + +export default async function OtherDocumentChapterPage({ + params, +}: { + params: Promise<{ slug: string; chapter: string }>; +}) { + const { slug, chapter: chapterSlug } = await params; + const bookCode = slugToBookCode(slug); + + const [meta, content, sidebarItems] = await Promise.all([ + getOtherDocumentMeta(bookCode), + getOtherDocumentContent(bookCode, chapterSlug), + getOtherDocumentSidebarItems(bookCode), + ]); + + if (!meta || !content) notFound(); + + const currentIdx = sidebarItems.findIndex( + (item) => item.slug === chapterSlug, + ); + const prev = currentIdx > 0 ? sidebarItems[currentIdx - 1] : null; + const next = + currentIdx < sidebarItems.length - 1 ? sidebarItems[currentIdx + 1] : null; + + return ( +
+ + + ← {meta.title} + +

+ Chapter {content.chapter}: + {content.title} +

+ + +
+ {/* Left sidebar */} + + + {/* Main content */} +
+ {/* Top navigation */} + + + {/* Content */} + + +
+ {content.paragraphs.map((paragraph, i) => ( +
+ ))} +
+ + + + {/* Bottom navigation */} + +
+
+
+ ); +} + +// --------------------------------------------------------------------------- +// Sidebar +// --------------------------------------------------------------------------- + +function ChapterSidebar({ + items, + currentSlug, + docSlug, +}: { + items: OtherDocumentSidebarItem[]; + currentSlug: string; + docSlug: string; +}) { + return ( + + ); +} + +// --------------------------------------------------------------------------- +// Prev / Next navigation +// --------------------------------------------------------------------------- + +function ChapterNav({ + prev, + next, + sidebarItems, + currentSlug, + docSlug, + docTitle, +}: { + prev: OtherDocumentSidebarItem | null; + next: OtherDocumentSidebarItem | null; + sidebarItems: OtherDocumentSidebarItem[]; + currentSlug: string; + docSlug: string; + docTitle: string; +}) { + const mobileNavGroups: MobileNavGroup[] = []; + + if (sidebarItems.length > 0) { + mobileNavGroups.push({ + title: "Chapters", + items: sidebarItems.map((i) => ({ + slug: i.slug, + label: i.label, + href: `/other-documents/${docSlug}/${i.slug}`, + type: i.type, + })), + }); + } + + return ( +
+
+ + {prev ? ( + + + + Ch. {prev.chapter} + + Previous + + ) : ( +
+ )} +
+ + {next ? ( + + + Ch. {next.chapter} + + Next + + + ) : ( +
+ )} +
+ ); +} diff --git a/app/other-documents/[slug]/page.tsx b/app/other-documents/[slug]/page.tsx new file mode 100644 index 00000000..64e9c437 --- /dev/null +++ b/app/other-documents/[slug]/page.tsx @@ -0,0 +1,75 @@ +export const dynamic = "force-dynamic"; + +import Link from "next/link"; +import { notFound } from "next/navigation"; +import { getOtherDocumentMeta, slugToBookCode } from "@/lib/other-documents"; +import { PageHeader } from "@/components/page-header"; +import { Card, CardContent } from "@/components/ui/card"; + +export default async function OtherDocumentIndexPage({ + params, +}: { + params: Promise<{ slug: string }>; +}) { + const { slug } = await params; + const bookCode = slugToBookCode(slug); + + const meta = await getOtherDocumentMeta(bookCode); + if (!meta) notFound(); + + return ( +
+ + + ← Other Documents + +

+ {meta.title} +

+

{meta.author}

+ + +
+ {/* Description */} + {meta.description && ( +

{meta.description}

+ )} + + {/* Chapters */} + {meta.chapters.length > 0 && ( +
+

+ Chapters +

+ + +
    + {meta.chapters.map((ch) => ( +
  • + +
    + + Chapter {ch.number} + +

    + {ch.title} +

    +
    + +
    • + ))} +
+ + +
+ )} +
+
+ ); +} diff --git a/app/other-documents/other-documents-grid.tsx b/app/other-documents/other-documents-grid.tsx new file mode 100644 index 00000000..b2902e3d --- /dev/null +++ b/app/other-documents/other-documents-grid.tsx @@ -0,0 +1,141 @@ +"use client"; + +import { useState, useMemo } from "react"; +import Link from "next/link"; +import { + Card, + CardHeader, + CardTitle, + CardDescription, + CardContent, +} from "@/components/ui/card"; +import { Badge } from "@/components/ui/badge"; +import { Input } from "@/components/ui/input"; +import { cn } from "@/lib/utils"; +import { Search } from "lucide-react"; + +export type OtherDocumentInfo = { + slug: string; + title: string; + description: string; + year: number | null; + totalChapters: number; + href: string; +}; + +/** + * Strip diacritics / accents so that "Krsna" matches "Kṛṣṇa", + * "Gita" matches "Gītā", "Srimad" matches "Śrīmad", etc. + */ +function normalize(text: string): string { + return ( + text + .toLowerCase() + .replace(/ṛ/g, "r") + .replace(/ṝ/g, "r") + .replace(/ṣ/g, "s") + .replace(/ś/g, "s") + .replace(/ṭ/g, "t") + .replace(/ḍ/g, "d") + .replace(/ṇ/g, "n") + .replace(/ñ/g, "n") + .replace(/ṅ/g, "n") + .replace(/ḥ/g, "h") + .replace(/ṁ/g, "m") + .replace(/ṃ/g, "m") + .replace(/ḷ/g, "l") + .normalize("NFD") + .replace(/[\u0300-\u036f]/g, "") + ); +} + +export function OtherDocumentsGrid({ + items, +}: { + items: OtherDocumentInfo[]; +}) { + const [query, setQuery] = useState(""); + + const normalizedItems = useMemo( + () => + items.map((item) => ({ + item, + searchText: normalize( + `${item.title} ${item.description} ${item.slug}`, + ), + })), + [items], + ); + + const filtered = useMemo(() => { + if (!query.trim()) return items; + const q = normalize(query.trim()); + return normalizedItems + .filter((entry) => entry.searchText.includes(q)) + .map((entry) => entry.item); + }, [items, normalizedItems, query]); + + return ( +
+ {/* Search input */} +
+ + setQuery(e.target.value)} + className="pl-9" + /> +
+ + {/* Results count when searching */} + {query.trim() && ( +

+ {filtered.length === 0 + ? "No documents found" + : `${filtered.length} document${filtered.length === 1 ? "" : "s"} found`} +

+ )} + + {/* Grid */} +
+ {filtered.map((item) => ( + + + + + {item.title} + + {item.description && ( + + {item.description} + + )} + + +
+ {item.year && ( + + {item.year} + + )} + {item.totalChapters > 0 && ( + + {item.totalChapters}{" "} + {item.totalChapters === 1 ? "chapter" : "chapters"} + + )} +
+ + + + ))} +
+
+ ); +} diff --git a/app/other-documents/page.tsx b/app/other-documents/page.tsx new file mode 100644 index 00000000..40dfe103 --- /dev/null +++ b/app/other-documents/page.tsx @@ -0,0 +1,42 @@ +export const dynamic = "force-dynamic"; + +import Link from "next/link"; +import { PageHeader } from "@/components/page-header"; +import { getAllOtherDocuments, bookCodeToSlug } from "@/lib/other-documents"; +import { OtherDocumentsGrid } from "./other-documents-grid"; + +export default async function OtherDocumentsPage() { + const documents = await getAllOtherDocuments(); + + const items = documents.map((doc) => ({ + slug: bookCodeToSlug(doc.book), + title: doc.title, + description: doc.description ?? "", + year: doc.year, + totalChapters: doc.totalChapters, + href: `/other-documents/${bookCodeToSlug(doc.book)}`, + })); + + return ( +
+ + + ← Home + +

+ Other Documents +

+

+ Miscellaneous documents, analyses, and writings by Śrīla Prabhupāda +

+ + +
+ +
+
+ ); +} diff --git a/app/page.tsx b/app/page.tsx index 2a6d7fc9..b70c7b22 100644 --- a/app/page.tsx +++ b/app/page.tsx @@ -9,7 +9,14 @@ import { import { Badge } from "@/components/ui/badge"; import { PageHeader } from "@/components/page-header"; import { Logo } from "@/components/logo"; -import { BookOpen, FileText, BookA, PenLine } from "lucide-react"; +import { + BookOpen, + FileText, + BookA, + PenLine, + FileStack, + NotebookPen, +} from "lucide-react"; const allowRtf = process.env.ALLOW_RTF === "true"; @@ -76,6 +83,35 @@ export default function Home() { + {/* Journals */} + + + +
+
+ +
+
+ + Journals + + + Personal diaries & journals + +
+
+ + +

+ Śrīla Prabhupāda's personal diaries — including the + historic Jaladuta Diary from his 1965 voyage to America and + The Beginning, his 1966 New York journal documenting the + founding of the Hare Kṛṣṇa movement. +

+ + + + {/* Glossary */} @@ -104,6 +140,35 @@ export default function Home() { + {/* Other Documents */} + + + +
+
+ +
+
+ + Other Documents + + + Miscellaneous analyses & writings + +
+
+ + +

+ Miscellaneous documents, analyses, and writings by Śrīla + Prabhupāda that don't fit neatly into the books or essays + collections — including structured analyses with verse + references. +

+ + + + {/* RTF Document Viewer — only visible when ALLOW_RTF=true */} {allowRtf && ( diff --git a/components/mobile-nav.tsx b/components/mobile-nav.tsx index 60c318d7..9a027fb6 100644 --- a/components/mobile-nav.tsx +++ b/components/mobile-nav.tsx @@ -87,11 +87,11 @@ export function MobileNav({ )}
- {group.items.map((item) => { + {group.items.map((item, ii) => { if (item.isParent) { return (
{item.label} @@ -103,7 +103,7 @@ export function MobileNav({ const isChild = item.parentChapter != null && !item.isParent; return ( - + ][\s\S]*?<\/script>/gi, ""); + clean = clean.replace(/][\s\S]*?<\/style>/gi, ""); + clean = clean.replace(/][\s\S]*?<\/iframe>/gi, ""); + clean = clean.replace( + /<\/?(?:script|style|iframe|object|embed|form|input|textarea|select|button)[^>]*>/gi, + "", + ); + clean = clean.replace(/\s+on\w+\s*=\s*(?:"[^"]*"|'[^']*'|[^\s>]+)/gi, ""); + clean = clean.replace(/href\s*=\s*"javascript:[^"]*"/gi, 'href=""'); + clean = clean.replace(/href\s*=\s*'javascript:[^']*'/gi, "href=''"); + return clean; +} + +/** Normalize paragraphs — handles both native arrays and stringified JSON. */ +function normalizeParagraphs(paragraphs: string[] | JsonValue): string[] { + let result: string[]; + + if (typeof paragraphs === "string") { + try { + const parsed = JSON.parse(paragraphs); + result = Array.isArray(parsed) ? parsed : [paragraphs]; + } catch { + result = [paragraphs]; + } + } else if (Array.isArray(paragraphs)) { + result = paragraphs as string[]; + } else { + result = []; + } + + return result.map(sanitizeHtml); +} + +// --------------------------------------------------------------------------- +// Local JSON helpers (fallback when DB is not seeded) +// --------------------------------------------------------------------------- + +function readLocalJson(filePath: string): T | null { + if (!fs.existsSync(filePath)) return null; + try { + return JSON.parse(fs.readFileSync(filePath, "utf-8")) as T; + } catch { + return null; + } +} + +function getLocalChapter(chapterNum: number): BkChapter | null { + const filePath = path.join( + BK_JSON_DIR, + "chapters", + String(chapterNum).padStart(2, "0") + ".json", + ); + return readLocalJson(filePath); +} + +// --------------------------------------------------------------------------- +// Book meta +// --------------------------------------------------------------------------- + +/** + * Book-level metadata (title, author, chapter list, etc.) + */ +export async function getBkMeta(): Promise { + const meta = await prisma.bookMeta.findUnique({ where: { book: BOOK } }); + if (meta) return meta.data as unknown as BkMeta; + + // Fallback to local JSON + const introPath = path.join(BK_JSON_DIR, "intro.json"); + return readLocalJson(introPath); +} + +// --------------------------------------------------------------------------- +// Chapters +// --------------------------------------------------------------------------- + +/** + * A numbered chapter (1–2). + */ +export async function getBkChapter( + chapterNum: number, +): Promise { + if (chapterNum < 1 || chapterNum > 2 || !Number.isInteger(chapterNum)) { + return null; + } + + const doc = await prisma.proseChapter.findUnique({ + where: { + book_volume_slug: { + book: BOOK, + volume: NO_VOLUME, + slug: String(chapterNum), + }, + }, + }); + + if (doc) { + return { + book: doc.book, + chapter: doc.chapter, + title: doc.title, + type: doc.type, + slug: doc.slug, + paragraphs: normalizeParagraphs(doc.paragraphs), + }; + } + + // Fallback to local JSON + const local = getLocalChapter(chapterNum); + if (!local) return null; + return { + ...local, + paragraphs: local.paragraphs.map(sanitizeHtml), + }; +} + +// --------------------------------------------------------------------------- +// Unified content accessor +// --------------------------------------------------------------------------- + +/** + * Get content by URL slug. + * + * "1" … "2" → chapter + */ +export async function getBkContent( + slug: string, +): Promise { + const num = Number(slug); + if (!Number.isNaN(num) && Number.isInteger(num) && num >= 1 && num <= 2) { + return getBkChapter(num); + } + + return null; +} + +// --------------------------------------------------------------------------- +// Sidebar navigation +// --------------------------------------------------------------------------- + +/** + * Build the full sidebar navigation list. + * + * Order: chapters 1–2 (no intro segments) + */ +export async function getBkSidebarItems(): Promise { + const [meta, allDocs] = await Promise.all([ + getBkMeta(), + prisma.proseChapter.findMany({ + where: { book: BOOK, volume: NO_VOLUME }, + orderBy: { order: "asc" }, + select: { + slug: true, + title: true, + type: true, + chapter: true, + order: true, + }, + }), + ]); + + // If we got DB results, build from those + if (meta && allDocs.length > 0) { + return buildSidebarFromDocs(meta); + } + + // Fallback: build from local JSON + return buildSidebarFromLocal(meta); +} + +function buildSidebarFromDocs(meta: BkMeta): BkSidebarItem[] { + const items: BkSidebarItem[] = []; + + // Chapters 1–2 (use meta.chapters for consistent title/ordering) + for (const ch of meta.chapters) { + items.push({ + slug: String(ch.number), + label: `Chapter ${ch.number}: ${ch.title}`, + type: "chapter", + chapter: ch.number, + }); + } + + return items; +} + +function buildSidebarFromLocal(meta: BkMeta | null): BkSidebarItem[] { + const items: BkSidebarItem[] = []; + + // Chapters from meta or scan local chapter JSONs + if (meta && meta.chapters) { + for (const ch of meta.chapters) { + items.push({ + slug: String(ch.number), + label: `Chapter ${ch.number}: ${ch.title}`, + type: "chapter", + chapter: ch.number, + }); + } + } else { + // Fallback: scan local chapter JSONs + const chaptersDir = path.join(BK_JSON_DIR, "chapters"); + if (fs.existsSync(chaptersDir)) { + const files = fs + .readdirSync(chaptersDir) + .filter((f) => f.endsWith(".json")) + .sort(); + for (const file of files) { + const ch = readLocalJson(path.join(chaptersDir, file)); + if (ch) { + const num = ch.chapter ?? parseInt(file.replace(".json", ""), 10); + items.push({ + slug: String(num), + label: `Chapter ${num}: ${ch.title}`, + type: "chapter", + chapter: num, + }); + } + } + } + } + + return items; +} diff --git a/lib/cbhc.ts b/lib/cbhc.ts new file mode 100644 index 00000000..487d0c78 --- /dev/null +++ b/lib/cbhc.ts @@ -0,0 +1,299 @@ +/** + * Data access layer for Śrī Caitanya-bhāgavata Commentary (CBHC). + * + * Reads from MongoDB via Prisma, same as the rest of the app. + * CBHC is a simple prose book with no volume structure and no intro segments — + * just 4 chapters directly. We use volume=0 as the sentinel for "no volumes". + * + * When the DB isn't seeded yet, falls back to reading local JSON files + * from lib/jsons/books/cbhc/. + * + * Collections used: + * - book_meta (via Prisma BookMeta model) + * - prose_chapters (via Prisma ProseChapter model) + */ + +import { prisma } from "@/lib/db"; +import type { JsonValue } from "@prisma/client/runtime/library"; +import fs from "fs"; +import path from "path"; + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +/** Volume sentinel for books without volumes. */ +const NO_VOLUME = 0; + +const BOOK = "cbhc"; + +const CBHC_JSON_DIR = path.join(process.cwd(), "lib", "jsons", "books", "cbhc"); + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export type CbhcMeta = { + book: string; + title: string; + author: string; + year?: number | null; + description: string; + totalChapters: number; + chapters: CbhcChapterEntry[]; +}; + +export type CbhcChapterEntry = { + number: number; + title: string; +}; + +export type CbhcChapter = { + book: string; + chapter: number | null; + title: string; + type: string; + slug?: string; + paragraphs: string[]; +}; + +/** Union type used in sidebar / table-of-contents navigation. */ +export type CbhcSidebarItem = { + /** URL-safe slug: "1", "2", "3", "4" */ + slug: string; + /** Human-readable label for the sidebar */ + label: string; + /** "chapter" */ + type: string; + /** Chapter number */ + chapter: number | null; +}; + +// --------------------------------------------------------------------------- +// HTML sanitisation (mirrors lib/api.ts) +// --------------------------------------------------------------------------- + +function sanitizeHtml(html: string): string { + let clean = html.replace(/][\s\S]*?<\/script>/gi, ""); + clean = clean.replace(/][\s\S]*?<\/style>/gi, ""); + clean = clean.replace(/][\s\S]*?<\/iframe>/gi, ""); + clean = clean.replace( + /<\/?(?:script|style|iframe|object|embed|form|input|textarea|select|button)[^>]*>/gi, + "", + ); + clean = clean.replace(/\s+on\w+\s*=\s*(?:"[^"]*"|'[^']*'|[^\s>]+)/gi, ""); + clean = clean.replace(/href\s*=\s*"javascript:[^"]*"/gi, 'href=""'); + clean = clean.replace(/href\s*=\s*'javascript:[^']*'/gi, "href=''"); + return clean; +} + +/** Normalize paragraphs — handles both native arrays and stringified JSON. */ +function normalizeParagraphs(paragraphs: string[] | JsonValue): string[] { + let result: string[]; + + if (typeof paragraphs === "string") { + try { + const parsed = JSON.parse(paragraphs); + result = Array.isArray(parsed) ? parsed : [paragraphs]; + } catch { + result = [paragraphs]; + } + } else if (Array.isArray(paragraphs)) { + result = paragraphs as string[]; + } else { + result = []; + } + + return result.map(sanitizeHtml); +} + +// --------------------------------------------------------------------------- +// Local JSON helpers (fallback when DB is not seeded) +// --------------------------------------------------------------------------- + +function readLocalJson(filePath: string): T | null { + if (!fs.existsSync(filePath)) return null; + try { + return JSON.parse(fs.readFileSync(filePath, "utf-8")) as T; + } catch { + return null; + } +} + +function getLocalChapter(chapterNum: number): CbhcChapter | null { + const filePath = path.join( + CBHC_JSON_DIR, + "chapters", + String(chapterNum).padStart(2, "0") + ".json", + ); + return readLocalJson(filePath); +} + +// --------------------------------------------------------------------------- +// Book meta +// --------------------------------------------------------------------------- + +/** + * Book-level metadata (title, author, chapter list, etc.) + */ +export async function getCbhcMeta(): Promise { + const meta = await prisma.bookMeta.findUnique({ where: { book: BOOK } }); + if (meta) return meta.data as unknown as CbhcMeta; + + // Fallback to local JSON + const introPath = path.join(CBHC_JSON_DIR, "intro.json"); + return readLocalJson(introPath); +} + +// --------------------------------------------------------------------------- +// Chapters +// --------------------------------------------------------------------------- + +/** + * A numbered chapter (1–4). + */ +export async function getCbhcChapter( + chapterNum: number, +): Promise { + if (chapterNum < 1 || chapterNum > 4 || !Number.isInteger(chapterNum)) { + return null; + } + + const doc = await prisma.proseChapter.findUnique({ + where: { + book_volume_slug: { + book: BOOK, + volume: NO_VOLUME, + slug: String(chapterNum), + }, + }, + }); + + if (doc) { + return { + book: doc.book, + chapter: doc.chapter, + title: doc.title, + type: doc.type, + slug: doc.slug, + paragraphs: normalizeParagraphs(doc.paragraphs), + }; + } + + // Fallback to local JSON + const local = getLocalChapter(chapterNum); + if (!local) return null; + return { + ...local, + paragraphs: local.paragraphs.map(sanitizeHtml), + }; +} + +// --------------------------------------------------------------------------- +// Unified content accessor +// --------------------------------------------------------------------------- + +/** + * Get content by URL slug. + * + * "1" … "4" → chapter + */ +export async function getCbhcContent( + slug: string, +): Promise { + const num = Number(slug); + if (!Number.isNaN(num) && Number.isInteger(num) && num >= 1 && num <= 4) { + return getCbhcChapter(num); + } + + return null; +} + +// --------------------------------------------------------------------------- +// Sidebar navigation +// --------------------------------------------------------------------------- + +/** + * Build the full sidebar navigation list. + * + * Order: chapters 1–4 (no intro segments) + */ +export async function getCbhcSidebarItems(): Promise { + const [meta, allDocs] = await Promise.all([ + getCbhcMeta(), + prisma.proseChapter.findMany({ + where: { book: BOOK, volume: NO_VOLUME }, + orderBy: { order: "asc" }, + select: { + slug: true, + title: true, + type: true, + chapter: true, + order: true, + }, + }), + ]); + + // If we got DB results, build from those + if (meta && allDocs.length > 0) { + return buildSidebarFromDocs(meta); + } + + // Fallback: build from local JSON + return buildSidebarFromLocal(meta); +} + +function buildSidebarFromDocs(meta: CbhcMeta): CbhcSidebarItem[] { + const items: CbhcSidebarItem[] = []; + + // Chapters 1–4 (use meta.chapters for consistent title/ordering) + for (const ch of meta.chapters) { + items.push({ + slug: String(ch.number), + label: `Chapter ${ch.number}: ${ch.title}`, + type: "chapter", + chapter: ch.number, + }); + } + + return items; +} + +function buildSidebarFromLocal(meta: CbhcMeta | null): CbhcSidebarItem[] { + const items: CbhcSidebarItem[] = []; + + // Chapters from meta or scan local chapter JSONs + if (meta && meta.chapters) { + for (const ch of meta.chapters) { + items.push({ + slug: String(ch.number), + label: `Chapter ${ch.number}: ${ch.title}`, + type: "chapter", + chapter: ch.number, + }); + } + } else { + // Fallback: scan local chapter JSONs + const chaptersDir = path.join(CBHC_JSON_DIR, "chapters"); + if (fs.existsSync(chaptersDir)) { + const files = fs + .readdirSync(chaptersDir) + .filter((f) => f.endsWith(".json")) + .sort(); + for (const file of files) { + const ch = readLocalJson(path.join(chaptersDir, file)); + if (ch) { + const num = ch.chapter ?? parseInt(file.replace(".json", ""), 10); + items.push({ + slug: String(num), + label: `Chapter ${num}: ${ch.title}`, + type: "chapter", + chapter: num, + }); + } + } + } + } + + return items; +} diff --git a/lib/journals.ts b/lib/journals.ts new file mode 100644 index 00000000..ccdb8cfd --- /dev/null +++ b/lib/journals.ts @@ -0,0 +1,289 @@ +/** + * Data access layer for Journals. + * + * Uses dedicated Prisma models: + * - JournalMeta (collection: journal_meta) + * - JournalChapter (collection: journal_chapters) + */ + +import { prisma } from "@/lib/db"; +import type { JsonValue } from "@prisma/client/runtime/library"; +import fs from "fs"; +import path from "path"; + +const LOCAL_JSON_DIR = path.join(process.cwd(), "lib", "jsons", "journals"); + +// Types +export type JournalMetaData = { + book: string; + title: string; + author: string; + year: number | null; + description: string | null; + totalChapters: number; + chapters: JournalChapterEntry[]; +}; + +export type JournalSubChapter = { slug: string; title: string }; + +export type JournalMonthGroup = { + title: string; + subChapters: JournalSubChapter[]; +}; + +export type JournalChapterGroup = { + title: string; + subChapters: (JournalSubChapter | JournalMonthGroup)[]; +}; + +export type JournalChapterEntry = + | JournalSubChapter + | JournalMonthGroup + | JournalChapterGroup; + +export type JournalChapterData = { + book: string; + chapter: number; + title: string; + type: string; + paragraphs: string[]; +}; + +export type JournalSummary = { + book: string; + title: string; + author: string; + year: number | null; + description: string | null; + totalChapters: number; +}; + +export type JournalSidebarItem = { + slug: string; + label: string; + type: string; + chapter: number; +}; + +// HTML sanitisation +function sanitizeHtml(html: string): string { + let clean = html.replace(/][\s\S]*?<\/script>/gi, ""); + clean = clean.replace(/][\s\S]*?<\/style>/gi, ""); + clean = clean.replace(/][\s\S]*?<\/iframe>/gi, ""); + clean = clean.replace( + /<\/?(?:script|style|iframe|object|embed|form|input|textarea|select|button)[^>]*>/gi, + "", + ); + clean = clean.replace(/\s+on\w+\s*=\s*(?:"[^"]*"|'[^']*'|[^\s>]+)/gi, ""); + clean = clean.replace(/href\s*=\s*"javascript:[^"]*"/gi, 'href=""'); + clean = clean.replace(/href\s*=\s*'javascript:[^']*'/gi, "href=''"); + return clean; +} + +function normalizeParagraphs(paragraphs: string[] | JsonValue): string[] { + let result: string[]; + if (typeof paragraphs === "string") { + try { + const parsed = JSON.parse(paragraphs); + result = Array.isArray(parsed) ? parsed : [paragraphs]; + } catch { + result = [paragraphs]; + } + } else if (Array.isArray(paragraphs)) { + result = paragraphs as string[]; + } else { + result = []; + } + return result.map(sanitizeHtml); +} + +// Local JSON fallback helpers +function findLocalDirForBook(bookCode: string): string | null { + if (!fs.existsSync(LOCAL_JSON_DIR)) return null; + const entries = fs.readdirSync(LOCAL_JSON_DIR); + for (const entry of entries) { + const introPath = path.join(LOCAL_JSON_DIR, entry, "intro.json"); + if (fs.existsSync(introPath)) { + try { + const intro = JSON.parse(fs.readFileSync(introPath, "utf-8")); + if (intro.book === bookCode) return entry; + } catch { + /* skip */ + } + } + } + return null; +} + +function readLocalJson(filePath: string): T | null { + try { + if (!fs.existsSync(filePath)) return null; + return JSON.parse(fs.readFileSync(filePath, "utf-8")) as T; + } catch { + return null; + } +} + +function getAllLocalMetas(): JournalMetaData[] { + if (!fs.existsSync(LOCAL_JSON_DIR)) return []; + const entries = fs.readdirSync(LOCAL_JSON_DIR).sort(); + const metas: JournalMetaData[] = []; + for (const entry of entries) { + const introPath = path.join(LOCAL_JSON_DIR, entry, "intro.json"); + const intro = readLocalJson(introPath); + if (intro) metas.push(intro); + } + return metas; +} + +function getLocalMeta(bookCode: string): JournalMetaData | null { + const dir = findLocalDirForBook(bookCode); + if (!dir) return null; + return readLocalJson( + path.join(LOCAL_JSON_DIR, dir, "intro.json"), + ); +} + +function getLocalChapterBySlug( + bookCode: string, + slug: string, +): JournalChapterData | null { + const dir = findLocalDirForBook(bookCode); + if (!dir) return null; + // Try slug directly as filename (e.g. "preface.json", "apr-3.json") + const filePath = path.join(LOCAL_JSON_DIR, dir, "chapters", `${slug}.json`); + const data = readLocalJson(filePath); + if (!data) return null; + return { ...data, paragraphs: data.paragraphs.map(sanitizeHtml) }; +} + +// Public API +export async function getAllJournals(): Promise { + try { + const results = await prisma.journalMeta.findMany({ + orderBy: { book: "asc" }, + }); + if (results.length > 0) { + return results.map((row) => { + const data = row.data as unknown as JournalMetaData; + return { + book: row.book, + title: data.title, + author: data.author, + year: data.year ?? null, + description: data.description ?? null, + totalChapters: data.totalChapters, + }; + }); + } + } catch { + /* fall through */ + } + const localMetas = getAllLocalMetas(); + return localMetas.map((meta) => ({ + book: meta.book, + title: meta.title, + author: meta.author, + year: meta.year ?? null, + description: meta.description ?? null, + totalChapters: meta.totalChapters, + })); +} + +export async function getJournalMeta( + bookCode: string, +): Promise { + try { + const row = await prisma.journalMeta.findUnique({ + where: { book: bookCode }, + }); + if (row) return row.data as unknown as JournalMetaData; + } catch { + /* fall through */ + } + return getLocalMeta(bookCode); +} + +export async function getJournalChapterBySlug( + bookCode: string, + slug: string, +): Promise { + if (!slug) return null; + try { + const doc = await prisma.journalChapter.findUnique({ + where: { book_slug: { book: bookCode, slug } }, + }); + if (doc) + return { + book: doc.book, + chapter: doc.chapter, + title: doc.title, + type: doc.type, + paragraphs: normalizeParagraphs(doc.paragraphs), + }; + } catch { + /* fall through */ + } + return getLocalChapterBySlug(bookCode, slug); +} + +export async function getJournalContent( + bookCode: string, + slug: string, +): Promise { + return getJournalChapterBySlug(bookCode, slug); +} + +function isSubChapter(ch: JournalChapterEntry): ch is JournalSubChapter { + return "slug" in ch && !("subChapters" in ch); +} + +function hasSubChapters( + ch: JournalChapterEntry, +): ch is JournalMonthGroup | JournalChapterGroup { + return "subChapters" in ch && Array.isArray(ch.subChapters); +} + +export async function getJournalSidebarItems( + bookCode: string, +): Promise { + const meta = await getJournalMeta(bookCode); + if (!meta) return []; + + const items: JournalSidebarItem[] = []; + + function addEntry(entry: JournalChapterEntry) { + if (isSubChapter(entry)) { + items.push({ + slug: entry.slug, + label: entry.title, + type: "chapter", + chapter: 0, + }); + } else if (hasSubChapters(entry)) { + // Group heading (place name or month name) + items.push({ + slug: "", + label: entry.title, + type: "heading", + chapter: 0, + }); + for (const sc of entry.subChapters) { + addEntry(sc); + } + } + } + + for (const ch of meta.chapters) { + addEntry(ch); + } + + return items; +} + +export function bookCodeToSlug(bookCode: string): string { + return bookCode; +} +export function slugToBookCode(slug: string): string { + return slug; +} diff --git a/lib/mog.ts b/lib/mog.ts new file mode 100644 index 00000000..6b35631f --- /dev/null +++ b/lib/mog.ts @@ -0,0 +1,430 @@ +/** + * Data access layer for Message of Godhead (MOG). + * + * Reads from MongoDB via Prisma, same as the rest of the app. + * MOG is a prose book with no volume structure — + * we use volume=0 as the sentinel for "no volumes". + * + * When the DB isn't seeded yet, falls back to reading local JSON files + * from lib/jsons/books/mog/. + * + * Collections used: + * - book_meta (via Prisma BookMeta model) + * - prose_chapters (via Prisma ProseChapter model) + */ + +import { prisma } from "@/lib/db"; +import type { JsonValue } from "@prisma/client/runtime/library"; +import fs from "fs"; +import path from "path"; + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +/** Volume sentinel for books without volumes. */ +const NO_VOLUME = 0; + +const BOOK = "mog"; + +const MOG_JSON_DIR = path.join(process.cwd(), "lib", "jsons", "books", "mog"); + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export type MogMeta = { + book: string; + title: string; + author: string; + year?: number; + description: string; + totalChapters: number; + chapters: MogChapterEntry[]; +}; + +export type MogChapterEntry = { + number: number; + title: string; +}; + +export type MogIntroSegment = { + slug: string; + title: string; + type: string; + book: string; + order: number; + paragraphs: string[]; +}; + +export type MogChapter = { + book: string; + chapter: number | null; + title: string; + type: string; + slug?: string; + paragraphs: string[]; +}; + +/** Union type used in sidebar / table-of-contents navigation. */ +export type MogSidebarItem = { + /** URL-safe slug: "intro-front-pages", "1", "2", "3" */ + slug: string; + /** Human-readable label for the sidebar */ + label: string; + /** "intro-segment" | "chapter" */ + type: string; + /** Chapter number, or null for non-chapter items */ + chapter: number | null; +}; + +// --------------------------------------------------------------------------- +// HTML sanitisation (mirrors lib/api.ts) +// --------------------------------------------------------------------------- + +function sanitizeHtml(html: string): string { + let clean = html.replace(/][\s\S]*?<\/script>/gi, ""); + clean = clean.replace(/][\s\S]*?<\/style>/gi, ""); + clean = clean.replace(/][\s\S]*?<\/iframe>/gi, ""); + clean = clean.replace( + /<\/?(?:script|style|iframe|object|embed|form|input|textarea|select|button)[^>]*>/gi, + "", + ); + clean = clean.replace(/\s+on\w+\s*=\s*(?:"[^"]*"|'[^']*'|[^\s>]+)/gi, ""); + clean = clean.replace(/href\s*=\s*"javascript:[^"]*"/gi, 'href=""'); + clean = clean.replace(/href\s*=\s*'javascript:[^']*'/gi, "href=''"); + return clean; +} + +/** Normalize paragraphs — handles both native arrays and stringified JSON. */ +function normalizeParagraphs(paragraphs: string[] | JsonValue): string[] { + let result: string[]; + + if (typeof paragraphs === "string") { + try { + const parsed = JSON.parse(paragraphs); + result = Array.isArray(parsed) ? parsed : [paragraphs]; + } catch { + result = [paragraphs]; + } + } else if (Array.isArray(paragraphs)) { + result = paragraphs as string[]; + } else { + result = []; + } + + return result.map(sanitizeHtml); +} + +// --------------------------------------------------------------------------- +// Local JSON helpers (fallback when DB is not seeded) +// --------------------------------------------------------------------------- + +function readLocalJson(filePath: string): T | null { + if (!fs.existsSync(filePath)) return null; + try { + return JSON.parse(fs.readFileSync(filePath, "utf-8")) as T; + } catch { + return null; + } +} + +function getLocalIntroSegments(): MogIntroSegment[] { + const introDir = path.join(MOG_JSON_DIR, "intro"); + if (!fs.existsSync(introDir)) return []; + + const results: MogIntroSegment[] = []; + const files = ["front-pages.json"]; + for (const file of files) { + const filePath = path.join(introDir, file); + const seg = readLocalJson(filePath); + if (seg) results.push(seg); + } + return results.sort((a, b) => (a.order ?? 0) - (b.order ?? 0)); +} + +function getLocalIntroSegment(rawSlug: string): MogIntroSegment | null { + const filePath = path.join(MOG_JSON_DIR, "intro", `${rawSlug}.json`); + return readLocalJson(filePath); +} + +function getLocalChapter(chapterNum: number): MogChapter | null { + const filePath = path.join( + MOG_JSON_DIR, + "chapters", + String(chapterNum).padStart(2, "0") + ".json", + ); + return readLocalJson(filePath); +} + +// --------------------------------------------------------------------------- +// Book meta +// --------------------------------------------------------------------------- + +/** + * Book-level metadata (title, author, chapter list, etc.) + */ +export async function getMogMeta(): Promise { + const meta = await prisma.bookMeta.findUnique({ where: { book: BOOK } }); + if (meta) return meta.data as unknown as MogMeta; + + // Fallback to local JSON + const introPath = path.join(MOG_JSON_DIR, "intro.json"); + return readLocalJson(introPath); +} + +// --------------------------------------------------------------------------- +// Intro segments +// --------------------------------------------------------------------------- + +/** + * All intro segments, ordered by their `order` field. + */ +export async function getMogIntroSegments(): Promise { + const results = await prisma.proseChapter.findMany({ + where: { book: BOOK, volume: NO_VOLUME, type: "intro-segment" }, + orderBy: { order: "asc" }, + }); + + if (results.length > 0) { + return results.map((doc) => ({ + slug: doc.slug, + title: doc.title, + type: doc.type, + book: doc.book, + order: doc.order, + paragraphs: normalizeParagraphs(doc.paragraphs), + })); + } + + // Fallback to local JSON + return getLocalIntroSegments(); +} + +/** + * A single intro segment by slug. + * Accepts both "intro-front-pages" (URL slug) and "front-pages" (raw slug). + */ +export async function getMogIntroSegment( + slug: string, +): Promise { + // DB stores with "intro-" prefix; accept either form + const dbSlug = slug.startsWith("intro-") ? slug : `intro-${slug}`; + const rawSlug = slug.startsWith("intro-") ? slug.replace(/^intro-/, "") : slug; + + const doc = await prisma.proseChapter.findUnique({ + where: { + book_volume_slug: { book: BOOK, volume: NO_VOLUME, slug: dbSlug }, + }, + }); + + if (doc) { + return { + slug: doc.slug, + title: doc.title, + type: doc.type, + book: doc.book, + order: doc.order, + paragraphs: normalizeParagraphs(doc.paragraphs), + }; + } + + // Fallback to local JSON + const local = getLocalIntroSegment(rawSlug); + if (!local) return null; + return { + ...local, + slug: dbSlug, + paragraphs: local.paragraphs.map(sanitizeHtml), + }; +} + +// --------------------------------------------------------------------------- +// Chapters +// --------------------------------------------------------------------------- + +/** + * A numbered chapter (1–3). + */ +export async function getMogChapter( + chapterNum: number, +): Promise { + if (chapterNum < 1 || chapterNum > 3 || !Number.isInteger(chapterNum)) { + return null; + } + + const doc = await prisma.proseChapter.findUnique({ + where: { + book_volume_slug: { + book: BOOK, + volume: NO_VOLUME, + slug: String(chapterNum), + }, + }, + }); + + if (doc) { + return { + book: doc.book, + chapter: doc.chapter, + title: doc.title, + type: doc.type, + slug: doc.slug, + paragraphs: normalizeParagraphs(doc.paragraphs), + }; + } + + // Fallback to local JSON + const local = getLocalChapter(chapterNum); + if (!local) return null; + return { + ...local, + paragraphs: local.paragraphs.map(sanitizeHtml), + }; +} + +// --------------------------------------------------------------------------- +// Unified content accessor +// --------------------------------------------------------------------------- + +/** + * Get content by URL slug. + * + * "intro-front-pages" → intro segment + * "1" … "3" → chapter + */ +export async function getMogContent( + slug: string, +): Promise { + if (slug.startsWith("intro-")) { + return getMogIntroSegment(slug); + } + + const num = Number(slug); + if (!Number.isNaN(num) && Number.isInteger(num) && num >= 1 && num <= 3) { + return getMogChapter(num); + } + + return null; +} + +// --------------------------------------------------------------------------- +// Sidebar navigation +// --------------------------------------------------------------------------- + +/** + * Build the full sidebar navigation list. + * + * Order: front-pages → chapters 1–3 + */ +export async function getMogSidebarItems(): Promise { + const [meta, allDocs] = await Promise.all([ + getMogMeta(), + prisma.proseChapter.findMany({ + where: { book: BOOK, volume: NO_VOLUME }, + orderBy: { order: "asc" }, + select: { + slug: true, + title: true, + type: true, + chapter: true, + order: true, + }, + }), + ]); + + // If we got DB results, build from those + if (meta && allDocs.length > 0) { + return buildSidebarFromDocs(meta, allDocs); + } + + // Fallback: build from local JSON + return buildSidebarFromLocal(meta); +} + +function buildSidebarFromDocs( + meta: MogMeta, + allDocs: { + slug: string; + title: string; + type: string; + chapter: number | null; + order: number | null; + }[], +): MogSidebarItem[] { + const items: MogSidebarItem[] = []; + + // Intro segments (front-pages) + for (const doc of allDocs) { + if (doc.type === "intro-segment") { + items.push({ + slug: doc.slug, + label: doc.title, + type: "intro-segment", + chapter: null, + }); + } + } + + // Chapters 1–3 (use meta.chapters for consistent title/ordering) + for (const ch of meta.chapters) { + items.push({ + slug: String(ch.number), + label: `${ch.number}. ${ch.title}`, + type: "chapter", + chapter: ch.number, + }); + } + + return items; +} + +function buildSidebarFromLocal(meta: MogMeta | null): MogSidebarItem[] { + const items: MogSidebarItem[] = []; + + // Local intro segments (front matter) + const introSegs = getLocalIntroSegments(); + for (const seg of introSegs) { + items.push({ + slug: `intro-${seg.slug}`, + label: seg.title, + type: "intro-segment", + chapter: null, + }); + } + + // Chapters from meta or scan local chapter JSONs + if (meta && meta.chapters) { + for (const ch of meta.chapters) { + items.push({ + slug: String(ch.number), + label: `${ch.number}. ${ch.title}`, + type: "chapter", + chapter: ch.number, + }); + } + } else { + // Fallback: scan local chapter JSONs + const chaptersDir = path.join(MOG_JSON_DIR, "chapters"); + if (fs.existsSync(chaptersDir)) { + const files = fs + .readdirSync(chaptersDir) + .filter((f) => f.endsWith(".json")) + .sort(); + for (const file of files) { + const ch = readLocalJson(path.join(chaptersDir, file)); + if (ch) { + const num = ch.chapter ?? parseInt(file.replace(".json", ""), 10); + items.push({ + slug: String(num), + label: `${num}. ${ch.title}`, + type: "chapter", + chapter: num, + }); + } + } + } + } + + return items; +} diff --git a/lib/other-documents.ts b/lib/other-documents.ts new file mode 100644 index 00000000..9b37f56b --- /dev/null +++ b/lib/other-documents.ts @@ -0,0 +1,327 @@ +/** + * Data access layer for Other Documents. + * + * Uses dedicated Prisma models: + * - OtherDocumentMeta (collection: other_document_meta) + * - OtherDocumentChapter (collection: other_document_chapters) + * + * Falls back to reading local JSON files from + * lib/jsons/other-documents// when the database is unavailable. + * + * Book codes are plain slugs (e.g. "analysis-bg", "real-peace-formula"). + */ + +import { prisma } from "@/lib/db"; +import type { JsonValue } from "@prisma/client/runtime/library"; +import fs from "fs"; +import path from "path"; + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +const LOCAL_JSON_DIR = path.join( + process.cwd(), + "lib", + "jsons", + "other-documents", +); + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export type OtherDocumentMetaData = { + book: string; + title: string; + author: string; + year: number | null; + description: string | null; + totalChapters: number; + chapters: OtherDocumentChapterEntry[]; +}; + +export type OtherDocumentChapterEntry = { + number: number; + title: string; +}; + +export type OtherDocumentChapterData = { + book: string; + chapter: number; + title: string; + type: string; + paragraphs: string[]; +}; + +/** Lightweight summary for the top-level index page. */ +export type OtherDocumentSummary = { + book: string; + title: string; + author: string; + year: number | null; + description: string | null; + totalChapters: number; +}; + +/** Used in sidebar / table-of-contents navigation. */ +export type OtherDocumentSidebarItem = { + slug: string; + label: string; + type: string; + chapter: number; +}; + +// --------------------------------------------------------------------------- +// HTML sanitisation (same pattern used across all readers) +// --------------------------------------------------------------------------- + +function sanitizeHtml(html: string): string { + let clean = html.replace(/][\s\S]*?<\/script>/gi, ""); + clean = clean.replace(/][\s\S]*?<\/style>/gi, ""); + clean = clean.replace(/][\s\S]*?<\/iframe>/gi, ""); + clean = clean.replace( + /<\/?(?:script|style|iframe|object|embed|form|input|textarea|select|button)[^>]*>/gi, + "", + ); + clean = clean.replace(/\s+on\w+\s*=\s*(?:"[^"]*"|'[^']*'|[^\s>]+)/gi, ""); + clean = clean.replace(/href\s*=\s*"javascript:[^"]*"/gi, 'href=""'); + clean = clean.replace(/href\s*=\s*'javascript:[^']*'/gi, "href=''"); + return clean; +} + +function normalizeParagraphs(paragraphs: string[] | JsonValue): string[] { + let result: string[]; + + if (typeof paragraphs === "string") { + try { + const parsed = JSON.parse(paragraphs); + result = Array.isArray(parsed) ? parsed : [paragraphs]; + } catch { + result = [paragraphs]; + } + } else if (Array.isArray(paragraphs)) { + result = paragraphs as string[]; + } else { + result = []; + } + + return result.map(sanitizeHtml); +} + +// --------------------------------------------------------------------------- +// Helpers: local JSON fallback +// --------------------------------------------------------------------------- + +function findLocalDirForBook(bookCode: string): string | null { + if (!fs.existsSync(LOCAL_JSON_DIR)) return null; + + const entries = fs.readdirSync(LOCAL_JSON_DIR); + for (const entry of entries) { + const introPath = path.join(LOCAL_JSON_DIR, entry, "intro.json"); + if (fs.existsSync(introPath)) { + try { + const intro = JSON.parse(fs.readFileSync(introPath, "utf-8")); + if (intro.book === bookCode) return entry; + } catch { + // skip + } + } + } + return null; +} + +function readLocalJson(filePath: string): T | null { + try { + if (!fs.existsSync(filePath)) return null; + return JSON.parse(fs.readFileSync(filePath, "utf-8")) as T; + } catch { + return null; + } +} + +function getAllLocalMetas(): OtherDocumentMetaData[] { + if (!fs.existsSync(LOCAL_JSON_DIR)) return []; + + const entries = fs.readdirSync(LOCAL_JSON_DIR).sort(); + const metas: OtherDocumentMetaData[] = []; + + for (const entry of entries) { + const introPath = path.join(LOCAL_JSON_DIR, entry, "intro.json"); + const intro = readLocalJson(introPath); + if (intro) metas.push(intro); + } + + return metas; +} + +function getLocalMeta(bookCode: string): OtherDocumentMetaData | null { + const dir = findLocalDirForBook(bookCode); + if (!dir) return null; + return readLocalJson( + path.join(LOCAL_JSON_DIR, dir, "intro.json"), + ); +} + +function getLocalChapter( + bookCode: string, + chapterNum: number, +): OtherDocumentChapterData | null { + const dir = findLocalDirForBook(bookCode); + if (!dir) return null; + + const filePath = path.join( + LOCAL_JSON_DIR, + dir, + "chapters", + String(chapterNum).padStart(2, "0") + ".json", + ); + + const data = readLocalJson(filePath); + if (!data) return null; + + return { + ...data, + paragraphs: data.paragraphs.map(sanitizeHtml), + }; +} + +// --------------------------------------------------------------------------- +// Public API +// --------------------------------------------------------------------------- + +/** + * List all other documents (lightweight summaries for the index page). + */ +export async function getAllOtherDocuments(): Promise { + try { + const results = await prisma.otherDocumentMeta.findMany({ + orderBy: { book: "asc" }, + }); + + if (results.length > 0) { + return results.map((row) => { + const data = row.data as unknown as OtherDocumentMetaData; + return { + book: row.book, + title: data.title, + author: data.author, + year: data.year ?? null, + description: data.description ?? null, + totalChapters: data.totalChapters, + }; + }); + } + } catch { + // fall through to local JSON + } + + const localMetas = getAllLocalMetas(); + return localMetas.map((meta) => ({ + book: meta.book, + title: meta.title, + author: meta.author, + year: meta.year ?? null, + description: meta.description ?? null, + totalChapters: meta.totalChapters, + })); +} + +/** + * Get metadata for a single other document by book code. + */ +export async function getOtherDocumentMeta( + bookCode: string, +): Promise { + try { + const row = await prisma.otherDocumentMeta.findUnique({ + where: { book: bookCode }, + }); + + if (row) { + return row.data as unknown as OtherDocumentMetaData; + } + } catch { + // fall through to local JSON + } + + return getLocalMeta(bookCode); +} + +/** + * Get a single chapter from an other document. + */ +export async function getOtherDocumentChapter( + bookCode: string, + chapterNum: number, +): Promise { + if (chapterNum < 1 || !Number.isInteger(chapterNum)) return null; + + try { + const doc = await prisma.otherDocumentChapter.findUnique({ + where: { + book_slug: { + book: bookCode, + slug: String(chapterNum), + }, + }, + }); + + if (doc) { + return { + book: doc.book, + chapter: doc.chapter, + title: doc.title, + type: doc.type, + paragraphs: normalizeParagraphs(doc.paragraphs), + }; + } + } catch { + // fall through to local JSON + } + + return getLocalChapter(bookCode, chapterNum); +} + +/** + * Unified content accessor — get content by URL slug. + */ +export async function getOtherDocumentContent( + bookCode: string, + slug: string, +): Promise { + const num = Number(slug); + if (Number.isNaN(num) || !Number.isInteger(num) || num < 1) return null; + return getOtherDocumentChapter(bookCode, num); +} + +/** + * Build the sidebar navigation list for an other document. + */ +export async function getOtherDocumentSidebarItems( + bookCode: string, +): Promise { + const meta = await getOtherDocumentMeta(bookCode); + if (!meta) return []; + + return meta.chapters.map((ch) => ({ + slug: String(ch.number), + label: `${ch.number}. ${ch.title}`, + type: "chapter", + chapter: ch.number, + })); +} + +/** + * Derive the URL slug from a book code. + */ +export function bookCodeToSlug(bookCode: string): string { + return bookCode; +} + +/** + * Derive the book code from a URL slug. + */ +export function slugToBookCode(slug: string): string { + return slug; +} diff --git a/lib/pk.ts b/lib/pk.ts new file mode 100644 index 00000000..ae67b2ab --- /dev/null +++ b/lib/pk.ts @@ -0,0 +1,306 @@ +/** + * Data access layer for Prayers of King Kulaśekhara (PK). + * + * Reads from MongoDB via Prisma, same as the rest of the app. + * PK is a prose book with no volume structure and no intro segments — + * we use volume=0 as the sentinel for "no volumes". + * + * When the DB isn't seeded yet, falls back to reading local JSON files + * from lib/jsons/books/pk/. + * + * Collections used: + * - book_meta (via Prisma BookMeta model) + * - prose_chapters (via Prisma ProseChapter model) + */ + +import { prisma } from "@/lib/db"; +import type { JsonValue } from "@prisma/client/runtime/library"; +import fs from "fs"; +import path from "path"; + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +/** Volume sentinel for books without volumes. */ +const NO_VOLUME = 0; + +const BOOK = "pk"; + +const PK_JSON_DIR = path.join(process.cwd(), "lib", "jsons", "books", "pk"); + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export type PkMeta = { + book: string; + title: string; + author: string; + year?: number; + description: string; + totalChapters: number; + chapters: PkChapterEntry[]; +}; + +export type PkChapterEntry = { + number: number; + title: string; +}; + +export type PkChapter = { + book: string; + chapter: number | null; + title: string; + type: string; + slug?: string; + paragraphs: string[]; +}; + +/** Union type used in sidebar / table-of-contents navigation. */ +export type PkSidebarItem = { + /** URL-safe slug: "1", "2", … "6" */ + slug: string; + /** Human-readable label for the sidebar */ + label: string; + /** "chapter" */ + type: string; + /** Chapter number */ + chapter: number | null; +}; + +// --------------------------------------------------------------------------- +// HTML sanitisation (mirrors lib/api.ts) +// --------------------------------------------------------------------------- + +function sanitizeHtml(html: string): string { + let clean = html.replace(/][\s\S]*?<\/script>/gi, ""); + clean = clean.replace(/][\s\S]*?<\/style>/gi, ""); + clean = clean.replace(/][\s\S]*?<\/iframe>/gi, ""); + clean = clean.replace( + /<\/?(?:script|style|iframe|object|embed|form|input|textarea|select|button)[^>]*>/gi, + "", + ); + clean = clean.replace(/\s+on\w+\s*=\s*(?:"[^"]*"|'[^']*'|[^\s>]+)/gi, ""); + clean = clean.replace(/href\s*=\s*"javascript:[^"]*"/gi, 'href=""'); + clean = clean.replace(/href\s*=\s*'javascript:[^']*'/gi, "href=''"); + return clean; +} + +/** Normalize paragraphs — handles both native arrays and stringified JSON. */ +function normalizeParagraphs(paragraphs: string[] | JsonValue): string[] { + let result: string[]; + + if (typeof paragraphs === "string") { + try { + const parsed = JSON.parse(paragraphs); + result = Array.isArray(parsed) ? parsed : [paragraphs]; + } catch { + result = [paragraphs]; + } + } else if (Array.isArray(paragraphs)) { + result = paragraphs as string[]; + } else { + result = []; + } + + return result.map(sanitizeHtml); +} + +// --------------------------------------------------------------------------- +// Local JSON helpers (fallback when DB is not seeded) +// --------------------------------------------------------------------------- + +function readLocalJson(filePath: string): T | null { + if (!fs.existsSync(filePath)) return null; + try { + return JSON.parse(fs.readFileSync(filePath, "utf-8")) as T; + } catch { + return null; + } +} + +function getLocalChapter(chapterNum: number): PkChapter | null { + const filePath = path.join( + PK_JSON_DIR, + "chapters", + String(chapterNum).padStart(2, "0") + ".json", + ); + return readLocalJson(filePath); +} + +// --------------------------------------------------------------------------- +// Book meta +// --------------------------------------------------------------------------- + +/** + * Book-level metadata (title, author, chapter list, etc.) + */ +export async function getPkMeta(): Promise { + const meta = await prisma.bookMeta.findUnique({ where: { book: BOOK } }); + if (meta) return meta.data as unknown as PkMeta; + + // Fallback to local JSON + const introPath = path.join(PK_JSON_DIR, "intro.json"); + return readLocalJson(introPath); +} + +// --------------------------------------------------------------------------- +// Chapters +// --------------------------------------------------------------------------- + +/** + * A numbered chapter (1–6). + */ +export async function getPkChapter( + chapterNum: number, +): Promise { + if (chapterNum < 1 || chapterNum > 6 || !Number.isInteger(chapterNum)) { + return null; + } + + const doc = await prisma.proseChapter.findUnique({ + where: { + book_volume_slug: { + book: BOOK, + volume: NO_VOLUME, + slug: String(chapterNum), + }, + }, + }); + + if (doc) { + return { + book: doc.book, + chapter: doc.chapter, + title: doc.title, + type: doc.type, + slug: doc.slug, + paragraphs: normalizeParagraphs(doc.paragraphs), + }; + } + + // Fallback to local JSON + const local = getLocalChapter(chapterNum); + if (!local) return null; + return { + ...local, + paragraphs: local.paragraphs.map(sanitizeHtml), + }; +} + +// --------------------------------------------------------------------------- +// Unified content accessor +// --------------------------------------------------------------------------- + +/** + * Get content by URL slug. + * + * "1" … "6" → chapter + */ +export async function getPkContent(slug: string): Promise { + const num = Number(slug); + if (!Number.isNaN(num) && Number.isInteger(num) && num >= 1 && num <= 6) { + return getPkChapter(num); + } + + return null; +} + +// --------------------------------------------------------------------------- +// Sidebar navigation +// --------------------------------------------------------------------------- + +/** + * Build the full sidebar navigation list. + * + * Order: chapters 1–6 (no intro segments for PK) + */ +export async function getPkSidebarItems(): Promise { + const [meta, allDocs] = await Promise.all([ + getPkMeta(), + prisma.proseChapter.findMany({ + where: { book: BOOK, volume: NO_VOLUME }, + orderBy: { order: "asc" }, + select: { + slug: true, + title: true, + type: true, + chapter: true, + order: true, + }, + }), + ]); + + // If we got DB results, build from those + if (meta && allDocs.length > 0) { + return buildSidebarFromDocs(meta, allDocs); + } + + // Fallback: build from local JSON + return buildSidebarFromLocal(meta); +} + +function buildSidebarFromDocs( + meta: PkMeta, + _allDocs: { + slug: string; + title: string; + type: string; + chapter: number | null; + order: number | null; + }[], +): PkSidebarItem[] { + const items: PkSidebarItem[] = []; + + // Chapters 1–6 (use meta.chapters for consistent title/ordering) + for (const ch of meta.chapters) { + items.push({ + slug: String(ch.number), + label: `${ch.number}. ${ch.title}`, + type: "chapter", + chapter: ch.number, + }); + } + + return items; +} + +function buildSidebarFromLocal(meta: PkMeta | null): PkSidebarItem[] { + const items: PkSidebarItem[] = []; + + // Chapters from meta or scan local chapter JSONs + if (meta && meta.chapters) { + for (const ch of meta.chapters) { + items.push({ + slug: String(ch.number), + label: `${ch.number}. ${ch.title}`, + type: "chapter", + chapter: ch.number, + }); + } + } else { + // Fallback: scan local chapter JSONs + const chaptersDir = path.join(PK_JSON_DIR, "chapters"); + if (fs.existsSync(chaptersDir)) { + const files = fs + .readdirSync(chaptersDir) + .filter((f) => f.endsWith(".json")) + .sort(); + for (const file of files) { + const ch = readLocalJson(path.join(chaptersDir, file)); + if (ch) { + const num = ch.chapter ?? parseInt(file.replace(".json", ""), 10); + items.push({ + slug: String(num), + label: `${num}. ${ch.title}`, + type: "chapter", + chapter: num, + }); + } + } + } + } + + return items; +} diff --git a/lib/va.ts b/lib/va.ts new file mode 100644 index 00000000..2a5aee1e --- /dev/null +++ b/lib/va.ts @@ -0,0 +1,167 @@ +/** + * Data access layer for Vaiśiṣṭyāṣṭakam (VA). + * + * Reads from MongoDB via Prisma. + * VA is a poetic work with 8 chapters (Vaiśiṣṭyas) and no intro segments. + * + * Collections used: + * - book_meta (via Prisma BookMeta model) + * - prose_chapters (via Prisma ProseChapter model) + */ + +import { prisma } from "@/lib/db"; +import type { JsonValue } from "@prisma/client/runtime/library"; + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +const BOOK = "va"; + +/** Volume sentinel for books without volumes. */ +const NO_VOLUME = 0; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export type VaMeta = { + book: string; + title: string; + author: string; + year: number; + description: string; + totalChapters: number; + chapters: VaChapterEntry[]; +}; + +export type VaChapterEntry = { + number: number; + title: string; +}; + +export type VaChapter = { + book: string; + chapter: number | null; + title: string; + type: string; + slug?: string; + paragraphs: string[]; +}; + +/** Used in sidebar / table-of-contents navigation. */ +export type VaSidebarItem = { + slug: string; + label: string; + type: string; + chapter: number | null; +}; + +// --------------------------------------------------------------------------- +// HTML sanitisation +// --------------------------------------------------------------------------- + +function sanitizeHtml(html: string): string { + let clean = html.replace(/][\s\S]*?<\/script>/gi, ""); + clean = clean.replace(/][\s\S]*?<\/style>/gi, ""); + clean = clean.replace(/][\s\S]*?<\/iframe>/gi, ""); + clean = clean.replace( + /<\/?(?:script|style|iframe|object|embed|form|input|textarea|select|button)[^>]*>/gi, + "", + ); + clean = clean.replace(/\s+on\w+\s*=\s*(?:"[^"]*"|'[^']*'|[^\s>]+)/gi, ""); + clean = clean.replace(/href\s*=\s*"javascript:[^"]*"/gi, 'href=""'); + clean = clean.replace(/href\s*=\s*'javascript:[^']*'/gi, "href=''"); + return clean; +} + +function normalizeParagraphs(paragraphs: string[] | JsonValue): string[] { + let result: string[]; + + if (typeof paragraphs === "string") { + try { + const parsed = JSON.parse(paragraphs); + result = Array.isArray(parsed) ? parsed : [paragraphs]; + } catch { + result = [paragraphs]; + } + } else if (Array.isArray(paragraphs)) { + result = paragraphs as string[]; + } else { + result = []; + } + + return result.map(sanitizeHtml); +} + +// --------------------------------------------------------------------------- +// Book meta +// --------------------------------------------------------------------------- + +export async function getVaMeta(): Promise { + const meta = await prisma.bookMeta.findUnique({ where: { book: BOOK } }); + if (!meta) return null; + return meta.data as unknown as VaMeta; +} + +// --------------------------------------------------------------------------- +// Chapters +// --------------------------------------------------------------------------- + +export async function getVaChapter( + chapterNum: number, +): Promise { + if (chapterNum < 1 || chapterNum > 8 || !Number.isInteger(chapterNum)) { + return null; + } + + const doc = await prisma.proseChapter.findUnique({ + where: { + book_volume_slug: { + book: BOOK, + volume: NO_VOLUME, + slug: String(chapterNum), + }, + }, + }); + if (!doc) return null; + + return { + book: doc.book, + chapter: doc.chapter, + title: doc.title, + type: doc.type, + slug: doc.slug, + paragraphs: normalizeParagraphs(doc.paragraphs), + }; +} + +// --------------------------------------------------------------------------- +// Unified content accessor +// --------------------------------------------------------------------------- + +export async function getVaContent( + slug: string, +): Promise { + const num = Number(slug); + if (!Number.isNaN(num) && Number.isInteger(num) && num >= 1 && num <= 8) { + return getVaChapter(num); + } + return null; +} + +// --------------------------------------------------------------------------- +// Sidebar navigation +// --------------------------------------------------------------------------- + +export async function getVaSidebarItems(): Promise { + const meta = await getVaMeta(); + if (!meta) return []; + + return meta.chapters.map((ch) => ({ + slug: String(ch.number), + label: `${ch.number}. ${ch.title}`, + type: "chapter", + chapter: ch.number, + })); +} diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 0b95cf5c..ad0ba8f7 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -118,6 +118,86 @@ model Essay { @@map("essays") } +// --------------------------------------------------------------------------- +// OtherDocumentMeta — metadata for miscellaneous documents +// +// book: slug identifier (e.g. "analysis-bg", "real-peace-formula") +// data: JSON blob containing title, author, year, description, chapters list +// --------------------------------------------------------------------------- +model OtherDocumentMeta { + id String @id @default(auto()) @map("_id") @db.ObjectId + book String @unique + data Json + + @@map("other_document_meta") +} + +// --------------------------------------------------------------------------- +// OtherDocumentChapter — chapters within other documents +// +// book: slug identifier matching OtherDocumentMeta.book +// chapter: chapter number +// title: chapter title +// type: "chapter" +// slug: URL-friendly identifier (chapter number as string) +// order: sort order +// paragraphs: array of HTML paragraph strings +// --------------------------------------------------------------------------- +model OtherDocumentChapter { + id String @id @default(auto()) @map("_id") @db.ObjectId + book String + chapter Int + title String + type String + slug String + order Int + paragraphs String[] + + @@unique([book, slug]) + @@index([book]) + @@map("other_document_chapters") +} + +// --------------------------------------------------------------------------- +// JournalMeta — metadata for journals (Jaladuta Diary, The Beginning, etc.) +// +// book: slug identifier (e.g. "jaladuta-diary", "the-beginning") +// data: JSON blob containing title, author, year, description, chapters list +// --------------------------------------------------------------------------- +model JournalMeta { + id String @id @default(auto()) @map("_id") @db.ObjectId + book String @unique + data Json + + @@map("journal_meta") +} + +// --------------------------------------------------------------------------- +// JournalChapter — chapters within journals +// +// book: slug identifier matching JournalMeta.book +// chapter: chapter number +// title: chapter title +// type: "chapter" | "preface" | "introduction" +// slug: URL-friendly identifier (chapter number as string) +// order: sort order +// paragraphs: array of HTML paragraph strings +// --------------------------------------------------------------------------- +model JournalChapter { + id String @id @default(auto()) @map("_id") @db.ObjectId + book String + chapter Int + title String + type String + slug String + order Int + paragraphs String[] + + @@unique([book, slug]) + @@index([book]) + @@map("journal_chapters") +} + // --------------------------------------------------------------------------- // GlossaryEntry — individual glossary term + definition // --------------------------------------------------------------------------- diff --git a/scripts/bk_extract.py b/scripts/bk_extract.py new file mode 100644 index 00000000..c6588abc --- /dev/null +++ b/scripts/bk_extract.py @@ -0,0 +1,590 @@ +#!/usr/bin/env python3 +"""Extract Bhagavāner Kathā (part of Renunciation through Wisdom) from others-before-1977.html. + +The book has 2 chapters (no intro segments): + 1. This World of Misery + 2. The Cause of Misery + +Source: public/html/others-before-1977.html, lines ~4182–4289. + Start:

containing "Bhagaväner kathä" + End:

containing "Indian Bhagavatams" (stop before) + +Chapter boundaries:

tags containing "Bk:Chapter" + +IMPORTANT: Unlike MOG/CBHC, this book's body text uses ScaGoudy font encoding. +Characters like ä, å, ñ, ö, ë, ò, ç, ì, ï, ü, é, à, ù must be decoded to +proper IAST Unicode (ā, ṛ, ṣ, ṭ, ṇ, ḍ, ś, ṅ, ñ, ū, ī, ṁ, ḥ). + +CSS classes: + p56 — Book title header (skip) + p17 — Chapter bookmark headers ("Bk:Chapter N") — boundary detection, skip + p5 — Chapter title headings (skip — we derive titles ourselves) + p6 — Body paragraphs (MAIN content) — HAS ScaGoudy encoding + p12 — Body paragraph variant (longer text) — HAS ScaGoudy encoding + p38 — Body paragraph variant — HAS ScaGoudy encoding + p9 — Sanskrit verse lines (italic, with ) — centered italic + p10 — Reference markers like [RTW 1.1] — include as paragraphs + p61 — Reference marker variant [RTW 1.2] — include + p57 — Footnote header ("Footnote") — include + p58 — Footnote text — include + p19 — Sanskrit verse block (multi-line) — centered italic + p59 — Sanskrit verse block variant — centered italic + p23 — Sanskrit verse line variant — centered italic + p36 — Sanskrit verse line variant — centered italic + p60 — Sanskrit verse line variant — centered italic + +Output: + lib/jsons/books/bk/intro.json — Book metadata + lib/jsons/books/bk/chapters/01.json — Chapter 1: This World of Misery + lib/jsons/books/bk/chapters/02.json — Chapter 2: The Cause of Misery + +Usage: + python3 scripts/bk_extract.py +""" + +import json +import os +import re +import sys + +from bs4 import BeautifulSoup, Tag + +# --------------------------------------------------------------------------- +# ScaGoudy → IAST Unicode mapping +# --------------------------------------------------------------------------- + +SCAGOUDY_TO_IAST = { + "ä": "ā", + "Ä": "Ā", + "å": "ṛ", + "Å": "Ṛ", + "ñ": "ṣ", + "Ñ": "Ṣ", + "ö": "ṭ", + "Ö": "Ṭ", + "ë": "ṇ", + "Ë": "Ṇ", + "ò": "ḍ", + "Ò": "Ḍ", + "ç": "ś", + "Ç": "Ś", + "ì": "ṅ", + "Ì": "Ṅ", + "ï": "ñ", + "Ï": "Ñ", + "ü": "ū", + "Ü": "Ū", + "é": "ī", + "É": "Ī", + "à": "ṁ", + "À": "Ṁ", + "ù": "ḥ", + "Ù": "Ḥ", + "è": "ṝ", + "û": "ṁl", + "ÿ": "ḷ", +} + +# Build a translation table for fast character-by-character replacement +_SCAGOUDY_TRANS = str.maketrans( + {k: v for k, v in SCAGOUDY_TO_IAST.items() if len(k) == 1} +) + + +def scagoudy_decode(text: str) -> str: + """Decode ScaGoudy-encoded text to proper IAST Unicode.""" + return text.translate(_SCAGOUDY_TRANS) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def decode_html_content(tag: Tag, apply_scagoudy: bool = True) -> str: + """Extract inner HTML from a tag, preserve formatting tags, strip spans, + and optionally apply ScaGoudy decoding. + + Steps: + 1. Get inner HTML + 2. Strip wrappers (keep content) + 3. Apply ScaGoudy decode to the entire result (if requested) + 4. Normalize whitespace + """ + inner = "".join(str(child) for child in tag.children) + # Remove span tags with CSS classes (RTF conversion artifacts) — keep content + inner = re.sub( + r']*class="[^"]*"[^>]*>(.*?)', r"\1", inner, flags=re.DOTALL + ) + # Clean up Apple-tab-span leftovers (tab characters) + inner = inner.replace("\t", " ") + # Normalize whitespace + inner = re.sub(r"\s+", " ", inner).strip() + # Apply ScaGoudy decoding + if apply_scagoudy: + inner = scagoudy_decode(inner) + return inner + + +def decode_html_content_as_verse(tag: Tag) -> str: + """Extract inner HTML, strip spans, ScaGoudy decode, wrap as centered italic. + + For Sanskrit verse lines (p9, p19, p59, p23, p36, p60). + Preserves
and reference tags like [Bg. 7.14]. + """ + inner = "".join(str(child) for child in tag.children) + # Remove span tags with CSS classes — keep content + inner = re.sub( + r']*class="[^"]*"[^>]*>(.*?)', r"\1", inner, flags=re.DOTALL + ) + # Clean up tab characters + inner = inner.replace("\t", " ") + # Normalize spaces (but preserve
tags) + # Replace multiple spaces with single space + inner = re.sub(r"[ ]+", " ", inner).strip() + # Remove trailing
tags + inner = re.sub(r"()+\s*$", "", inner).strip() + # Apply ScaGoudy decoding + inner = scagoudy_decode(inner) + # Wrap as centered italic + if inner: + return f'

{inner}
' + return "" + + +def decode_text(tag: Tag) -> str: + """Get plain text from a tag.""" + return tag.get_text().strip() + + +def get_css_class(tag: Tag) -> str: + """Get the first CSS class of a tag, or empty string.""" + classes = tag.get("class", []) + return classes[0] if classes else "" + + +def is_empty(tag: Tag) -> bool: + """Check if a tag is empty or whitespace-only.""" + text = tag.get_text().strip() + return not text or text == "\xa0" + + +# --------------------------------------------------------------------------- +# CSS class categorization for BK +# --------------------------------------------------------------------------- + +# Book title header +BOOK_TITLE_CLASS = "p56" + +# Chapter bookmark headers (used for boundary detection) +CHAPTER_BOOKMARK_CLASS = "p17" + +# Chapter title (bold) — skip from body +CHAPTER_TITLE_CLASS = "p5" + +# Body paragraph classes (all need ScaGoudy decoding) +BODY_CLASSES = frozenset(["p6", "p12", "p38"]) + +# Sanskrit verse line classes (wrap as centered italic) +VERSE_CLASSES = frozenset(["p9", "p19", "p59", "p23", "p36", "p60"]) + +# Reference marker classes — include as paragraphs +REFERENCE_CLASSES = frozenset(["p10", "p61"]) + +# Footnote classes — include as paragraphs +FOOTNOTE_CLASSES = frozenset(["p57", "p58"]) + +# Classes to skip entirely +SKIP_CLASSES = frozenset(["p43", "p24"]) + +# TOC entry class +TOC_CLASS = "p2" + + +# --------------------------------------------------------------------------- +# Boundary detection +# --------------------------------------------------------------------------- + + +def find_bk_boundaries(all_paragraphs: list) -> tuple: + """Find the start and end paragraph indices of the BK content. + + Start:

containing "Bhagaväner kathä" + End:

containing "Indian Bhagavatams" (stop before) + """ + start_idx = None + end_idx = len(all_paragraphs) + + for i, p in enumerate(all_paragraphs): + cls = get_css_class(p) + text = decode_text(p) + + # Find start: p56 containing "Bhagaväner" or p2 TOC entry + if start_idx is None: + if cls == BOOK_TITLE_CLASS and "Bhag" in text and "kath" in text: + start_idx = i + continue + continue + + # Find end: p35 containing "Indian Bhagavatams" + if cls == "p35" and "Indian" in text and "Bhagavatam" in text: + end_idx = i + break + + if start_idx is None: + print("ERROR: Could not find 'Bhagaväner kathä' in others-before-1977.html!") + sys.exit(1) + + return start_idx, end_idx + + +def find_chapter_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list: + """Find all BK chapter bookmark tags (p17) in the BK range. + + Returns list of (paragraph_index, raw_title_text). + """ + bookmarks = [] + for i in range(start_idx, end_idx): + p = all_paragraphs[i] + cls = get_css_class(p) + if cls == CHAPTER_BOOKMARK_CLASS: + text = decode_text(p).strip() + if text and "Bk:" in text: + bookmarks.append((i, text)) + return bookmarks + + +# --------------------------------------------------------------------------- +# Paragraph extraction +# --------------------------------------------------------------------------- + + +def extract_paragraphs(all_paragraphs: list, start_idx: int, end_idx: int) -> list: + """Extract body paragraphs for a chapter section. + + Collects: + - p6, p12, p38 (body paragraphs — ScaGoudy decoded) + - p9, p19, p59, p23, p36, p60 (verse lines — centered italic, ScaGoudy decoded) + - p10, p61 (reference markers — ScaGoudy decoded) + - p57, p58 (footnotes — ScaGoudy decoded) + Skips: + - p56 (book title), p17 (bookmarks), p5 (chapter titles), p2 (TOC), empties + """ + result = [] + + for i in range(start_idx, end_idx): + p = all_paragraphs[i] + cls = get_css_class(p) + + # Skip empty paragraphs + if is_empty(p): + continue + + # Skip known skip classes + if cls in SKIP_CLASSES: + continue + + # Skip book title header + if cls == BOOK_TITLE_CLASS: + continue + + # Skip chapter bookmark headers + if cls == CHAPTER_BOOKMARK_CLASS: + continue + + # Skip chapter title headings (p5) + if cls == CHAPTER_TITLE_CLASS: + continue + + # Skip TOC entries (p2) + if cls == TOC_CLASS: + continue + + # Body paragraphs (ScaGoudy decoded) + if cls in BODY_CLASSES: + html = decode_html_content(p, apply_scagoudy=True) + if html: + result.append(html) + continue + + # Sanskrit verse lines (centered italic, ScaGoudy decoded) + if cls in VERSE_CLASSES: + html = decode_html_content_as_verse(p) + if html: + result.append(html) + continue + + # Reference markers (ScaGoudy decoded) + if cls in REFERENCE_CLASSES: + html = decode_html_content(p, apply_scagoudy=True) + if html: + result.append(html) + continue + + # Footnote classes (ScaGoudy decoded) + if cls in FOOTNOTE_CLASSES: + html = decode_html_content(p, apply_scagoudy=True) + if html: + result.append(html) + continue + + # Fallback: try to extract content from unknown classes + html = decode_html_content(p, apply_scagoudy=True) + if html: + print(f" ⚠ Unknown class '{cls}' at paragraph {i}: {html[:80]}...") + result.append(html) + + return result + + +# --------------------------------------------------------------------------- +# Main extraction logic +# --------------------------------------------------------------------------- + + +def extract_bk(soup: BeautifulSoup) -> dict: + """Extract the entire Bhagavāner Kathā from the parsed HTML. + + Returns a dict with: + "chapters": list of chapter dicts + "chapter_listing": list of {number, title} for intro.json + """ + all_paragraphs = list(soup.find_all("p")) + + # Step 1: Find BK boundaries + bk_start, bk_end = find_bk_boundaries(all_paragraphs) + print(f" BK content: paragraphs {bk_start} to {bk_end}") + print(f" Total paragraphs in range: {bk_end - bk_start}") + + # Step 2: Find all chapter bookmarks (p17 with "Bk:") + bookmarks = find_chapter_bookmarks(all_paragraphs, bk_start, bk_end) + print(f" Found {len(bookmarks)} BK chapter bookmarks:") + for pos, title in bookmarks: + print(f" Paragraph {pos}: {title}") + + # Step 3: Map bookmarks to chapters with proper titles + # Expected: 2 chapters + chapter_titles_map = { + 1: "This World of Misery", + 2: "The Cause of Misery", + } + + chapter_bookmarks = [] # (para_idx, chapter_num, normalized_title) + for idx, (pos, _raw_title) in enumerate(bookmarks): + ch_num = idx + 1 + ch_title = chapter_titles_map.get(ch_num, f"Chapter {ch_num}") + chapter_bookmarks.append((pos, ch_num, ch_title)) + + print(f"\n Mapped {len(chapter_bookmarks)} chapters:") + for pos, num, title in chapter_bookmarks: + print(f" Chapter {num}: {title} (paragraph {pos})") + + # Step 4: Extract chapters + chapters = [] + for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_bookmarks): + # Content starts after the bookmark line (skip p17 and p5 heading) + content_start = ch_pos + 1 + + # Content ends at next chapter bookmark or bk_end + if idx + 1 < len(chapter_bookmarks): + content_end = chapter_bookmarks[idx + 1][0] + else: + content_end = bk_end + + paragraphs = extract_paragraphs(all_paragraphs, content_start, content_end) + + ch = { + "book": "bk", + "chapter": ch_num, + "title": ch_title, + "type": "chapter", + "paragraphs": paragraphs, + } + chapters.append(ch) + + # Step 5: Build chapter listing for intro.json + chapter_listing = [ + {"number": ch["chapter"], "title": ch["title"]} for ch in chapters + ] + + return { + "chapters": chapters, + "chapter_listing": chapter_listing, + } + + +# --------------------------------------------------------------------------- +# Output +# --------------------------------------------------------------------------- + + +def write_json(data, filepath): + """Write data as formatted JSON to filepath, creating directories as needed.""" + os.makedirs(os.path.dirname(filepath), exist_ok=True) + with open(filepath, "w", encoding="utf-8") as f: + json.dump(data, f, ensure_ascii=False, indent=2) + + +def main(): + base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html") + output_base = os.path.join(base_dir, "lib", "jsons", "books", "bk") + + if not os.path.exists(html_path): + print(f"ERROR: {html_path} not found!") + sys.exit(1) + + print(f"Reading {html_path}...") + with open(html_path, "r", encoding="utf-8") as f: + soup = BeautifulSoup(f, "html.parser") + + print("Extracting Bhagavāner Kathā...\n") + result = extract_bk(soup) + + chapters = result["chapters"] + chapter_listing = result["chapter_listing"] + + # --- Write intro.json --- + intro_meta = { + "book": "bk", + "title": "Bhagavāner Kathā", + "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda", + "year": None, + "description": ( + "Early Bengali writings on the topics of Godhead " + "— part of 'Renunciation through Wisdom' (Vairāgya-vidyā)." + ), + "totalChapters": 2, + "chapters": chapter_listing, + } + intro_path = os.path.join(output_base, "intro.json") + write_json(intro_meta, intro_path) + print(f"\n Written: {intro_path}") + + # --- Write chapters --- + print(f"\n--- Chapters ---") + for ch in chapters: + ch_num_str = str(ch["chapter"]).zfill(2) + filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json") + write_json(ch, filepath) + print( + f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — " + f"{ch['title']}" + ) + + # --- Summary --- + total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters) + + print(f"\n{'=' * 60}") + print(f" EXTRACTION SUMMARY") + print(f"{'=' * 60}") + print(f" Chapters: {len(chapters)}") + for ch in chapters: + print( + f" Ch {ch['chapter']}: {len(ch['paragraphs'])} paragraphs — {ch['title']}" + ) + print(f" Total paragraphs: {total_chapter_paras}") + if chapters: + print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}") + print(f" Output directory: {output_base}/") + + # --- Samples --- + for ch in chapters: + ch_num = ch["chapter"] + print(f"\n--- Sample: Chapter {ch_num} ({ch['title']}) ---") + for i, p in enumerate(ch["paragraphs"][:3]): + preview = re.sub(r"<[^>]+>", "", p)[:120] + print(f" [{i}] {preview}...") + if len(ch["paragraphs"]) > 3: + print(f" ... ({len(ch['paragraphs']) - 3} more paragraphs)") + + # --- Quality Checks --- + print(f"\n--- Quality Checks ---") + + # Check all 2 chapters present + expected_chapters = set(range(1, 3)) + actual_chapters = set(ch["chapter"] for ch in chapters) + missing = expected_chapters - actual_chapters + extra = actual_chapters - expected_chapters + if missing: + print(f" ⚠ Missing chapters: {sorted(missing)}") + if extra: + print(f" ⚠ Extra chapters: {sorted(extra)}") + if not missing and not extra: + print(f" ✅ All 2 chapters present") + + # Check empty chapters + empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0] + if empty_chapters: + print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}") + else: + print(f" ✅ No empty chapters") + + # Check ScaGoudy decoding worked — look for common ScaGoudy characters + scagoudy_chars = set("äåñöëòçìïüéàù") + all_text = "" + for ch in chapters: + all_text += " ".join(ch["paragraphs"]) + remaining_scagoudy = [c for c in all_text if c in scagoudy_chars] + if remaining_scagoudy: + # Count occurrences + from collections import Counter + + counts = Counter(remaining_scagoudy) + print( + f" ⚠ Remaining ScaGoudy characters found (may be false positives): {dict(counts)}" + ) + else: + print(f" ✅ No ScaGoudy characters remaining — decoding looks clean") + + # Check for IAST characters (positive signal) + iast_chars = set("āṛṣṭṇḍśṅñūīṁḥ") + iast_found = [c for c in all_text if c in iast_chars] + if iast_found: + from collections import Counter + + counts = Counter(iast_found) + print(f" ✅ IAST Unicode characters present: {dict(counts)}") + else: + print(f" ⚠ No IAST Unicode characters found — decoding may have failed") + + # Check chapter 1 has [RTW 1.1] reference + if chapters: + ch1_text = " ".join(chapters[0]["paragraphs"]) + if "RTW" in ch1_text and "1.1" in ch1_text: + print(f" ✅ '[RTW 1.1]' reference found in Chapter 1") + else: + print(f" ⚠ '[RTW 1.1]' reference NOT found in Chapter 1") + + # Check chapter 2 has [RTW 1.2] reference + if len(chapters) >= 2: + ch2_text = " ".join(chapters[1]["paragraphs"]) + if "RTW" in ch2_text and "1.2" in ch2_text: + print(f" ✅ '[RTW 1.2]' reference found in Chapter 2") + else: + print(f" ⚠ '[RTW 1.2]' reference NOT found in Chapter 2") + + # Check for content leak from next work (Indian Bhagavatams) + if "Indian Bhagavatam" in all_text: + print(f" ⚠ Content from next work (Indian Bhagavatams) may have leaked!") + else: + print(f" ✅ No content leak from next work detected") + + # Check Kṛṣṇa appears (decoded from Kåñëa) + if "Kṛṣṇa" in all_text: + print(f" ✅ 'Kṛṣṇa' properly decoded (from 'Kåñëa')") + else: + print(f" ⚠ 'Kṛṣṇa' not found — ScaGoudy decoding may need review") + + # Check last chapter ends cleanly + if chapters: + last_ch = chapters[-1] + last_para = last_ch["paragraphs"][-1] if last_ch["paragraphs"] else "" + last_preview = re.sub(r"<[^>]+>", "", last_para)[:100] + print(f' ℹ Last paragraph of Ch {last_ch["chapter"]}: "{last_preview}..."') + + print(f"\nDone! ✨") + + +if __name__ == "__main__": + main() diff --git a/scripts/cbhc_extract.py b/scripts/cbhc_extract.py new file mode 100644 index 00000000..a137c907 --- /dev/null +++ b/scripts/cbhc_extract.py @@ -0,0 +1,444 @@ +#!/usr/bin/env python3 +"""Extract Śrī Caitanya-bhāgavata Commentary from others-before-1977.html. + +The book has 4 chapters (no intro segments): + 1. Chapter 1 + 2. Lord Caitanya's Appearance + 3. Chapter 3 + 4. Chapter 4 + +Source: public/html/others-before-1977.html, lines ~4139–4180. + Start:

Çré Caitanya-bhägavata-Commentary

+ End:

Bhagaväner kathä [Original Documents]

(stop before) + +Chapter boundaries:

tags containing "Caitanya-bhägavata" + +CSS classes: + p3 — Book title header (skip) + p17 — Chapter bookmark headers (boundary detection, skip from output) + p5 — Chapter title headings (skip — we derive titles ourselves) + p6 — Body paragraphs (MAIN content) + p38 — Body paragraph variant + p55 — Body paragraph variant (last paragraph of chapter 4) + p9 — End-of-chapter markers like "(End Chapter One)" — include + +Output: + lib/jsons/books/cbhc/intro.json — Book metadata + lib/jsons/books/cbhc/chapters/01.json — Chapter 1 + lib/jsons/books/cbhc/chapters/02.json — Chapter 2 + lib/jsons/books/cbhc/chapters/03.json — Chapter 3 + lib/jsons/books/cbhc/chapters/04.json — Chapter 4 + +Usage: + python3 scripts/cbhc_extract.py +""" + +import json +import os +import re +import sys + +from bs4 import BeautifulSoup, Tag + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def decode_html_content(tag: Tag) -> str: + """Extract inner HTML from a tag, preserve formatting tags, strip spans.""" + inner = "".join(str(child) for child in tag.children) + # Remove span tags with CSS classes (RTF conversion artifacts) + inner = re.sub( + r']*class="[^"]*"[^>]*>(.*?)', r"\1", inner, flags=re.DOTALL + ) + # Clean up Apple-tab-span leftovers (tab characters) + inner = inner.replace("\t", " ") + # Normalize whitespace + inner = re.sub(r"\s+", " ", inner).strip() + return inner + + +def decode_text(tag: Tag) -> str: + """Get plain text from a tag.""" + return tag.get_text().strip() + + +def get_css_class(tag: Tag) -> str: + """Get the first CSS class of a tag, or empty string.""" + classes = tag.get("class", []) + return classes[0] if classes else "" + + +def is_empty(tag: Tag) -> bool: + """Check if a tag is empty or whitespace-only.""" + text = tag.get_text().strip() + return not text or text == "\xa0" + + +# --------------------------------------------------------------------------- +# CSS class categorization for CBHC +# --------------------------------------------------------------------------- + +# Book title header +BOOK_TITLE_CLASS = "p3" + +# Chapter bookmark headers (used for boundary detection) +CHAPTER_BOOKMARK_CLASS = "p17" + +# Chapter title (bold, e.g., "Chapter 1") — skip from body +CHAPTER_TITLE_CLASS = "p5" + +# Body paragraph classes +BODY_CLASSES = frozenset(["p6", "p38", "p55"]) + +# End-of-chapter markers — include as a paragraph +END_MARKER_CLASS = "p9" + +# Classes to skip entirely +SKIP_CLASSES = frozenset(["p43", "p24"]) + + +# --------------------------------------------------------------------------- +# Boundary detection +# --------------------------------------------------------------------------- + + +def find_cbhc_boundaries(all_paragraphs: list) -> tuple: + """Find the start and end paragraph indices of the CBHC content. + + Start:

containing "Caitanya-bhägavata-Commentary" + End:

containing "Bhagaväner kathä" (next work) + """ + start_idx = None + end_idx = len(all_paragraphs) + + for i, p in enumerate(all_paragraphs): + cls = get_css_class(p) + text = decode_text(p) + + # Find start: p3 containing "Caitanya-bhägavata" + if start_idx is None: + if cls == BOOK_TITLE_CLASS and "Caitanya-bh" in text: + start_idx = i + continue + continue + + # Find end: p2 containing "Bhagaväner" (next work boundary) + if cls == "p2" and "Bhag" in text and "kath" in text: + end_idx = i + break + + if start_idx is None: + print( + "ERROR: Could not find 'Çré Caitanya-bhägavata-Commentary' " + "in others-before-1977.html!" + ) + sys.exit(1) + + return start_idx, end_idx + + +def find_chapter_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list: + """Find all CBHC chapter bookmark tags (p17) in the CBHC range. + + Returns list of (paragraph_index, raw_title_text). + """ + bookmarks = [] + for i in range(start_idx, end_idx): + p = all_paragraphs[i] + cls = get_css_class(p) + if cls == CHAPTER_BOOKMARK_CLASS: + text = decode_text(p).strip() + if text and "Caitanya-bh" in text: + bookmarks.append((i, text)) + return bookmarks + + +# --------------------------------------------------------------------------- +# Paragraph extraction +# --------------------------------------------------------------------------- + + +def extract_paragraphs(all_paragraphs: list, start_idx: int, end_idx: int) -> list: + """Extract body paragraphs for a chapter section. + + Collects p6, p38, p55, and p9 (end markers). + Skips p3 (book header), p17 (bookmarks), p5 (chapter titles), p2, empties. + """ + result = [] + + for i in range(start_idx, end_idx): + p = all_paragraphs[i] + cls = get_css_class(p) + + # Skip empty paragraphs + if is_empty(p): + continue + + # Skip known skip classes + if cls in SKIP_CLASSES: + continue + + # Skip book title header + if cls == BOOK_TITLE_CLASS: + continue + + # Skip chapter bookmark headers + if cls == CHAPTER_BOOKMARK_CLASS: + continue + + # Skip chapter title headings (p5) + if cls == CHAPTER_TITLE_CLASS: + continue + + # Skip p2 (TOC entries) + if cls == "p2": + continue + + # Body paragraphs + if cls in BODY_CLASSES: + html = decode_html_content(p) + if html: + result.append(html) + continue + + # End-of-chapter markers (p9) — include as a paragraph + if cls == END_MARKER_CLASS: + html = decode_html_content(p) + if html: + result.append(html) + continue + + # Fallback: try to extract content from unknown classes + html = decode_html_content(p) + if html: + print(f" ⚠ Unknown class '{cls}' at paragraph {i}: {html[:80]}...") + result.append(html) + + return result + + +# --------------------------------------------------------------------------- +# Main extraction logic +# --------------------------------------------------------------------------- + + +def extract_cbhc(soup: BeautifulSoup) -> dict: + """Extract the entire Śrī Caitanya-bhāgavata Commentary from the parsed HTML. + + Returns a dict with: + "chapters": list of chapter dicts + "chapter_listing": list of {number, title} for intro.json + """ + all_paragraphs = list(soup.find_all("p")) + + # Step 1: Find CBHC boundaries + cbhc_start, cbhc_end = find_cbhc_boundaries(all_paragraphs) + print(f" CBHC content: paragraphs {cbhc_start} to {cbhc_end}") + print(f" Total paragraphs in range: {cbhc_end - cbhc_start}") + + # Step 2: Find all chapter bookmarks (p17) + bookmarks = find_chapter_bookmarks(all_paragraphs, cbhc_start, cbhc_end) + print(f" Found {len(bookmarks)} CBHC chapter bookmarks:") + for pos, title in bookmarks: + print(f" Paragraph {pos}: {title}") + + # Step 3: Map bookmarks to chapters with proper titles + # Expected: 4 chapters + # We derive proper titles from the p5 heading that follows each bookmark + chapter_titles_map = { + 1: "Chapter 1", + 2: "Lord Caitanya's Appearance", + 3: "Chapter 3", + 4: "Chapter 4", + } + + chapter_bookmarks = [] # (para_idx, chapter_num, normalized_title) + for idx, (pos, _raw_title) in enumerate(bookmarks): + ch_num = idx + 1 + ch_title = chapter_titles_map.get(ch_num, f"Chapter {ch_num}") + chapter_bookmarks.append((pos, ch_num, ch_title)) + + print(f"\n Mapped {len(chapter_bookmarks)} chapters:") + for pos, num, title in chapter_bookmarks: + print(f" Chapter {num}: {title} (paragraph {pos})") + + # Step 4: Extract chapters + chapters = [] + for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_bookmarks): + # Content starts after the bookmark line (skip p17 and p5 heading) + content_start = ch_pos + 1 + + # Content ends at next chapter bookmark or cbhc_end + if idx + 1 < len(chapter_bookmarks): + content_end = chapter_bookmarks[idx + 1][0] + else: + content_end = cbhc_end + + paragraphs = extract_paragraphs(all_paragraphs, content_start, content_end) + + ch = { + "book": "cbhc", + "chapter": ch_num, + "title": ch_title, + "type": "chapter", + "paragraphs": paragraphs, + } + chapters.append(ch) + + # Step 5: Build chapter listing for intro.json + chapter_listing = [ + {"number": ch["chapter"], "title": ch["title"]} for ch in chapters + ] + + return { + "chapters": chapters, + "chapter_listing": chapter_listing, + } + + +# --------------------------------------------------------------------------- +# Output +# --------------------------------------------------------------------------- + + +def write_json(data, filepath): + """Write data as formatted JSON to filepath, creating directories as needed.""" + os.makedirs(os.path.dirname(filepath), exist_ok=True) + with open(filepath, "w", encoding="utf-8") as f: + json.dump(data, f, ensure_ascii=False, indent=2) + + +def main(): + base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html") + output_base = os.path.join(base_dir, "lib", "jsons", "books", "cbhc") + + if not os.path.exists(html_path): + print(f"ERROR: {html_path} not found!") + sys.exit(1) + + print(f"Reading {html_path}...") + with open(html_path, "r", encoding="utf-8") as f: + soup = BeautifulSoup(f, "html.parser") + + print("Extracting Śrī Caitanya-bhāgavata Commentary...\n") + result = extract_cbhc(soup) + + chapters = result["chapters"] + chapter_listing = result["chapter_listing"] + + # --- Write intro.json --- + intro_meta = { + "book": "cbhc", + "title": "Śrī Caitanya-bhāgavata Commentary", + "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda", + "year": None, + "description": ( + "Commentary on Śrī Caitanya-bhāgavata by Vṛndāvana dāsa Ṭhākura " + "— covering the first four chapters of Lord Caitanya's pastimes." + ), + "totalChapters": 4, + "chapters": chapter_listing, + } + intro_path = os.path.join(output_base, "intro.json") + write_json(intro_meta, intro_path) + print(f"\n Written: {intro_path}") + + # --- Write chapters --- + print(f"\n--- Chapters ---") + for ch in chapters: + ch_num_str = str(ch["chapter"]).zfill(2) + filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json") + write_json(ch, filepath) + print( + f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — " + f"{ch['title']}" + ) + + # --- Summary --- + total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters) + + print(f"\n{'=' * 60}") + print(f" EXTRACTION SUMMARY") + print(f"{'=' * 60}") + print(f" Chapters: {len(chapters)}") + for ch in chapters: + print( + f" Ch {ch['chapter']}: {len(ch['paragraphs'])} paragraphs — {ch['title']}" + ) + print(f" Total paragraphs: {total_chapter_paras}") + if chapters: + print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}") + print(f" Output directory: {output_base}/") + + # --- Samples --- + for ch in chapters: + ch_num = ch["chapter"] + print(f"\n--- Sample: Chapter {ch_num} ({ch['title']}) ---") + for i, p in enumerate(ch["paragraphs"][:3]): + preview = re.sub(r"<[^>]+>", "", p)[:120] + print(f" [{i}] {preview}...") + if len(ch["paragraphs"]) > 3: + print(f" ... ({len(ch['paragraphs']) - 3} more paragraphs)") + + # --- Quality Checks --- + print(f"\n--- Quality Checks ---") + + # Check all 4 chapters present + expected_chapters = set(range(1, 5)) + actual_chapters = set(ch["chapter"] for ch in chapters) + missing = expected_chapters - actual_chapters + extra = actual_chapters - expected_chapters + if missing: + print(f" ⚠ Missing chapters: {sorted(missing)}") + if extra: + print(f" ⚠ Extra chapters: {sorted(extra)}") + if not missing and not extra: + print(f" ✅ All 4 chapters present") + + # Check empty chapters + empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0] + if empty_chapters: + print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}") + else: + print(f" ✅ No empty chapters") + + # Check chapter 1 has "(End Chapter One)" + if chapters: + ch1_text = " ".join(chapters[0]["paragraphs"]) + if "End Chapter One" in ch1_text: + print(f" ✅ '(End Chapter One)' end marker found in Chapter 1") + else: + print(f" ⚠ '(End Chapter One)' end marker NOT found in Chapter 1") + + # Check chapter 2 mentions Lord Caitanya's appearance + if len(chapters) >= 2: + ch2_text = " ".join(chapters[1]["paragraphs"]) + if "appearance" in ch2_text.lower() or "born" in ch2_text.lower(): + print(f" ✅ Chapter 2 contains appearance/birth content") + else: + print(f" ⚠ Chapter 2 may not contain appearance content") + + # Check for content leak from next work (Bhagaväner kathä) + all_text = "" + for ch in chapters: + all_text += " ".join(ch["paragraphs"]) + if "Original Documents" in all_text: + print(f" ⚠ Content from next work (Bhagaväner kathä) may have leaked!") + else: + print(f" ✅ No content leak from next work detected") + + # Check last chapter ends cleanly + if chapters: + last_ch = chapters[-1] + last_para = last_ch["paragraphs"][-1] if last_ch["paragraphs"] else "" + last_preview = re.sub(r"<[^>]+>", "", last_para)[:100] + print(f' ℹ Last paragraph of Ch {last_ch["chapter"]}: "{last_preview}..."') + + print(f"\nDone! ✨") + + +if __name__ == "__main__": + main() diff --git a/scripts/cc_essays_extract.py b/scripts/cc_essays_extract.py new file mode 100644 index 00000000..3c91a046 --- /dev/null +++ b/scripts/cc_essays_extract.py @@ -0,0 +1,675 @@ +#!/usr/bin/env python3 +"""Extract five "Essays on Śrī Caitanya-caritāmṛta" from others-before-1977.html. + +Essay 1: "The Spiritual Masters" (cc-essays, order 1, 1960) +Essay 2: "The Absolute Truth" (cc-essays, order 2, 1957) +Essay 3: "The Causes of Lord Caitanya's Incarnation" (cc-essays, order 3, 1957) +Essay 4: "The Confidential Reasons for Lord Caitanya's Appearance" (cc-essays, order 4, 1957) +Essay 5: "The Glories of Lord Nityānanda" (cc-essays, order 5, 1957) + +All five are plain English (NO ScaGoudy decoding needed). + +Output: + lib/jsons/essays/the-spiritual-masters.json + lib/jsons/essays/the-absolute-truth.json + lib/jsons/essays/causes-of-lord-caitanyas-incarnation.json + lib/jsons/essays/confidential-reasons-for-lord-caitanyas-appearance.json + lib/jsons/essays/glories-of-lord-nityananda.json + +Usage: + python3 scripts/cc_essays_extract.py +""" + +import json +import os +import re +import sys + +from bs4 import BeautifulSoup, Tag + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def get_css_class(tag: Tag) -> str: + """Get the first CSS class of a tag, or empty string.""" + classes = tag.get("class", []) + return classes[0] if classes else "" + + +def is_empty(tag: Tag) -> bool: + """Check if a tag is empty or whitespace-only.""" + text = tag.get_text().strip() + return not text or text == "\xa0" + + +def clean_html_content(tag: Tag) -> str: + """Extract inner HTML from a tag, strip span class artifacts, preserve , ,
.""" + inner = "".join(str(child) for child in tag.children) + # Remove span tags with CSS classes (RTF conversion artifacts) but keep their content + inner = re.sub( + r']*class="[^"]*"[^>]*>(.*?)', r"\1", inner, flags=re.DOTALL + ) + # Remove any remaining bare or tags without classes + inner = re.sub(r"]*>", "", inner) + # Clean up Apple-tab-span leftovers (tab characters) + inner = inner.replace("\t", " ") + # Normalize whitespace (but preserve
tags) + inner = re.sub(r"\s+", " ", inner).strip() + return inner + + +def write_json(data, filepath): + """Write data as formatted JSON to filepath, creating directories as needed.""" + os.makedirs(os.path.dirname(filepath), exist_ok=True) + with open(filepath, "w", encoding="utf-8") as f: + json.dump(data, f, ensure_ascii=False, indent=2) + + +def is_divider(tag: Tag) -> bool: + """Check if a p9 tag is just a divider line (underscores) or a page header.""" + text = tag.get_text().strip() + # Dividers are lines of underscores + if re.match(r"^_+$", text): + return True + return False + + +def is_page_header(tag: Tag) -> bool: + """Check if a p9 tag is a page header like 'Essays on Shri Chaitanya Charitamrita Page 30'.""" + text = tag.get_text().strip() + if re.search(r"Page\s+\d+", text): + return True + return False + + +def is_editor_note(tag: Tag) -> bool: + """Check if a paragraph is an editor note like '[This file has been proofread...]'.""" + text = tag.get_text().strip() + if text.startswith("[") and ( + "proofread" in text.lower() or "—Ed" in text or "- Ed" in text or ".—Ed" in text + ): + return True + return False + + +# --------------------------------------------------------------------------- +# Essay configurations +# --------------------------------------------------------------------------- + +AUTHOR = "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda" + +ESSAYS = [ + { + "slug": "the-spiritual-masters", + "title": "The Spiritual Masters", + "author": AUTHOR, + "year": 1960, + "group": "cc-essays", + "order": 1, + }, + { + "slug": "the-absolute-truth", + "title": "The Absolute Truth", + "author": AUTHOR, + "year": 1957, + "group": "cc-essays", + "order": 2, + }, + { + "slug": "causes-of-lord-caitanyas-incarnation", + "title": "The Causes of Lord Caitanya's Incarnation", + "author": AUTHOR, + "year": 1957, + "group": "cc-essays", + "order": 3, + }, + { + "slug": "confidential-reasons-for-lord-caitanyas-appearance", + "title": "The Confidential Reasons for Lord Caitanya's Appearance", + "author": AUTHOR, + "year": 1957, + "group": "cc-essays", + "order": 4, + }, + { + "slug": "glories-of-lord-nityananda", + "title": "The Glories of Lord Nityānanda", + "author": AUTHOR, + "year": 1957, + "group": "cc-essays", + "order": 5, + }, +] + +# --------------------------------------------------------------------------- +# Extraction logic +# --------------------------------------------------------------------------- + + +def parse_all_paragraphs(html_path: str) -> list[Tag]: + """Parse the entire HTML and return all

tags.""" + with open(html_path, "r", encoding="utf-8") as f: + content = f.read() + soup = BeautifulSoup(content, "html.parser") + return list(soup.find_all("p")) + + +def find_tag_index(tags: list[Tag], start_from: int, predicate) -> int | None: + """Find index of first tag matching predicate, starting from start_from.""" + for i in range(start_from, len(tags)): + if predicate(tags[i]): + return i + return None + + +def extract_body_paragraphs( + tags: list[Tag], start_idx: int, end_idx: int, include_p9_content: bool = True +) -> list[str]: + """Extract body paragraphs from tags[start_idx] to tags[end_idx] (exclusive). + + Includes p6 paragraphs (main body) and optionally p9 paragraphs that + contain actual content (Sanskrit verses, section markers) but skips + p9 paragraphs that are dividers or page headers. + """ + paragraphs = [] + body_classes = {"p6"} + if include_p9_content: + body_classes.add("p9") + + for i in range(start_idx, end_idx): + tag = tags[i] + css = get_css_class(tag) + + if css not in body_classes: + continue + + if is_empty(tag): + continue + + # For p9 tags, skip dividers and page headers + if css == "p9": + if is_divider(tag) or is_page_header(tag): + continue + + html = clean_html_content(tag) + if html and html != " ": + paragraphs.append(html) + + return paragraphs + + +def extract_body_paragraphs_extended( + tags: list[Tag], + start_idx: int, + end_idx: int, + body_classes: set[str] | None = None, + skip_editor_notes: bool = False, +) -> list[str]: + """Extract body paragraphs from tags[start_idx] to tags[end_idx] (exclusive). + + Like extract_body_paragraphs but with configurable body classes and + optional editor-note skipping. + """ + if body_classes is None: + body_classes = {"p6"} + + paragraphs = [] + + for i in range(start_idx, end_idx): + tag = tags[i] + css = get_css_class(tag) + + if css not in body_classes: + continue + + if is_empty(tag): + continue + + # For p9 tags, skip dividers and page headers + if css == "p9": + if is_divider(tag) or is_page_header(tag): + continue + + # Skip editor notes if requested + if skip_editor_notes and is_editor_note(tag): + continue + + html = clean_html_content(tag) + if html and html != " ": + paragraphs.append(html) + + return paragraphs + + +def extract_essay_1(tags: list[Tag]) -> dict: + """Extract 'The Spiritual Masters' essay. + + Start:

(The Spiritual Masters).

+ Body: p6 and p9 paragraphs after start + End: Next

(contains [BTG 1960]) + """ + # Find the title tag + start_idx = find_tag_index( + tags, + 0, + lambda t: get_css_class(t) == "p8" and "Spiritual Masters" in t.get_text(), + ) + if start_idx is None: + raise ValueError("Could not find Essay 1 start marker (p8 'Spiritual Masters')") + + # Find the end marker: next p10 + end_idx = find_tag_index(tags, start_idx + 1, lambda t: get_css_class(t) == "p10") + if end_idx is None: + raise ValueError("Could not find Essay 1 end marker (p10)") + + print(f" Essay 1: tags[{start_idx}] → tags[{end_idx}] (exclusive)") + + # Body starts after the p8 title tag + paragraphs = extract_body_paragraphs(tags, start_idx + 1, end_idx) + + essay = dict(ESSAYS[0]) + essay["paragraphs"] = paragraphs + return essay + + +def extract_essay_2(tags: list[Tag]) -> dict: + """Extract 'The Absolute Truth' essay. + + Start:

(Second Part)

+ Body: p6 paragraphs after start + End: Next

after start + """ + # Find the start marker + start_idx = find_tag_index( + tags, + 0, + lambda t: get_css_class(t) == "p9" and "(Second Part)" in t.get_text(), + ) + if start_idx is None: + raise ValueError("Could not find Essay 2 start marker (p9 '(Second Part)')") + + # Find the end marker: next p11 + end_idx = find_tag_index(tags, start_idx + 1, lambda t: get_css_class(t) == "p11") + if end_idx is None: + raise ValueError("Could not find Essay 2 end marker (p11)") + + print(f" Essay 2: tags[{start_idx}] → tags[{end_idx}] (exclusive)") + + # Body starts after the "(Second Part)" header + paragraphs = extract_body_paragraphs(tags, start_idx + 1, end_idx) + + essay = dict(ESSAYS[1]) + essay["paragraphs"] = paragraphs + return essay + + +def extract_essay_3(tags: list[Tag]) -> dict: + """Extract 'The Causes of Lord Caitanya's Incarnation' essay. + + Start:

containing "Essays on Shri Chaitanya Charitamrita. III" + Body: p6 paragraphs after start + End: Next

after start. + The

____

divider is skipped. + But the

Thus ends the III part...

IS included. + """ + # Find the start marker — note the ". III" with period + start_idx = find_tag_index( + tags, + 0, + lambda t: ( + get_css_class(t) == "p9" + and "Essays on Shri Chaitanya Charitamrita. III" in t.get_text() + ), + ) + if start_idx is None: + raise ValueError( + "Could not find Essay 3 start marker (p9 'Essays on Shri Chaitanya Charitamrita. III')" + ) + + # Find the end marker: next p11 + end_idx = find_tag_index(tags, start_idx + 1, lambda t: get_css_class(t) == "p11") + if end_idx is None: + raise ValueError("Could not find Essay 3 end marker (p11)") + + print(f" Essay 3: tags[{start_idx}] → tags[{end_idx}] (exclusive)") + + # Body starts after the header tag. + # We collect p6 paragraphs. The divider (p9 with underscores) is skipped + # automatically by extract_body_paragraphs. The "Thus ends the III part..." + # paragraph is a p6, so it will be included. + paragraphs = extract_body_paragraphs(tags, start_idx + 1, end_idx) + + essay = dict(ESSAYS[2]) + essay["paragraphs"] = paragraphs + return essay + + +def extract_essay_4(tags: list[Tag]) -> dict: + """Extract 'The Confidential Reasons for Lord Caitanya's Appearance' essay. + + Start:

containing "4th Chapter" + Body: p6 paragraphs after start (skip editor note, skip p4/p5/p29/p30 headers) + End: Next

after start (next chapter header "Ädi-lélä, Chapter 5") + """ + # Find the start marker: p30 containing "4th Chapter" + start_idx = find_tag_index( + tags, + 0, + lambda t: get_css_class(t) == "p30" and "4th Chapter" in t.get_text(), + ) + if start_idx is None: + raise ValueError("Could not find Essay 4 start marker (p30 '4th Chapter')") + + # Find the end marker: next p4 after the start (next chapter header) + end_idx = find_tag_index(tags, start_idx + 1, lambda t: get_css_class(t) == "p4") + if end_idx is None: + raise ValueError("Could not find Essay 4 end marker (p4)") + + print(f" Essay 4: tags[{start_idx}] → tags[{end_idx}] (exclusive)") + + # Collect p6 paragraphs, skipping editor notes + paragraphs = extract_body_paragraphs_extended( + tags, + start_idx + 1, + end_idx, + body_classes={"p6"}, + skip_editor_notes=True, + ) + + essay = dict(ESSAYS[3]) + essay["paragraphs"] = paragraphs + return essay + + +def extract_essay_5(tags: list[Tag]) -> dict: + """Extract 'The Glories of Lord Nityānanda' essay. + + Start:

containing "1.V" (i.e. "Shi Chaitanya Charitamrita 1.V") + Body: p6 and p34 paragraphs after start (skip editor note, skip p4/p5/p31 headers, + skip p9 divider underscores) + End: Next

after start ("Easy Journey To Other Planets") + """ + # Find the start marker: p31 containing "1.V" + start_idx = find_tag_index( + tags, + 0, + lambda t: get_css_class(t) == "p31" and "1.V" in t.get_text(), + ) + if start_idx is None: + raise ValueError("Could not find Essay 5 start marker (p31 containing '1.V')") + + # Find the end marker: next p35 after the start + end_idx = find_tag_index(tags, start_idx + 1, lambda t: get_css_class(t) == "p35") + if end_idx is None: + raise ValueError("Could not find Essay 5 end marker (p35)") + + print(f" Essay 5: tags[{start_idx}] → tags[{end_idx}] (exclusive)") + + # Collect p6 and p34 paragraphs, skipping editor notes and p9 dividers + paragraphs = extract_body_paragraphs_extended( + tags, + start_idx + 1, + end_idx, + body_classes={"p6", "p34"}, + skip_editor_notes=True, + ) + + essay = dict(ESSAYS[4]) + essay["paragraphs"] = paragraphs + return essay + + +# --------------------------------------------------------------------------- +# Quality checks +# --------------------------------------------------------------------------- + + +def quality_check(essay: dict, essay_num: int) -> None: + """Run quality checks on an extracted essay.""" + paragraphs = essay["paragraphs"] + title = essay["title"] + + print(f"\n--- Quality Checks: Essay {essay_num} ({title}) ---") + + # Check paragraph count is reasonable + if len(paragraphs) < 5: + print(f" ⚠ Only {len(paragraphs)} paragraphs (suspiciously low)") + elif len(paragraphs) > 150: + print(f" ⚠ {len(paragraphs)} paragraphs (suspiciously high)") + else: + print(f" ✅ {len(paragraphs)} paragraphs (reasonable)") + + # Check for residual span class artifacts + span_count = sum(1 for p in paragraphs if 'class="' in p and "]+>", "", p).strip() + if plain.startswith("[") and ("proofread" in plain.lower() or "—Ed" in plain): + print(f" ⚠ Editor note leaked into paragraphs") + break + else: + print(f" ✅ No editor notes leaked into paragraphs") + + # Check that divider lines were excluded + for p in paragraphs: + plain = re.sub(r"<[^>]+>", "", p).strip() + if re.match(r"^_+$", plain): + print(f" ⚠ Divider line leaked into paragraphs") + break + else: + print(f" ✅ No divider lines leaked into paragraphs") + + # Essay-specific checks + if essay_num == 1: + if "spiritual master" in all_text.lower(): + print(f" ✅ 'spiritual master' found in text") + else: + print(f" ⚠ 'spiritual master' not found") + + # Check that reference line [BTG 1960] was excluded + for p in paragraphs: + plain = re.sub(r"<[^>]+>", "", p).strip() + if "BTG 1960" in plain and plain.startswith("["): + print(f" ⚠ Reference line '[BTG 1960]' leaked into paragraphs") + break + else: + print(f" ✅ Reference line properly excluded") + + # Check that title was excluded + for p in paragraphs: + plain = re.sub(r"<[^>]+>", "", p).strip() + if ( + plain == "(The Spiritual Masters)." + or plain == "(The Spiritual Masters)." + ): + print(f" ⚠ Title line leaked into paragraphs") + break + else: + print(f" ✅ Title line properly excluded") + + elif essay_num == 2: + if "impersonal Brahman" in all_text or "Absolute Truth" in all_text: + print(f" ✅ 'Absolute Truth' / 'impersonal Brahman' found") + else: + print(f" ⚠ Key terms for Essay 2 not found") + + # Check "(Second Part)" header was excluded + for p in paragraphs: + plain = re.sub(r"<[^>]+>", "", p).strip() + if plain == "(Second Part)": + print(f" ⚠ Header '(Second Part)' leaked into paragraphs") + break + else: + print(f" ✅ Header '(Second Part)' properly excluded") + + elif essay_num == 3: + if "incarnation" in all_text.lower() or "descent" in all_text.lower(): + print(f" ✅ 'incarnation'/'descent' found") + else: + print(f" ⚠ Key terms for Essay 3 not found") + + # Check that "Thus ends the III part..." is included + found_ending = False + for p in paragraphs: + if "Thus ends the III part" in p: + found_ending = True + break + if found_ending: + print(f" ✅ 'Thus ends the III part...' paragraph included") + else: + print(f" ⚠ 'Thus ends the III part...' paragraph missing") + + elif essay_num == 4: + if "Radharani" in all_text or "confidential" in all_text.lower(): + print(f" ✅ 'Radharani'/'confidential' found") + else: + print(f" ⚠ Key terms for Essay 4 not found") + + # Check that the summary paragraph is included (last paragraph) + found_summary = False + for p in paragraphs: + if "author then summarize" in p.lower(): + found_summary = True + break + if found_summary: + print(f" ✅ Summary paragraph ('The author then summarize...') included") + else: + print(f" ⚠ Summary paragraph missing") + + # Check that the first body paragraph mentions "three principal purposes" + if paragraphs: + if "three principal purposes" in paragraphs[0]: + print(f" ✅ First paragraph mentions 'three principal purposes'") + else: + print(f" ⚠ First paragraph doesn't mention 'three principal purposes'") + + # Check that p4/p5/p29/p30 header lines were excluded + for p in paragraphs: + plain = re.sub(r"<[^>]+>", "", p).strip() + if "4th Chapter" in plain and len(plain) < 50: + print(f" ⚠ Header '4th Chapter' leaked into paragraphs") + break + else: + print(f" ✅ Header lines properly excluded") + + elif essay_num == 5: + if "Nityananda" in all_text: + print(f" ✅ 'Nityananda' found") + else: + print(f" ⚠ 'Nityananda' not found") + + # Check that the final p34 description paragraph is included + if paragraphs: + last_plain = re.sub(r"<[^>]+>", "", paragraphs[-1]).strip() + if ( + "lusture" in last_plain + or "turban" in last_plain + or "wonderful feature" in last_plain + ): + print(f" ✅ Final description paragraph (p34) included") + else: + print(f" ⚠ Final description paragraph (p34) may be missing") + print(f" Last paragraph preview: {last_plain[:80]}...") + + # Check that the first body paragraph mentions "5th chapter" + if paragraphs: + if "5th chapter" in paragraphs[0].lower(): + print(f" ✅ First paragraph mentions '5th chapter'") + else: + print(f" ⚠ First paragraph doesn't mention '5th chapter'") + + # Check that p31 header was excluded + for p in paragraphs: + plain = re.sub(r"<[^>]+>", "", p).strip() + if "Shi Chaitanya Charitamrita 1.V" in plain: + print( + f" ⚠ Header 'Shi Chaitanya Charitamrita 1.V' leaked into paragraphs" + ) + break + else: + print(f" ✅ Header lines properly excluded") + + # Sample output + print(f"\n First paragraph (preview):") + if paragraphs: + preview = re.sub(r"<[^>]+>", "", paragraphs[0])[:120] + print(f" {preview}...") + print(f" Last paragraph (preview):") + if paragraphs: + last = re.sub(r"<[^>]+>", "", paragraphs[-1])[:120] + print(f" {last}...") + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def main(): + base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html") + output_dir = os.path.join(base_dir, "lib", "jsons", "essays") + + if not os.path.exists(html_path): + print(f"ERROR: {html_path} not found!") + sys.exit(1) + + print(f"Reading {html_path}...") + tags = parse_all_paragraphs(html_path) + print(f" Parsed {len(tags)}

tags total\n") + + # Extract all five essays + print("Locating essays...") + essay_1 = extract_essay_1(tags) + essay_2 = extract_essay_2(tags) + essay_3 = extract_essay_3(tags) + essay_4 = extract_essay_4(tags) + essay_5 = extract_essay_5(tags) + + all_essays = [essay_1, essay_2, essay_3, essay_4, essay_5] + + # Write JSON files + print(f"\nWriting JSON files...") + for essay in all_essays: + output_path = os.path.join(output_dir, f"{essay['slug']}.json") + write_json(essay, output_path) + print(f" Written: {output_path}") + + # Quality checks + for i, essay in enumerate(all_essays, 1): + quality_check(essay, i) + + # Summary + print(f"\n{'=' * 60}") + print(f" EXTRACTION SUMMARY") + print(f"{'=' * 60}") + for i, essay in enumerate(all_essays, 1): + print( + f" Essay {i}: {essay['title']}" + f" ({len(essay['paragraphs'])} paragraphs, {essay['year']})" + ) + print( + f" slug={essay['slug']}, group={essay['group']}, order={essay['order']}" + ) + print(f"{'=' * 60}") + print(f" Total essays: {len(all_essays)}") + print(f" Total paragraphs: {sum(len(e['paragraphs']) for e in all_essays)}") + print(f"\nDone! ✨") + + +if __name__ == "__main__": + main() diff --git a/scripts/jaladuta_diary_extract.py b/scripts/jaladuta_diary_extract.py new file mode 100644 index 00000000..21b26ab4 --- /dev/null +++ b/scripts/jaladuta_diary_extract.py @@ -0,0 +1,688 @@ +#!/usr/bin/env python3 +"""Extract The Jaladuta Diary (1965) from others-before-1977.html. + +Each diary day entry becomes its own chapter file. +Chapters are grouped by month in the intro.json metadata. + +Structure: + - Preface and Introduction are standalone chapters (slugs "preface", "introduction") + - Day entries use slugs like "apr-3", "may-2", "aug-13", "sep-1", etc. + - The "Memoranda" entry at the end uses slug "memoranda" + - Months are parent groups in intro.json (non-clickable headings) + +Output: + lib/jsons/journals/jaladuta-diary/intro.json + lib/jsons/journals/jaladuta-diary/chapters/preface.json + lib/jsons/journals/jaladuta-diary/chapters/introduction.json + lib/jsons/journals/jaladuta-diary/chapters/apr-3.json + lib/jsons/journals/jaladuta-diary/chapters/apr-5.json + ... + lib/jsons/journals/jaladuta-diary/chapters/nov-28.json + lib/jsons/journals/jaladuta-diary/chapters/memoranda.json + +Usage: + python3 scripts/jaladuta_diary_extract.py +""" + +import json +import os +import re +import sys + +from bs4 import BeautifulSoup, Tag + +# --------------------------------------------------------------------------- +# ScaGoudy → IAST Unicode mapping +# --------------------------------------------------------------------------- +SCAGOUDY_TO_IAST = { + "ä": "ā", + "Ä": "Ā", + "å": "ṛ", + "Å": "Ṛ", + "ñ": "ṣ", + "Ñ": "Ṣ", + "ö": "ṭ", + "Ö": "Ṭ", + "ë": "ṇ", + "Ë": "Ṇ", + "ò": "ḍ", + "Ò": "Ḍ", + "ç": "ś", + "Ç": "Ś", + "ì": "ṅ", + "Ì": "Ṅ", + "ï": "ñ", + "Ï": "Ñ", + "ü": "ū", + "Ü": "Ū", + "é": "ī", + "É": "Ī", + "à": "ṁ", + "ù": "ḥ", + "è": "ṝ", + "ÿ": "ḷ", +} + + +def decode_scagoudy(text: str) -> str: + """Replace ScaGoudy-encoded characters with proper IAST Unicode.""" + for src, dst in SCAGOUDY_TO_IAST.items(): + text = text.replace(src, dst) + return text + + +def get_css_class(tag: Tag) -> str: + classes = tag.get("class", []) + return classes[0] if classes else "" + + +def inner_html(tag: Tag) -> str: + """Get inner HTML of a tag, decode ScaGoudy, clean up spans.""" + html = "".join(str(child) for child in tag.children) + html = decode_scagoudy(html) + # text → text + html = re.sub( + r']*class="s3"[^>]*>(.*?)', r"\1", html, flags=re.DOTALL + ) + # ... → keep inner text + html = re.sub( + r']*class="s5"[^>]*>(.*?)', r"\1", html, flags=re.DOTALL + ) + # ... → keep inner text + html = re.sub( + r']*class="s1[0-9]+"[^>]*>(.*?)', r"\1", html, flags=re.DOTALL + ) + # ... → keep inner text + html = re.sub( + r']*class="s2[0-9]+"[^>]*>(.*?)', r"\1", html, flags=re.DOTALL + ) + # Remove any remaining spans but keep content + html = re.sub(r"]*>(.*?)", r"\1", html, flags=re.DOTALL) + # Clean up Apple-tab-span leftovers + html = html.replace('\t', "\t") + # Collapse multiple spaces (but not around
) + html = re.sub(r"[ \t]+", " ", html) + html = html.strip() + # Remove leading
tags + html = re.sub(r"^([\s]*)+", "", html) + return html + + +def write_json(data: dict, path: str): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + f.write("\n") + + +# --------------------------------------------------------------------------- +# Month abbreviation helpers +# --------------------------------------------------------------------------- +MONTH_ABBR = { + "april": "apr", + "may": "may", + "june": "jun", + "july": "jul", + "august": "aug", + "september": "sep", + "october": "oct", + "november": "nov", +} + +MONTH_ORDER = ["apr", "may", "jun", "jul", "aug", "sep", "oct", "nov"] + +DAY_NAMES = { + "monday", + "tuesday", + "wednesday", + "thursday", + "friday", + "saturday", + "sunday", +} + + +def parse_day_header(bold_text: str): + """Parse the bold header of a diary entry to extract day name and date number. + + Handles formats like: + "SATURDAY 3" + "13 FRIDAY" + "SATURDAY" (date-less, like Aug 14) + "MEMORANDA" + Returns (day_name, date_number) or (None, None) if not a day entry. + """ + text = bold_text.strip().split("\n")[0].strip() + text = re.sub(r"", " ", text).strip() + + if text.upper() == "MEMORANDA": + return ("MEMORANDA", None) + + # Try "DAY NUMBER" pattern + m = re.match(r"([A-Za-z]+)\s+(\d+)", text) + if m: + day = m.group(1).upper() + num = int(m.group(2)) + if day in {d.upper() for d in DAY_NAMES}: + return (day, num) + + # Try "NUMBER DAY" pattern + m = re.match(r"(\d+)\s+([A-Za-z]+)", text) + if m: + num = int(m.group(1)) + day = m.group(2).upper() + if day in {d.upper() for d in DAY_NAMES}: + return (day, num) + + # Try just "DAY" (no number — like SATURDAY 14 where 14 is on next line) + word = text.split()[0].upper() if text.split() else "" + if word in {d.upper() for d in DAY_NAMES}: + # Try to find number after + rest = text[len(word) :].strip() + m2 = re.match(r"(\d+)", rest) + if m2: + return (word, int(m2.group(1))) + return (word, None) + + return (None, None) + + +# --------------------------------------------------------------------------- +# Main extraction +# --------------------------------------------------------------------------- +def main(): + base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html") + output_dir = os.path.join(base_dir, "lib", "jsons", "journals", "jaladuta-diary") + chapters_dir = os.path.join(output_dir, "chapters") + + if not os.path.exists(html_path): + print(f"ERROR: {html_path} not found!") + sys.exit(1) + + print(f"Reading {html_path}...") + with open(html_path, "r", encoding="utf-8") as f: + content = f.read() + + soup = BeautifulSoup(content, "html.parser") + all_tags = soup.find_all("p") + print(f" Parsed {len(all_tags)}

tags total") + + # Find the Jaladuta Diary section boundaries + start_idx = None + end_idx = None + for i, tag in enumerate(all_tags): + text = tag.get_text().strip() + cls = get_css_class(tag) + if cls == "p35" and "Jaladuta Diary" in text: + start_idx = i + if start_idx is not None and i > start_idx: + # End at "Prayers Written by Śrīla Prabhupāda aboard the Jaladuta" or the next p4 bookmark after JD + if cls == "p4" and "JD 8" in text: + end_idx = i + break + + if start_idx is None: + print("ERROR: Could not find Jaladuta Diary start") + sys.exit(1) + if end_idx is None: + # Fallback: use the memoranda end + for i, tag in enumerate(all_tags): + if ( + i > start_idx + and "MEMORANDA" in tag.get_text() + and get_css_class(tag) == "p110" + ): + end_idx = i + 1 + break + + print(f" Section: tags[{start_idx}] → tags[{end_idx}]") + + section_tags = all_tags[start_idx:end_idx] + + # --------------------------------------------------------------------------- + # Phase 1: Extract Preface and Introduction + # --------------------------------------------------------------------------- + preface_paragraphs = [] + intro_paragraphs = [] + current_section = None # "preface" | "introduction" | None + + # Also track where months/diary entries begin + diary_start_idx = None + + for i, tag in enumerate(section_tags): + cls = get_css_class(tag) + text = tag.get_text().strip() + + # Detect Preface header + if cls == "p3" and text == "Preface": + current_section = "preface" + continue + if cls == "p153": # bookmark + continue + + # Detect Introduction header + if cls in ("p11", "p157") and "Introduction" in text: + if cls == "p157": + current_section = "introduction" + continue + elif cls == "p11": + current_section = "introduction" + continue + + # Detect month start (diary entries begin) + if cls == "p11" and re.match(r"JD \d+:", text): + diary_start_idx = i + current_section = None + break + + if cls == "p29": + # Bold month header like "APRIL 1965" — this is diary territory + diary_start_idx = i + current_section = None + break + + # Collect preface/intro paragraphs + if current_section == "preface" and cls in ("p154", "p155", "p156"): + html = inner_html(tag) + if html: + preface_paragraphs.append(html) + elif current_section == "introduction" and cls in ("p154", "p155", "p156"): + html = inner_html(tag) + if html: + intro_paragraphs.append(html) + + print(f"\n Preface: {len(preface_paragraphs)} paragraphs") + print(f" Introduction: {len(intro_paragraphs)} paragraphs") + + # --------------------------------------------------------------------------- + # Phase 2: Extract diary entries grouped by month + # --------------------------------------------------------------------------- + # Walk through remaining tags, tracking current month + + current_month = None # "apr", "may", etc. + current_month_full = None # "April 1965", etc. + + # Section headers that appear mid-month + section_headers = {} # Maps tag index to header text + + # All day entries: list of { month, day, date, slug, title, paragraphs } + day_entries = [] + current_entry = None + + MONTH_MAP = { + "APRIL": ("apr", "April 1965"), + "MAY": ("may", "May 1965"), + "JUNE": ("jun", "June 1965"), + "JULY": ("jul", "July 1965"), + "AUGUST": ("aug", "August 1965"), + "SEPTEMBER": ("sep", "September 1965"), + "OCTOBER": ("oct", "October 1965"), + "NOVEMBER": ("nov", "November 1965"), + } + + def finalize_entry(): + nonlocal current_entry + if current_entry and current_entry["paragraphs"]: + day_entries.append(current_entry) + current_entry = None + + diary_tags = section_tags[diary_start_idx:] if diary_start_idx else [] + + for i, tag in enumerate(diary_tags): + cls = get_css_class(tag) + text = tag.get_text().strip() + + # Skip bookmarks + if cls in ("p11", "p4", "p39", "p153"): + continue + + # Detect month headers (p29 bold) + if cls == "p29": + html = inner_html(tag) + # Check if this is a month header like "APRIL 1965" or "MAY 1965" + for month_key, (abbr, full) in MONTH_MAP.items(): + if month_key in text.upper(): + finalize_entry() + current_month = abbr + current_month_full = full + break + else: + # Could be a section header like "Verse 1." — skip if not month + # Or it could be a section header within a month + pass + continue + + # Detect section headers (p68 class — "The Journey", "America") + if cls == "p68": + finalize_entry() + html = inner_html(tag) + # Create a section-header entry + if html and current_month: + section_entry = { + "month": current_month, + "month_full": current_month_full, + "day": None, + "date": None, + "slug": None, + "title": None, + "paragraphs": [f"{re.sub('<[^>]+>', ' ', html).strip()}"], + "is_section_header": True, + } + # Extract title text + plain = re.sub("<[^>]+>", " ", html) + plain = re.sub(r"\s+", " ", plain).strip() + section_entry["title"] = plain + day_entries.append(section_entry) + continue + + # Detect diary entries (p110 class) + if cls == "p110": + finalize_entry() + + html = inner_html(tag) + if not html: + continue + + # Parse the bold header + b_tag = tag.find("b") + if b_tag: + bold_text = b_tag.get_text().strip() + day_name, date_num = parse_day_header(bold_text) + else: + day_name, date_num = (None, None) + + if day_name == "MEMORANDA": + current_entry = { + "month": current_month, + "month_full": current_month_full, + "day": "MEMORANDA", + "date": None, + "slug": "memoranda", + "title": "Memoranda", + "paragraphs": [html], + "is_section_header": False, + } + elif day_name and date_num and current_month: + slug = f"{current_month}-{date_num}" + title = f"{day_name.capitalize()} {date_num}" + current_entry = { + "month": current_month, + "month_full": current_month_full, + "day": day_name, + "date": date_num, + "slug": slug, + "title": title, + "paragraphs": [html], + "is_section_header": False, + } + elif day_name and current_month: + # Day without date number (rare — like "SATURDAY" alone on Aug 14) + # Try to infer from context or just skip + slug = f"{current_month}-{day_name.lower()}" + title = day_name.capitalize() + current_entry = { + "month": current_month, + "month_full": current_month_full, + "day": day_name, + "date": None, + "slug": slug, + "title": title, + "paragraphs": [html], + "is_section_header": False, + } + else: + # Can't parse — treat as continuation or standalone + current_entry = { + "month": current_month, + "month_full": current_month_full, + "day": None, + "date": None, + "slug": None, + "title": text[:50], + "paragraphs": [html], + "is_section_header": False, + } + continue + + # Detect p12 paragraphs (continuation text in some entries like Sep 16) + if cls == "p12" and current_entry: + html = inner_html(tag) + if html: + current_entry["paragraphs"].append(html) + continue + + # p155 (source citations within diary section) + if cls == "p155" and current_entry: + html = inner_html(tag) + if html: + current_entry["paragraphs"].append(html) + continue + + finalize_entry() + + print(f"\n Total day entries extracted: {len(day_entries)}") + + # Count by month + month_counts = {} + for entry in day_entries: + m = entry["month"] + month_counts[m] = month_counts.get(m, 0) + 1 + for m in MONTH_ORDER: + if m in month_counts: + print(f" {m}: {month_counts[m]} entries") + + # --------------------------------------------------------------------------- + # Phase 3: Fix up entries — handle dateless entries, dedup slugs + # --------------------------------------------------------------------------- + + # Some entries have no date — try to fix known cases + # Aug "SATURDAY" without number is Aug 14 (follows Aug 13 FRIDAY) + for i, entry in enumerate(day_entries): + if entry.get("is_section_header"): + continue + if entry["date"] is None and entry["day"] and entry["day"] != "MEMORANDA": + # Try to infer from previous entry + if i > 0: + prev = day_entries[i - 1] + if prev["date"] and prev["month"] == entry["month"]: + inferred = prev["date"] + 1 + entry["date"] = inferred + entry["slug"] = f"{entry['month']}-{inferred}" + entry["title"] = f"{entry['day'].capitalize()} {inferred}" + print( + f" Inferred date: {entry['slug']} (from previous {prev['slug']})" + ) + + # Ensure unique slugs + seen_slugs = set() + for entry in day_entries: + if entry.get("is_section_header"): + continue + slug = entry["slug"] + if slug and slug in seen_slugs: + # Append suffix + suffix = 2 + while f"{slug}-{suffix}" in seen_slugs: + suffix += 1 + entry["slug"] = f"{slug}-{suffix}" + print(f" Deduped slug: {slug} → {entry['slug']}") + if slug: + seen_slugs.add(entry["slug"]) + + # Merge section headers into the next entry's paragraphs (prepend) + merged_entries = [] + pending_header = None + for entry in day_entries: + if entry.get("is_section_header"): + pending_header = entry["paragraphs"][0] + continue + if pending_header: + entry["paragraphs"] = [pending_header] + entry["paragraphs"] + pending_header = None + merged_entries.append(entry) + day_entries = merged_entries + + # --------------------------------------------------------------------------- + # Phase 4: Build intro.json with hierarchical month groups + # --------------------------------------------------------------------------- + + # Assign chapter numbers: preface=1, introduction=2, then day entries sequentially + chapter_num = 1 + + all_chapters = [] + + # Preface + preface_chapter = { + "slug": "preface", + "title": "Preface", + "number": chapter_num, + } + all_chapters.append(preface_chapter) + chapter_num += 1 + + # Introduction + intro_chapter = { + "slug": "introduction", + "title": "Introduction", + "number": chapter_num, + } + all_chapters.append(intro_chapter) + chapter_num += 1 + + # Group day entries by month + months_data = [] # List of { month, title, subChapters: [...] } + current_month_group = None + + for entry in day_entries: + month = entry["month"] + month_full = entry["month_full"] + + if current_month_group is None or current_month_group["month"] != month: + current_month_group = { + "month": month, + "title": month_full, + "subChapters": [], + } + months_data.append(current_month_group) + + entry["number"] = chapter_num + current_month_group["subChapters"].append( + { + "slug": entry["slug"], + "title": entry["title"], + } + ) + chapter_num += 1 + + # Build the intro.json chapters array + # Front matter chapters (preface, introduction) are top-level + # Month groups use the hierarchical sub-chapter pattern + intro_chapters = [] + + # Add preface and introduction as simple chapters + intro_chapters.append({"slug": "preface", "title": "Preface"}) + intro_chapters.append({"slug": "introduction", "title": "Introduction"}) + + # Add month groups + for mg in months_data: + intro_chapters.append( + { + "title": mg["title"], + "subChapters": mg["subChapters"], + } + ) + + intro_json = { + "book": "jaladuta-diary", + "title": "The Jaladuta Diary, 1965 — Journey to the U.S.A.", + "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda", + "year": 1965, + "description": "Śrīla Prabhupāda's personal diary kept during his historic voyage from Calcutta to New York aboard the cargo ship Jaladuta in 1965.", + "totalChapters": chapter_num - 1, + "chapters": intro_chapters, + } + + # --------------------------------------------------------------------------- + # Phase 5: Write JSON files + # --------------------------------------------------------------------------- + + # Clean output directory + if os.path.exists(chapters_dir): + for f in os.listdir(chapters_dir): + os.remove(os.path.join(chapters_dir, f)) + + # Write intro.json + write_json(intro_json, os.path.join(output_dir, "intro.json")) + print(f"\n Written: intro.json ({len(intro_chapters)} top-level entries)") + + # Write preface + write_json( + { + "book": "jaladuta-diary", + "chapter": 1, + "title": "Preface", + "type": "preface", + "slug": "preface", + "paragraphs": preface_paragraphs, + }, + os.path.join(chapters_dir, "preface.json"), + ) + print(f" Written: preface.json ({len(preface_paragraphs)} paragraphs)") + + # Write introduction + write_json( + { + "book": "jaladuta-diary", + "chapter": 2, + "title": "Introduction", + "type": "introduction", + "slug": "introduction", + "paragraphs": intro_paragraphs, + }, + os.path.join(chapters_dir, "introduction.json"), + ) + print(f" Written: introduction.json ({len(intro_paragraphs)} paragraphs)") + + # Write day entries + ch_num = 3 + for entry in day_entries: + slug = entry["slug"] + if not slug: + print(f" ⚠ Skipping entry with no slug: {entry.get('title', '?')}") + continue + + chapter_data = { + "book": "jaladuta-diary", + "chapter": ch_num, + "title": entry["title"], + "type": "chapter", + "slug": slug, + "paragraphs": entry["paragraphs"], + } + write_json(chapter_data, os.path.join(chapters_dir, f"{slug}.json")) + ch_num += 1 + + total_files = 2 + len([e for e in day_entries if e.get("slug")]) + print(f"\n Total chapter files written: {total_files}") + + # --------------------------------------------------------------------------- + # Summary + # --------------------------------------------------------------------------- + print(f"\n{'=' * 60}") + print(f" EXTRACTION SUMMARY") + print(f"{'=' * 60}") + print(f" Preface: {len(preface_paragraphs)} paragraphs") + print(f" Introduction: {len(intro_paragraphs)} paragraphs") + print(f" Day entries: {len(day_entries)}") + for mg in months_data: + print(f" {mg['title']}: {len(mg['subChapters'])} entries") + for sc in mg["subChapters"]: + print(f" - {sc['slug']}: {sc['title']}") + print(f" Total chapters: {ch_num - 1}") + print(f"{'=' * 60}") + print(f"\nDone! ✨") + + +if __name__ == "__main__": + main() diff --git a/scripts/mog_extract.py b/scripts/mog_extract.py new file mode 100644 index 00000000..67bd8af0 --- /dev/null +++ b/scripts/mog_extract.py @@ -0,0 +1,539 @@ +#!/usr/bin/env python3 +"""Extract Message of Godhead (1955) from others-before-1977.html. + +The book has 1 intro segment (front-pages) and 3 chapters: + 1. Message of Godhead (Introductory) + 2. Knowledge Transcendental + 3. Work with Transcendental Effect (Karma Yoga) + +Output: + lib/jsons/books/mog/intro.json — Book metadata + lib/jsons/books/mog/intro/*.json — Intro segments (front-pages) + lib/jsons/books/mog/chapters/*.json — Chapters (01–03) + +Usage: + python3 scripts/mog_extract.py +""" + +import json +import os +import re +import sys + +from bs4 import BeautifulSoup, Tag + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def decode_html_content(tag: Tag) -> str: + """Extract inner HTML from a tag, preserve formatting tags, strip spans.""" + inner = "".join(str(child) for child in tag.children) + # Remove span tags with CSS classes (RTF conversion artifacts) + inner = re.sub( + r']*class="[^"]*"[^>]*>(.*?)', r"\1", inner, flags=re.DOTALL + ) + # Clean up Apple-tab-span leftovers (tab characters) + inner = inner.replace("\t", " ") + # Normalize whitespace + inner = re.sub(r"\s+", " ", inner).strip() + return inner + + +def decode_text(tag: Tag) -> str: + """Get plain text from a tag.""" + return tag.get_text().strip() + + +def get_css_class(tag: Tag) -> str: + """Get the first CSS class of a tag, or empty string.""" + classes = tag.get("class", []) + return classes[0] if classes else "" + + +def is_empty(tag: Tag) -> bool: + """Check if a tag is empty or whitespace-only.""" + text = tag.get_text().strip() + return not text or text == "\xa0" + + +# --------------------------------------------------------------------------- +# CSS class categorization for MOG +# --------------------------------------------------------------------------- + +# Front page classes +FRONT_TITLE_CLASS = "p3" # Book title header (bold) +FRONT_INFO_CLASS = "p19" # Author/address info +FRONT_DATE_CLASS = "p15" # Date line ("From the 30/8/55") + +# Chapter heading classes +CHAPTER_BOOKMARK_CLASS = ( + "p11" # Chapter bookmark headers (like "MESSAGE OF GODHEAD (Introductory)") +) +CHAPTER_TITLE_CLASS = "p5" # Chapter title (bold, duplicate of p11 — skip) + +# Body paragraph classes +BODY_CLASSES = frozenset(["p6", "p38", "p44", "p45", "p46", "p47"]) + +# Classes to skip entirely (empty/decorative/break) +SKIP_CLASSES = frozenset(["p43", "p24"]) + +# Boundary detection: classes that mark the start of a different work after MOG +NEXT_WORK_BOUNDARY_CLASSES = frozenset(["p2", "p3"]) + + +# --------------------------------------------------------------------------- +# Boundary detection +# --------------------------------------------------------------------------- + + +def find_mog_boundaries(all_paragraphs: list) -> tuple: + """Find the start and end paragraph indices of the MOG content. + + Start:

containing "Message of Godhead" + End: next

or

that starts a different book, + or

containing "Bhakti-rasāmṛta-sindhu" (the next work) + """ + start_idx = None + end_idx = len(all_paragraphs) + + for i, p in enumerate(all_paragraphs): + cls = get_css_class(p) + text = decode_text(p) + + # Find start: p3 containing "Message of Godhead" + if start_idx is None: + if cls == "p2" and "Message of Godhead" in text: + # The p2 is the TOC entry; the actual start is the next p3 + start_idx = i + continue + continue + + # Find end: look for the next work boundary + # The p6 containing "Bhakti-rasāmṛta-sindhu" starts a different work + if cls == "p6" and "Bhakti-ras" in text and "sindhu" in text: + end_idx = i + break + + # Also check for p2 that starts a different book section + if cls == "p2" and start_idx is not None and i > start_idx + 5: + text_lower = text.lower() + if "message of godhead" not in text_lower: + end_idx = i + break + + if start_idx is None: + print("ERROR: Could not find 'Message of Godhead' in others-before-1977.html!") + sys.exit(1) + + return start_idx, end_idx + + +def find_chapter_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list: + """Find all MOG chapter bookmark tags (p11) in the MOG range. + + Returns list of (paragraph_index, chapter_title). + """ + bookmarks = [] + for i in range(start_idx, end_idx): + p = all_paragraphs[i] + cls = get_css_class(p) + if cls == CHAPTER_BOOKMARK_CLASS: + text = decode_text(p).strip() + if text: + bookmarks.append((i, text)) + return bookmarks + + +# --------------------------------------------------------------------------- +# Paragraph extraction +# --------------------------------------------------------------------------- + + +def extract_paragraphs( + all_paragraphs: list, + start_idx: int, + end_idx: int, + is_front_pages: bool = False, +) -> list: + """Extract paragraphs for a section (front pages or chapter body). + + For front pages: collect p3, p19, p15 lines as paragraphs. + For chapters: collect body paragraphs (p6, p38, p44, p45, p46, p47). + Always skip p43, p24 (empty/break), p11 (bookmark), p5 (duplicate heading). + """ + result = [] + + for i in range(start_idx, end_idx): + p = all_paragraphs[i] + cls = get_css_class(p) + + # Skip empty paragraphs + if is_empty(p): + continue + + # Skip known empty/break classes + if cls in SKIP_CLASSES: + continue + + # Skip chapter bookmark headers (p11) — used for boundary detection only + if cls == CHAPTER_BOOKMARK_CLASS: + continue + + # Skip chapter title duplicate (p5) + if cls == CHAPTER_TITLE_CLASS: + continue + + # Skip p2 (TOC entry at start) + if cls == "p2": + continue + + if is_front_pages: + # For front pages, collect title, author/address, and date lines + if cls in (FRONT_TITLE_CLASS, FRONT_INFO_CLASS, FRONT_DATE_CLASS): + html = decode_html_content(p) + if html: + result.append(html) + continue + # Also include any other non-skip class if we encounter it + # (shouldn't happen much for front pages, but be permissive) + if cls in BODY_CLASSES: + # Body paragraph before first chapter = still front pages + html = decode_html_content(p) + if html: + result.append(html) + continue + else: + # For chapter body, collect all body paragraph classes + if cls in BODY_CLASSES: + html = decode_html_content(p) + if html: + result.append(html) + continue + + # p19 can appear inside chapters (e.g., "(Devanāgarī)" in chapter 2) + if cls == FRONT_INFO_CLASS: + html = decode_html_content(p) + if html: + result.append(html) + continue + + # Title page class inside chapter body (shouldn't happen, but be safe) + if cls == FRONT_TITLE_CLASS: + html = decode_html_content(p) + if html: + result.append(html) + continue + + # Fallback: try to extract content from unknown classes + html = decode_html_content(p) + if html: + result.append(html) + + return result + + +# --------------------------------------------------------------------------- +# Main extraction logic +# --------------------------------------------------------------------------- + + +def extract_mog(soup: BeautifulSoup) -> dict: + """Extract the entire Message of Godhead from the parsed HTML. + + Returns a dict with: + "intro_segments": list of intro segment dicts + "chapters": list of chapter dicts + "chapter_listing": list of {number, title} for intro.json + """ + all_paragraphs = list(soup.find_all("p")) + + # Step 1: Find MOG boundaries + mog_start, mog_end = find_mog_boundaries(all_paragraphs) + print(f" MOG content: paragraphs {mog_start} to {mog_end}") + print(f" Total paragraphs in range: {mog_end - mog_start}") + + # Step 2: Find all chapter bookmarks (p11) + bookmarks = find_chapter_bookmarks(all_paragraphs, mog_start, mog_end) + print(f" Found {len(bookmarks)} MOG chapter bookmarks:") + for pos, title in bookmarks: + print(f" Paragraph {pos}: {title}") + + # Step 3: Map bookmarks to chapters with proper titles + # Expected: 3 chapters + # Map raw bookmark text (case-insensitive) to normalized titles + chapter_titles = { + "message of godhead": "Message of Godhead (Introductory)", + "knowledge transcendental": "Knowledge Transcendental", + "work with transcendental effect": "Work with Transcendental Effect (Karma Yoga)", + } + + chapter_bookmarks = [] # (para_idx, chapter_num, normalized_title) + ch_num = 1 + for pos, raw_title in bookmarks: + # Try to match known chapter titles (case-insensitive) + matched_title = None + raw_lower = raw_title.lower().strip() + for key, nice_title in chapter_titles.items(): + if key in raw_lower: + matched_title = nice_title + break + + if matched_title is None: + # Fallback: use the raw title with title case + matched_title = raw_title.strip() + + chapter_bookmarks.append((pos, ch_num, matched_title)) + ch_num += 1 + + print(f"\n Mapped {len(chapter_bookmarks)} chapters:") + for pos, num, title in chapter_bookmarks: + print(f" Chapter {num}: {title} (paragraph {pos})") + + # Step 4: Extract front pages + # From MOG start to first chapter bookmark + intro_segments = [] + + if chapter_bookmarks: + front_pages_end = chapter_bookmarks[0][0] + else: + front_pages_end = mog_end + + front_pages_paragraphs = extract_paragraphs( + all_paragraphs, mog_start, front_pages_end, is_front_pages=True + ) + intro_segments.append( + { + "slug": "front-pages", + "title": "Front Pages", + "type": "intro-segment", + "book": "mog", + "order": 1, + "paragraphs": front_pages_paragraphs, + } + ) + + # Step 5: Extract chapters + chapters = [] + for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_bookmarks): + # Content starts after the bookmark line + # But we also need to skip the p5 duplicate heading that follows p11 + content_start = ch_pos + 1 + + # Content ends at next chapter bookmark or mog_end + if idx + 1 < len(chapter_bookmarks): + content_end = chapter_bookmarks[idx + 1][0] + else: + content_end = mog_end + + paragraphs = extract_paragraphs(all_paragraphs, content_start, content_end) + + ch = { + "book": "mog", + "chapter": ch_num, + "title": ch_title, + "type": "chapter", + "paragraphs": paragraphs, + } + chapters.append(ch) + + # Step 6: Build chapter listing for intro.json + chapter_listing = [ + {"number": ch["chapter"], "title": ch["title"]} for ch in chapters + ] + + return { + "intro_segments": intro_segments, + "chapters": chapters, + "chapter_listing": chapter_listing, + } + + +# --------------------------------------------------------------------------- +# Output +# --------------------------------------------------------------------------- + + +def write_json(data, filepath): + """Write data as formatted JSON to filepath, creating directories as needed.""" + os.makedirs(os.path.dirname(filepath), exist_ok=True) + with open(filepath, "w", encoding="utf-8") as f: + json.dump(data, f, ensure_ascii=False, indent=2) + + +def main(): + base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html") + output_base = os.path.join(base_dir, "lib", "jsons", "books", "mog") + + if not os.path.exists(html_path): + print(f"ERROR: {html_path} not found!") + sys.exit(1) + + print(f"Reading {html_path}...") + with open(html_path, "r", encoding="utf-8") as f: + soup = BeautifulSoup(f, "html.parser") + + print("Extracting Message of Godhead (1955)...\n") + result = extract_mog(soup) + + intro_segments = result["intro_segments"] + chapters = result["chapters"] + chapter_listing = result["chapter_listing"] + + # --- Write intro.json --- + intro_meta = { + "book": "mog", + "title": "Message of Godhead", + "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda", + "year": 1955, + "description": "An early handwritten work presenting the essence of Bhagavad-gītā's teachings on transcendental knowledge, karma-yoga, and the Personality of Godhead.", + "totalChapters": 3, + "chapters": chapter_listing, + } + intro_path = os.path.join(output_base, "intro.json") + write_json(intro_meta, intro_path) + print(f"\n Written: {intro_path}") + + # --- Write intro segments --- + print(f"\n--- Intro Segments ---") + for seg in intro_segments: + filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json") + write_json(seg, filepath) + print( + f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}" + ) + + # --- Write chapters --- + print(f"\n--- Chapters ---") + for ch in chapters: + ch_num_str = str(ch["chapter"]).zfill(2) + filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json") + write_json(ch, filepath) + print( + f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — " + f"{ch['title']}" + ) + + # --- Summary --- + total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments) + total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters) + total_paras = total_intro_paras + total_chapter_paras + + print(f"\n{'=' * 60}") + print(f" EXTRACTION SUMMARY") + print(f"{'=' * 60}") + print(f" Intro segments: {len(intro_segments)}") + for seg in intro_segments: + print( + f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs" + ) + print(f" Chapters: {len(chapters)}") + for ch in chapters: + print( + f" Ch {ch['chapter']}: {len(ch['paragraphs'])} paragraphs — {ch['title']}" + ) + print(f" Total paragraphs: {total_paras}") + print(f" Intro: {total_intro_paras}") + print(f" Chapters: {total_chapter_paras}") + if chapters: + print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}") + print(f" Output directory: {output_base}/") + + # --- Samples --- + print(f"\n--- Sample: Front Pages ---") + if intro_segments: + front = intro_segments[0] + for i, p in enumerate(front["paragraphs"][:5]): + preview = re.sub(r"<[^>]+>", "", p)[:120] + print(f" [{i}] {preview}") + + if chapters: + print(f"\n--- Sample: Chapter 1 ({chapters[0]['title']}) ---") + ch1 = chapters[0] + for i, p in enumerate(ch1["paragraphs"][:3]): + preview = re.sub(r"<[^>]+>", "", p)[:120] + print(f" [{i}] {preview}...") + + if len(chapters) >= 2: + print(f"\n--- Sample: Chapter 2 ({chapters[1]['title']}) ---") + ch2 = chapters[1] + for i, p in enumerate(ch2["paragraphs"][:3]): + preview = re.sub(r"<[^>]+>", "", p)[:120] + print(f" [{i}] {preview}...") + + if len(chapters) >= 3: + print(f"\n--- Sample: Chapter 3 ({chapters[2]['title']}) ---") + ch3 = chapters[2] + for i, p in enumerate(ch3["paragraphs"][:3]): + preview = re.sub(r"<[^>]+>", "", p)[:120] + print(f" [{i}] {preview}...") + + # --- Quality Checks --- + print(f"\n--- Quality Checks ---") + + # Check all 3 chapters present + expected_chapters = set(range(1, 4)) + actual_chapters = set(ch["chapter"] for ch in chapters) + missing = expected_chapters - actual_chapters + extra = actual_chapters - expected_chapters + if missing: + print(f" ⚠ Missing chapters: {sorted(missing)}") + if extra: + print(f" ⚠ Extra chapters: {sorted(extra)}") + if not missing and not extra: + print(f" ✅ All 3 chapters present") + + # Check intro segments + if intro_segments: + print( + f" ✅ Front pages segment present ({len(intro_segments[0]['paragraphs'])} paragraphs)" + ) + else: + print(f" ⚠ Front pages segment missing!") + + # Check empty chapters + empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0] + if empty_chapters: + print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}") + else: + print(f" ✅ No empty chapters") + + # Check for "——The Author." in chapter 1 + if chapters: + ch1_text = " ".join(chapters[0]["paragraphs"]) + if "The Author" in ch1_text: + print(f" ✅ '——The Author.' attribution found in Chapter 1") + else: + print(f" ⚠ '——The Author.' attribution NOT found in Chapter 1") + + # Check chapter 2 has "(Devanāgarī)" line + if len(chapters) >= 2: + ch2_text = " ".join(chapters[1]["paragraphs"]) + if "Devan" in ch2_text: + print(f" ✅ '(Devanāgarī)' line found in Chapter 2") + else: + print(f" ℹ '(Devanāgarī)' line not found in Chapter 2 (may be normal)") + + # Verify no Bhakti-rasāmṛta-sindhu content leaked in + all_text = "" + for ch in chapters: + all_text += " ".join(ch["paragraphs"]) + if "Text 22" in all_text or "BHAKTI RASAMRITA" in all_text.upper(): + print(f" ⚠ Content from Bhakti-rasāmṛta-sindhu may have leaked into MOG!") + else: + print(f" ✅ No content leak from next work detected") + + # Check last chapter ends cleanly + if chapters: + last_ch = chapters[-1] + last_para = last_ch["paragraphs"][-1] if last_ch["paragraphs"] else "" + last_preview = re.sub(r"<[^>]+>", "", last_para)[:80] + print(f' ℹ Last paragraph of Ch {last_ch["chapter"]}: "{last_preview}..."') + + print(f"\nDone! ✨") + + +if __name__ == "__main__": + main() diff --git a/scripts/od_analysis_bg_extract.py b/scripts/od_analysis_bg_extract.py new file mode 100644 index 00000000..07f76443 --- /dev/null +++ b/scripts/od_analysis_bg_extract.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python3 +"""Extract "An Analysis of Bhagwat Geeta" from others-before-1977.html. + +This document is a structured analysis organized into three numbered sections +(topics) with numbered points and verse references. It is plain English text +(no ScaGoudy decoding needed). + +Source: public/html/others-before-1977.html + - Title:

An Analysis of Bhagwat Geeta

(line ~15182) + - Section 1: "Mans eternal relation with Godhead" — 45

items + - Section 2: "Man's Duty Toward Godhead" — 34

items + - Section 3: "Man's ultimate necessity and perfection of life" — 3

items + - End boundary:

Solution of Present Crisises... (line ~15276) + +CSS classes: + p116 — Title (bold) + p39 — Section headers ("1: Mans eternal relation with Godhead") + p5 — Duplicate section title (bold, with ) — SKIP + p6 — Body paragraphs (plain English) + p34 — Math calculation at the end — SKIP + p11 — Next document title (end boundary) + +Output (prose book structure): + lib/jsons/other-documents/analysis-of-bhagavad-gita/ + intro.json # metadata + chapters/ + 01.json # Chapter 1 + 02.json # Chapter 2 + 03.json # Chapter 3 + +Usage: + python3 scripts/od_analysis_bg_extract.py +""" + +import json +import os +import re +import sys + +from bs4 import BeautifulSoup, Tag + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +BOOK_CODE = "od-analysis-bg" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def get_css_class(tag: Tag) -> str: + """Get the first CSS class of a tag, or empty string.""" + classes = tag.get("class", []) + return classes[0] if classes else "" + + +def is_empty(tag: Tag) -> bool: + """Check if a tag is empty or whitespace-only.""" + text = tag.get_text().strip() + return not text or text == "\xa0" + + +def clean_text(tag: Tag) -> str: + """Extract inner HTML from a tag, strip span class artifacts, normalize whitespace.""" + inner = "".join(str(child) for child in tag.children) + # Remove span tags with CSS classes (RTF conversion artifacts) but keep their content + inner = re.sub( + r']*class="[^"]*"[^>]*>(.*?)', r"\1", inner, flags=re.DOTALL + ) + # Remove any remaining bare or tags + inner = re.sub(r"]*>", "", inner) + # Remove and tags (not needed for body paragraphs) + inner = re.sub(r"", "", inner) + # Clean up Apple-tab-span leftovers (tab characters) + inner = inner.replace("\t", " ") + # Strip
tags + inner = re.sub(r"", " ", inner) + # Normalize whitespace + inner = re.sub(r"\s+", " ", inner).strip() + # Fix HTML entities + inner = inner.replace("&", "&") + return inner + + +def write_json(data, filepath): + """Write data as formatted JSON to filepath, creating directories as needed.""" + os.makedirs(os.path.dirname(filepath), exist_ok=True) + with open(filepath, "w", encoding="utf-8") as f: + json.dump(data, f, ensure_ascii=False, indent=2) + print(f" ✅ Wrote {filepath}") + + +# --------------------------------------------------------------------------- +# Extraction +# --------------------------------------------------------------------------- + + +def extract_chapters(soup: BeautifulSoup) -> list[dict]: + """Extract the chapter data from 'An Analysis of Bhagwat Geeta'.""" + + # Find the title tag:

An Analysis of Bhagwat Geeta

+ title_tag = None + for p in soup.find_all("p", class_="p116"): + text = p.get_text().strip() + if "Analysis of Bhagwat Geeta" in text: + title_tag = p + break + + if not title_tag: + print("❌ Could not find title 'An Analysis of Bhagwat Geeta'") + sys.exit(1) + + print(f" 📍 Found title: {title_tag.get_text().strip()}") + + # Collect all

tags after the title until we hit the end boundary + chapters = [] + current_chapter = None + + tag = title_tag.find_next_sibling() + while tag: + if not isinstance(tag, Tag) or tag.name != "p": + tag = tag.find_next_sibling() + continue + + css = get_css_class(tag) + text = tag.get_text().strip() + + # End boundary: next document starts + if css == "p11" and "Solution of Present Crisises" in text: + break + + # Skip empty tags + if is_empty(tag): + tag = tag.find_next_sibling() + continue + + # Section header:

1: Mans eternal relation with Godhead

+ if css == "p39": + # Parse section number and title + match = re.match(r"(\d+):\s*(.+)", text) + if match: + num = int(match.group(1)) + title = match.group(2).strip() + current_chapter = { + "number": num, + "title": title, + "paragraphs": [], + } + chapters.append(current_chapter) + print(f" 📖 Section {num}: {title}") + tag = tag.find_next_sibling() + continue + + # Skip p5 (duplicate section title in bold with span) + if css == "p5": + tag = tag.find_next_sibling() + continue + + # Skip p34 (math calculation at the end) + if css == "p34": + tag = tag.find_next_sibling() + continue + + # Body paragraph:

+ if css == "p6" and current_chapter is not None: + cleaned = clean_text(tag) + if cleaned: + current_chapter["paragraphs"].append(cleaned) + tag = tag.find_next_sibling() + continue + + # Any other class — skip + tag = tag.find_next_sibling() + + return chapters + + +# --------------------------------------------------------------------------- +# Quality check +# --------------------------------------------------------------------------- + + +def quality_check(chapters: list[dict]): + """Validate the extracted chapters.""" + print("\n🔍 Quality check:\n") + + print(f" Chapters: {len(chapters)}") + assert len(chapters) == 3, f"Expected 3 chapters, got {len(chapters)}" + + total_paragraphs = 0 + for ch in chapters: + count = len(ch["paragraphs"]) + total_paragraphs += count + print(f' Chapter {ch["number"]}: "{ch["title"]}" — {count} paragraphs') + + # Spot-check first paragraph + if ch["paragraphs"]: + first = ch["paragraphs"][0] + print(f" First: {first[:80]}...") + + # Spot-check last paragraph + if len(ch["paragraphs"]) > 1: + last = ch["paragraphs"][-1] + print(f" Last: {last[:80]}...") + + print(f"\n Total paragraphs: {total_paragraphs}") + + # Expected counts (from manual inspection): + # Section 1: 45 items (numbered 1-45) + # Section 2: 34 items (numbered 1-34) + # Section 3: 3 items (numbered 1-3) + expected_total = 82 # 45 + 34 + 3 + if total_paragraphs == expected_total: + print(f" ✅ Paragraph count matches expected: {expected_total}") + else: + print(f" ⚠️ Expected {expected_total} paragraphs, got {total_paragraphs}") + # This is a warning, not a hard failure — some items may have sub-items + # or the HTML structure may slightly differ + + # Check that paragraphs are not empty + for ch in chapters: + for i, p in enumerate(ch["paragraphs"]): + assert p.strip(), f"Empty paragraph in chapter {ch['number']}, index {i}" + + print("\n ✅ All checks passed!") + + +# --------------------------------------------------------------------------- +# Output +# --------------------------------------------------------------------------- + + +def write_output(chapters: list[dict], output_dir: str): + """Write intro.json and per-chapter JSON files.""" + + # Build intro.json + intro = { + "book": BOOK_CODE, + "title": "An Analysis of Bhagwat Geeta", + "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda", + "year": None, + "description": "A structured analysis of Bhagavad-gītā organized into three topics with numbered points and verse references.", + "totalChapters": len(chapters), + "chapters": [{"number": ch["number"], "title": ch["title"]} for ch in chapters], + } + + write_json(intro, os.path.join(output_dir, "intro.json")) + + # Write per-chapter files + chapters_dir = os.path.join(output_dir, "chapters") + for ch in chapters: + chapter_data = { + "book": BOOK_CODE, + "chapter": ch["number"], + "title": ch["title"], + "type": "chapter", + "paragraphs": ch["paragraphs"], + } + filename = f"{ch['number']:02d}.json" + write_json(chapter_data, os.path.join(chapters_dir, filename)) + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def main(): + base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html") + output_dir = os.path.join( + base_dir, "lib", "jsons", "other-documents", "analysis-of-bhagavad-gita" + ) + + if not os.path.exists(html_path): + print(f"❌ HTML file not found: {html_path}") + sys.exit(1) + + # Clean up old single-file JSON if it exists + old_json = os.path.join( + base_dir, "lib", "jsons", "other-documents", "analysis-of-bhagavad-gita.json" + ) + if os.path.exists(old_json): + os.remove(old_json) + print(f" 🗑️ Removed old single-file JSON: {old_json}") + + print(f"📂 Reading {html_path}...") + with open(html_path, "r", encoding="utf-8") as f: + soup = BeautifulSoup(f, "html.parser") + + print("\n📝 Extracting 'An Analysis of Bhagwat Geeta'...\n") + chapters = extract_chapters(soup) + + quality_check(chapters) + + write_output(chapters, output_dir) + print(f"\n🎉 Done! Output directory: {output_dir}") + + +if __name__ == "__main__": + main() diff --git a/scripts/pk_extract.py b/scripts/pk_extract.py new file mode 100644 index 00000000..4c3dd67f --- /dev/null +++ b/scripts/pk_extract.py @@ -0,0 +1,533 @@ +#!/usr/bin/env python3 +"""Extract Prayers of King Kulaśekhara (1958) from others-before-1977.html. + +The book has 6 chapters (one per verse/prayer): + 1. Prayer 1 — "Śrī Vallabha" (prayer to Mukunda) + 2. Prayer 2 — "Jayatu Jayatu" (glories to Devakī-nandana) + 3. Prayer 3 — "Mukunda Mūrdhnā" (prayer for remembrance) + 4. Prayer 4 — "Nāhaṁ Vande" (neither heaven nor hell) + 5. Prayer 5 — "Nāsthā Dharme" (unflinching devotion) + 6. Prayer 6 — "Divi Vā Bhuvi" (remembrance at death) + +Output: + lib/jsons/books/pk/intro.json — Book metadata + lib/jsons/books/pk/chapters/*.json — Chapters (01–06) + +Usage: + python3 scripts/pk_extract.py +""" + +import json +import os +import re +import sys + +from bs4 import BeautifulSoup, Tag + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def decode_html_content(tag: Tag) -> str: + """Extract inner HTML from a tag, preserve formatting tags, strip spans.""" + inner = "".join(str(child) for child in tag.children) + # Remove span tags with CSS classes (RTF conversion artifacts) + inner = re.sub( + r']*class="[^"]*"[^>]*>(.*?)', r"\1", inner, flags=re.DOTALL + ) + # Clean up Apple-tab-span leftovers (tab characters) + inner = inner.replace("\t", " ") + # Normalize whitespace + inner = re.sub(r"\s+", " ", inner).strip() + return inner + + +def decode_text(tag: Tag) -> str: + """Get plain text from a tag.""" + return tag.get_text().strip() + + +def get_css_class(tag: Tag) -> str: + """Get the first CSS class of a tag, or empty string.""" + classes = tag.get("class", []) + return classes[0] if classes else "" + + +def is_empty(tag: Tag) -> bool: + """Check if a tag is empty or whitespace-only.""" + text = tag.get_text().strip() + return not text or text == "\xa0" + + +# --------------------------------------------------------------------------- +# CSS class categorization for PK +# --------------------------------------------------------------------------- + +# Document header +DOC_HEADER_CLASS = "p4" # "Prayers of King Kulaçekhara 1958 [Handwritten]" +TITLE_CLASS = "p115" # Bold title + +# Chapter bookmark class +CHAPTER_BOOKMARK_CLASS = "p17" # "Prayers of King Kulaçekhara 1950: N" + +# Verse number markers — skip +VERSE_MARKER_CLASSES = frozenset(["p108", "p9"]) + +# Devanagari marker — skip +DEVANAGARI_MARKER_CLASS = "p36" + +# Section header class (Wordings., Synonyms., Translation., Purport.) +SECTION_HEADER_CLASSES = frozenset(["p20", "p121"]) + +# Body paragraph classes +BODY_CLASSES = frozenset(["p6", "p34", "p38", "p71"]) + +# Page number class — skip +PAGE_NUMBER_CLASSES = frozenset(["p15", "p41", "p124"]) + +# Empty/break classes — skip +SKIP_CLASSES = frozenset(["p24", "p94", "p119", "p122", "p123", "p120"]) + +# Next work boundary +NEXT_WORK_BOUNDARY_CLASS = "p125" + + +# --------------------------------------------------------------------------- +# Boundary detection +# --------------------------------------------------------------------------- + + +def find_pk_boundaries(all_elements: list) -> tuple: + """Find the start and end element indices of the PK content. + + Start:

containing "Prayers of King Kula" + End:

that starts the next work ("Shaktipujah") + """ + start_idx = None + end_idx = len(all_elements) + + for i, el in enumerate(all_elements): + if el.name != "p": + continue + cls = get_css_class(el) + text = decode_text(el) + + # Find start: p4 containing "Prayers of King" + if start_idx is None: + if cls == DOC_HEADER_CLASS and "Prayers of King" in text: + start_idx = i + continue + continue + + # Find end: p125 that starts a different work + if cls == NEXT_WORK_BOUNDARY_CLASS: + end_idx = i + break + + if start_idx is None: + print( + "ERROR: Could not find 'Prayers of King Kulaçekhara' in others-before-1977.html!" + ) + sys.exit(1) + + return start_idx, end_idx + + +def find_chapter_bookmarks(all_elements: list, start_idx: int, end_idx: int) -> list: + """Find all PK chapter bookmark tags (p17) in the PK range. + + Returns list of (element_index, verse_number). + """ + bookmarks = [] + for i in range(start_idx, end_idx): + el = all_elements[i] + if el.name != "p": + continue + cls = get_css_class(el) + if cls == CHAPTER_BOOKMARK_CLASS: + text = decode_text(el).strip() + if "Prayers of King" in text or "Kulaçekhara" in text.replace("ç", "ç"): + # Extract the verse number from the end + match = re.search(r":\s*(\d+)\s*$", text) + if match: + verse_num = int(match.group(1)) + bookmarks.append((i, verse_num)) + return bookmarks + + +# --------------------------------------------------------------------------- +# Table synonym extraction +# --------------------------------------------------------------------------- + + +def extract_synonyms_from_table(table: Tag) -> str: + """Extract synonyms from a and convert to text. + + Each cell contains a

with "word=meaning" format. + Convert to "word—meaning; word—meaning; ..." format. + """ + entries = [] + for row in table.find_all("tr"): + for cell in row.find_all("td"): + # Get all

tags inside the cell + for p in cell.find_all("p"): + text = decode_text(p).strip() + if not text or text == "\xa0": + continue + # Must contain '=' to be a synonym entry + if "=" not in text: + continue + # Replace = with em-dash + text = text.replace("=", "—", 1) + # Clean trailing whitespace/period if needed + text = text.strip() + entries.append(text) + + if not entries: + return "" + return "; ".join(entries) + + +# --------------------------------------------------------------------------- +# Paragraph extraction for a chapter +# --------------------------------------------------------------------------- + + +def extract_chapter_paragraphs( + all_elements: list, + start_idx: int, + end_idx: int, +) -> list: + """Extract paragraphs for a chapter (verse). + + Handles section headers (Wordings., Synonyms., Translation., Purport.), + body paragraphs, and synonym tables. + + Returns a list of HTML paragraph strings. + """ + result = [] + + # Build a set of

tags that are inside tables, so we don't double-process + table_p_tags = set() + for i in range(start_idx, end_idx): + el = all_elements[i] + if el.name == "table": + for p in el.find_all("p"): + table_p_tags.add(id(p)) + + for i in range(start_idx, end_idx): + el = all_elements[i] + + # Handle tables (synonyms) + if el.name == "table": + table_cls = get_css_class(el) + if table_cls == "t1": + synonyms_text = extract_synonyms_from_table(el) + if synonyms_text: + result.append(synonyms_text) + continue + + # Only process

tags from here + if el.name != "p": + continue + + # Skip

tags that are inside tables (already processed) + if id(el) in table_p_tags: + continue + + cls = get_css_class(el) + + # Skip empty paragraphs + if is_empty(el): + continue + + # Skip page numbers + if cls in PAGE_NUMBER_CLASSES: + continue + + # Skip verse markers + if cls in VERSE_MARKER_CLASSES: + text = decode_text(el) + # p9 can also be "(Devanägaré)" or prayer number markers — skip all + continue + + # Skip devanagari markers + if cls == DEVANAGARI_MARKER_CLASS: + continue + + # Skip chapter bookmarks + if cls == CHAPTER_BOOKMARK_CLASS: + continue + + # Skip document header and title (only at very start) + if cls == DOC_HEADER_CLASS: + continue + if cls == TITLE_CLASS: + continue + + # Skip empty/break classes + if cls in SKIP_CLASSES: + continue + + # Section headers: Wordings., Synonyms., Translation., Purport. + if cls in SECTION_HEADER_CLASSES: + text = decode_text(el).strip() + # Normalize section header names + text_lower = text.lower().rstrip(".") + if "wording" in text_lower: + result.append("Wordings") + elif ( + "synonym" in text_lower + or "synomym" in text_lower + or "synonysm" in text_lower + ): + result.append("Synonyms") + elif "translation" in text_lower: + result.append("Translation") + elif "purport" in text_lower: + result.append("Purport") + else: + # Unknown header — include as bold + html = decode_html_content(el) + if html: + result.append(f"{html}") + continue + + # Body paragraphs + if cls in BODY_CLASSES: + html = decode_html_content(el) + if html: + result.append(html) + continue + + # Fallback: try to extract content from unknown classes + html = decode_html_content(el) + if html: + result.append(html) + + return result + + +# --------------------------------------------------------------------------- +# Main extraction logic +# --------------------------------------------------------------------------- + + +def extract_pk(soup: BeautifulSoup) -> dict: + """Extract the entire Prayers of King Kulaśekhara from the parsed HTML. + + Returns a dict with: + "chapters": list of chapter dicts + "chapter_listing": list of {number, title} for intro.json + """ + # We need both

and

elements in document order + all_elements = list(soup.find_all(["p", "table"])) + + # Step 1: Find PK boundaries + pk_start, pk_end = find_pk_boundaries(all_elements) + print(f" PK content: elements {pk_start} to {pk_end}") + print(f" Total elements in range: {pk_end - pk_start}") + + # Step 2: Find all chapter bookmarks (p17) + bookmarks = find_chapter_bookmarks(all_elements, pk_start, pk_end) + print(f" Found {len(bookmarks)} PK chapter bookmarks:") + for pos, verse_num in bookmarks: + print(f" Element {pos}: Verse {verse_num}") + + # If we didn't find 6 bookmarks, also look for the first verse that starts + # before any p17 bookmark (verse 1 may start with p108 [1]) + # Actually, verse 1's bookmark IS in the HTML. Let's verify. + if len(bookmarks) < 6: + print(f" WARNING: Expected 6 bookmarks, found {len(bookmarks)}") + + # Step 3: Build chapter ranges + # Each bookmark marks the START of a verse section. + # Content for verse N runs from bookmark N to bookmark N+1 (or pk_end). + chapters = [] + for idx, (bk_pos, verse_num) in enumerate(bookmarks): + # Content starts after the bookmark + content_start = bk_pos + 1 + + # Content ends at next bookmark or pk_end + if idx + 1 < len(bookmarks): + content_end = bookmarks[idx + 1][0] + else: + content_end = pk_end + + paragraphs = extract_chapter_paragraphs( + all_elements, content_start, content_end + ) + + ch = { + "book": "pk", + "chapter": verse_num, + "title": f"Prayer {verse_num}", + "type": "chapter", + "paragraphs": paragraphs, + } + chapters.append(ch) + + # Step 4: Build chapter listing for intro.json + chapter_listing = [ + {"number": ch["chapter"], "title": ch["title"]} for ch in chapters + ] + + return { + "chapters": chapters, + "chapter_listing": chapter_listing, + } + + +# --------------------------------------------------------------------------- +# Output +# --------------------------------------------------------------------------- + + +def write_json(data, filepath): + """Write data as formatted JSON to filepath, creating directories as needed.""" + os.makedirs(os.path.dirname(filepath), exist_ok=True) + with open(filepath, "w", encoding="utf-8") as f: + json.dump(data, f, ensure_ascii=False, indent=2) + + +def main(): + base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html") + output_base = os.path.join(base_dir, "lib", "jsons", "books", "pk") + + if not os.path.exists(html_path): + print(f"ERROR: {html_path} not found!") + sys.exit(1) + + print(f"Reading {html_path}...") + with open(html_path, "r", encoding="utf-8") as f: + soup = BeautifulSoup(f, "html.parser") + + print("Extracting Prayers of King Kulaśekhara (1958)...\n") + result = extract_pk(soup) + + chapters = result["chapters"] + chapter_listing = result["chapter_listing"] + + # --- Write intro.json --- + intro_meta = { + "book": "pk", + "title": "Prayers of King Kulaśekhara (1958)", + "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda", + "year": 1958, + "description": "Early handwritten translations and purports of the Mukunda-mālā-stotra prayers (verses 1–6).", + "totalChapters": 6, + "chapters": chapter_listing, + } + intro_path = os.path.join(output_base, "intro.json") + write_json(intro_meta, intro_path) + print(f"\n Written: {intro_path}") + + # --- Write chapters --- + print(f"\n--- Chapters ---") + for ch in chapters: + ch_num_str = str(ch["chapter"]).zfill(2) + filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json") + write_json(ch, filepath) + print( + f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — " + f"{ch['title']}" + ) + + # --- Summary --- + total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters) + + print(f"\n{'=' * 60}") + print(f" EXTRACTION SUMMARY") + print(f"{'=' * 60}") + print(f" Chapters: {len(chapters)}") + for ch in chapters: + print( + f" Ch {ch['chapter']}: {len(ch['paragraphs'])} paragraphs — {ch['title']}" + ) + print(f" Total paragraphs: {total_chapter_paras}") + if chapters: + print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}") + print(f" Output directory: {output_base}/") + + # --- Samples --- + for ch_idx, ch in enumerate(chapters): + print(f"\n--- Sample: Chapter {ch['chapter']} ({ch['title']}) ---") + for i, p in enumerate(ch["paragraphs"][:5]): + preview = re.sub(r"<[^>]+>", "", p)[:120] + print(f" [{i}] {preview}") + if len(ch["paragraphs"]) > 5: + print(f" ... ({len(ch['paragraphs'])} total paragraphs)") + + # --- Quality Checks --- + print(f"\n--- Quality Checks ---") + + # Check all 6 chapters present + expected_chapters = set(range(1, 7)) + actual_chapters = set(ch["chapter"] for ch in chapters) + missing = expected_chapters - actual_chapters + extra = actual_chapters - expected_chapters + if missing: + print(f" ⚠ Missing chapters: {sorted(missing)}") + if extra: + print(f" ⚠ Extra chapters: {sorted(extra)}") + if not missing and not extra: + print(f" ✅ All 6 chapters present") + + # Check empty chapters + empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0] + if empty_chapters: + print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}") + else: + print(f" ✅ No empty chapters") + + # Check each chapter has expected sections + for ch in chapters: + ch_text = " ".join(ch["paragraphs"]) + has_wordings = "Wordings" in ch_text + has_synonyms = "Synonyms" in ch_text + has_translation = "Translation" in ch_text + has_purport = "Purport" in ch_text + sections = [] + if has_wordings: + sections.append("W") + if has_synonyms: + sections.append("S") + if has_translation: + sections.append("T") + if has_purport: + sections.append("P") + status = "✅" if len(sections) == 4 else "⚠" + print(f" {status} Ch {ch['chapter']}: sections [{', '.join(sections)}]") + + # Check synonym entries use em-dash format + synonym_count = 0 + for ch in chapters: + for p in ch["paragraphs"]: + if "—" in p and "" not in p: + synonym_count += 1 + break + print(f" ℹ Chapters with em-dash synonyms: {synonym_count}/6") + + # Verify no Shaktipujah content leaked in + all_text = "" + for ch in chapters: + all_text += " ".join(ch["paragraphs"]) + if "Shaktipujah" in all_text or "Baidya Nath" in all_text: + print(f" ⚠ Content from Shaktipujah may have leaked into PK!") + else: + print(f" ✅ No content leak from next work detected") + + # Check last chapter ends cleanly + if chapters: + last_ch = chapters[-1] + last_para = last_ch["paragraphs"][-1] if last_ch["paragraphs"] else "" + last_preview = re.sub(r"<[^>]+>", "", last_para)[:80] + print(f' ℹ Last paragraph of Ch {last_ch["chapter"]}: "{last_preview}..."') + + print(f"\nDone! ✨") + + +if __name__ == "__main__": + main() diff --git a/scripts/seed-bk.ts b/scripts/seed-bk.ts new file mode 100644 index 00000000..6c69c817 --- /dev/null +++ b/scripts/seed-bk.ts @@ -0,0 +1,169 @@ +/** + * seed-bk.ts — Push Bhagavāner Kathā JSON data into MongoDB. + * + * Uses the native MongoDB driver (same pattern as seed-cbhc.ts) to avoid + * Prisma replica-set write requirements. + * + * BK is a simple prose book with no volume structure and no intro segments. + * We use volume=0 as the sentinel for "no volumes". + * + * Collections written: + * - book_meta → one doc { book: "bk", data: } + * - prose_chapters → 2 chapters + * + * Usage: + * npx tsx scripts/seed-bk.ts + */ + +import dotenv from "dotenv"; +import fs from "fs"; +import path from "path"; +import { MongoClient } from "mongodb"; + +// Load env +dotenv.config({ path: ".env.local" }); +dotenv.config({ path: ".env" }); + +const DATABASE_URL = process.env.DATABASE_URL; +if (!DATABASE_URL) { + console.error("❌ DATABASE_URL is not set in .env"); + process.exit(1); +} + +const BASE_DIR = path.resolve(__dirname, ".."); +const BK_DIR = path.join(BASE_DIR, "lib", "jsons", "books", "bk"); + +/** Volume sentinel for books without volumes. */ +const NO_VOLUME = 0; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function readJson(filePath: string): T { + return JSON.parse(fs.readFileSync(filePath, "utf-8")); +} + +/** + * Insert documents one at a time with a small delay every N docs. + * Uses insertOne to avoid BSON wire-protocol corruption. + */ +async function insertBatched( + col: ReturnType["collection"]>, + docs: Record[], +) { + for (let i = 0; i < docs.length; i++) { + await col.insertOne(docs[i]); + if ((i + 1) % 100 === 0) { + await new Promise((r) => setTimeout(r, 100)); + } + } +} + +// --------------------------------------------------------------------------- +// Types for the JSON files +// --------------------------------------------------------------------------- + +type BkIntroJson = { + book: string; + title: string; + author: string; + year: number | null; + description: string; + totalChapters: number; + chapters: { number: number; title: string }[]; +}; + +type BkChapterJson = { + book: string; + chapter: number; + title: string; + type: string; + paragraphs: string[]; +}; + +// --------------------------------------------------------------------------- +// Seed function +// --------------------------------------------------------------------------- + +async function seedBk(db: ReturnType) { + const introPath = path.join(BK_DIR, "intro.json"); + if (!fs.existsSync(introPath)) { + console.error("❌ BK intro.json not found at", introPath); + process.exit(1); + } + + const intro = readJson(introPath); + + // ---- 1. Book meta ---- + const metaCol = db.collection("book_meta"); + await metaCol.deleteMany({ book: "bk" }); + await metaCol.insertOne({ book: "bk", data: intro }); + console.log(" 📖 BK book meta inserted"); + + // ---- 2. Prose chapters collection ---- + const proseCol = db.collection("prose_chapters"); + await proseCol.deleteMany({ book: "bk" }); + + const docs: Record[] = []; + + // Chapters (01.json through 02.json) + const chaptersDir = path.join(BK_DIR, "chapters"); + if (fs.existsSync(chaptersDir)) { + const chapterFiles = fs + .readdirSync(chaptersDir) + .filter((f) => f.endsWith(".json")) + .sort(); + + for (const file of chapterFiles) { + const ch = readJson(path.join(chaptersDir, file)); + const chNum = ch.chapter ?? parseInt(file.replace(".json", ""), 10); + docs.push({ + book: "bk", + volume: NO_VOLUME, + chapter: chNum, + title: ch.title, + type: "chapter", + slug: String(chNum), + order: 100 + chNum, + paragraphs: ch.paragraphs, + }); + } + console.log(` 📖 Chapters: ${chapterFiles.length}`); + } + + // Insert all docs + console.log(`\n Inserting ${docs.length} documents...`); + await insertBatched(proseCol, docs); + + const totalDocs = await proseCol.countDocuments({ book: "bk" }); + console.log(` ✅ BK: ${totalDocs} prose entries in MongoDB`); +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function main() { + console.log("🌱 Seeding Bhagavāner Kathā...\n"); + + const client = new MongoClient(DATABASE_URL!); + await client.connect(); + console.log("📡 Connected to MongoDB\n"); + + const db = client.db(); + + const start = Date.now(); + + await seedBk(db); + + const elapsed = ((Date.now() - start) / 1000).toFixed(1); + console.log(`\n🎉 Done in ${elapsed}s`); + + await client.close(); +} + +main().catch((err) => { + console.error("❌ Seed failed:", err); + process.exit(1); +}); diff --git a/scripts/seed-cbhc.ts b/scripts/seed-cbhc.ts new file mode 100644 index 00000000..87a08720 --- /dev/null +++ b/scripts/seed-cbhc.ts @@ -0,0 +1,169 @@ +/** + * seed-cbhc.ts — Push Śrī Caitanya-bhāgavata Commentary JSON data into MongoDB. + * + * Uses the native MongoDB driver (same pattern as seed-mog.ts) to avoid + * Prisma replica-set write requirements. + * + * CBHC is a simple prose book with no volume structure and no intro segments. + * We use volume=0 as the sentinel for "no volumes". + * + * Collections written: + * - book_meta → one doc { book: "cbhc", data: } + * - prose_chapters → 4 chapters + * + * Usage: + * npx tsx scripts/seed-cbhc.ts + */ + +import dotenv from "dotenv"; +import fs from "fs"; +import path from "path"; +import { MongoClient } from "mongodb"; + +// Load env +dotenv.config({ path: ".env.local" }); +dotenv.config({ path: ".env" }); + +const DATABASE_URL = process.env.DATABASE_URL; +if (!DATABASE_URL) { + console.error("❌ DATABASE_URL is not set in .env"); + process.exit(1); +} + +const BASE_DIR = path.resolve(__dirname, ".."); +const CBHC_DIR = path.join(BASE_DIR, "lib", "jsons", "books", "cbhc"); + +/** Volume sentinel for books without volumes. */ +const NO_VOLUME = 0; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function readJson(filePath: string): T { + return JSON.parse(fs.readFileSync(filePath, "utf-8")); +} + +/** + * Insert documents one at a time with a small delay every N docs. + * Uses insertOne to avoid BSON wire-protocol corruption. + */ +async function insertBatched( + col: ReturnType["collection"]>, + docs: Record[], +) { + for (let i = 0; i < docs.length; i++) { + await col.insertOne(docs[i]); + if ((i + 1) % 100 === 0) { + await new Promise((r) => setTimeout(r, 100)); + } + } +} + +// --------------------------------------------------------------------------- +// Types for the JSON files +// --------------------------------------------------------------------------- + +type CbhcIntroJson = { + book: string; + title: string; + author: string; + year: number | null; + description: string; + totalChapters: number; + chapters: { number: number; title: string }[]; +}; + +type CbhcChapterJson = { + book: string; + chapter: number; + title: string; + type: string; + paragraphs: string[]; +}; + +// --------------------------------------------------------------------------- +// Seed function +// --------------------------------------------------------------------------- + +async function seedCbhc(db: ReturnType) { + const introPath = path.join(CBHC_DIR, "intro.json"); + if (!fs.existsSync(introPath)) { + console.error("❌ CBHC intro.json not found at", introPath); + process.exit(1); + } + + const intro = readJson(introPath); + + // ---- 1. Book meta ---- + const metaCol = db.collection("book_meta"); + await metaCol.deleteMany({ book: "cbhc" }); + await metaCol.insertOne({ book: "cbhc", data: intro }); + console.log(" 📖 CBHC book meta inserted"); + + // ---- 2. Prose chapters collection ---- + const proseCol = db.collection("prose_chapters"); + await proseCol.deleteMany({ book: "cbhc" }); + + const docs: Record[] = []; + + // Chapters (01.json through 04.json) + const chaptersDir = path.join(CBHC_DIR, "chapters"); + if (fs.existsSync(chaptersDir)) { + const chapterFiles = fs + .readdirSync(chaptersDir) + .filter((f) => f.endsWith(".json")) + .sort(); + + for (const file of chapterFiles) { + const ch = readJson(path.join(chaptersDir, file)); + const chNum = ch.chapter ?? parseInt(file.replace(".json", ""), 10); + docs.push({ + book: "cbhc", + volume: NO_VOLUME, + chapter: chNum, + title: ch.title, + type: "chapter", + slug: String(chNum), + order: 100 + chNum, + paragraphs: ch.paragraphs, + }); + } + console.log(` 📖 Chapters: ${chapterFiles.length}`); + } + + // Insert all docs + console.log(`\n Inserting ${docs.length} documents...`); + await insertBatched(proseCol, docs); + + const totalDocs = await proseCol.countDocuments({ book: "cbhc" }); + console.log(` ✅ CBHC: ${totalDocs} prose entries in MongoDB`); +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function main() { + console.log("🌱 Seeding Śrī Caitanya-bhāgavata Commentary...\n"); + + const client = new MongoClient(DATABASE_URL!); + await client.connect(); + console.log("📡 Connected to MongoDB\n"); + + const db = client.db(); + + const start = Date.now(); + + await seedCbhc(db); + + const elapsed = ((Date.now() - start) / 1000).toFixed(1); + console.log(`\n🎉 Done in ${elapsed}s`); + + await client.close(); +} + +main().catch((err) => { + console.error("❌ Seed failed:", err); + process.exit(1); +}); diff --git a/scripts/seed-cc-essays.ts b/scripts/seed-cc-essays.ts new file mode 100644 index 00000000..927c7b6f --- /dev/null +++ b/scripts/seed-cc-essays.ts @@ -0,0 +1,163 @@ +/** + * seed-cc-essays.ts — Push all five "Essays on Śrī Caitanya-caritāmṛta" into MongoDB. + * + * Uses the native MongoDB driver (same pattern as seed-te.ts) to avoid + * Prisma replica-set write requirements. + * + * Collections written: + * - essays → 5 essay docs (upsert by slug) + * + * JSON sources: + * - lib/jsons/essays/the-spiritual-masters.json + * - lib/jsons/essays/the-absolute-truth.json + * - lib/jsons/essays/causes-of-lord-caitanyas-incarnation.json + * - lib/jsons/essays/confidential-reasons-for-lord-caitanyas-appearance.json + * - lib/jsons/essays/glories-of-lord-nityananda.json + * + * Usage: + * npx tsx scripts/seed-cc-essays.ts + */ + +import dotenv from "dotenv"; +import fs from "fs"; +import path from "path"; +import { MongoClient } from "mongodb"; + +// Load env +dotenv.config({ path: ".env.local" }); +dotenv.config({ path: ".env" }); + +const DATABASE_URL = process.env.DATABASE_URL; +if (!DATABASE_URL) { + console.error("❌ DATABASE_URL is not set in .env"); + process.exit(1); +} + +const BASE_DIR = path.resolve(__dirname, ".."); +const ESSAYS_DIR = path.join(BASE_DIR, "lib", "jsons", "essays"); + +const ESSAY_FILES = [ + "the-spiritual-masters.json", + "the-absolute-truth.json", + "causes-of-lord-caitanyas-incarnation.json", + "confidential-reasons-for-lord-caitanyas-appearance.json", + "glories-of-lord-nityananda.json", +]; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +type EssayJson = { + slug: string; + title: string; + author: string; + year: number; + group: string; + order: number; + paragraphs: string[]; +}; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function readJson(filePath: string): T { + return JSON.parse(fs.readFileSync(filePath, "utf-8")); +} + +// --------------------------------------------------------------------------- +// Seed function +// --------------------------------------------------------------------------- + +async function seedCcEssays(db: ReturnType) { + const essaysCol = db.collection("essays"); + + let inserted = 0; + let updated = 0; + let unchanged = 0; + + for (const filename of ESSAY_FILES) { + const jsonPath = path.join(ESSAYS_DIR, filename); + + if (!fs.existsSync(jsonPath)) { + console.error(` ❌ Missing: ${jsonPath}`); + console.error(" Run `python3 scripts/cc_essays_extract.py` first."); + process.exit(1); + } + + const essay = readJson(jsonPath); + + const result = await essaysCol.updateOne( + { slug: essay.slug }, + { + $set: { + slug: essay.slug, + title: essay.title, + author: essay.author, + year: essay.year, + group: essay.group, + order: essay.order, + paragraphs: essay.paragraphs, + }, + }, + { upsert: true }, + ); + + if (result.upsertedCount > 0) { + console.log( + ` 📝 Inserted "${essay.title}" (${essay.paragraphs.length} paragraphs)`, + ); + inserted++; + } else if (result.modifiedCount > 0) { + console.log( + ` 📝 Updated "${essay.title}" (${essay.paragraphs.length} paragraphs)`, + ); + updated++; + } else { + console.log(` 📝 "${essay.title}" already up to date`); + unchanged++; + } + } + + // Verify + const totalDocs = await essaysCol.countDocuments({ + group: "cc-essays", + }); + console.log( + `\n ✅ Verified: ${totalDocs} document(s) in essays collection with group "cc-essays"`, + ); + + // Summary + console.log( + `\n Summary: ${inserted} inserted, ${updated} updated, ${unchanged} unchanged`, + ); +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function main() { + console.log("🌱 Seeding CC Essays (5 essays)...\n"); + + const client = new MongoClient(DATABASE_URL!); + await client.connect(); + console.log("📡 Connected to MongoDB\n"); + + const db = client.db(); + + const start = Date.now(); + + await seedCcEssays(db); + + const elapsed = ((Date.now() - start) / 1000).toFixed(1); + console.log(`\n🎉 Done in ${elapsed}s`); + + await client.close(); +} + +main().catch((err) => { + console.error("❌ Seed failed:", err); + process.exit(1); +}); diff --git a/scripts/seed-journals.ts b/scripts/seed-journals.ts new file mode 100644 index 00000000..f7c8854d --- /dev/null +++ b/scripts/seed-journals.ts @@ -0,0 +1,171 @@ +/** + * seed-journals.ts — Push Journals into MongoDB. + * + * Collections written: + * - journal_meta → one doc per journal + * - journal_chapters → N chapter docs per journal + * + * Usage: + * npx tsx scripts/seed-journals.ts + */ + +import dotenv from "dotenv"; +import fs from "fs"; +import path from "path"; +import { MongoClient } from "mongodb"; + +dotenv.config({ path: ".env.local" }); +dotenv.config({ path: ".env" }); + +const DATABASE_URL = process.env.DATABASE_URL; +if (!DATABASE_URL) { + console.error("❌ DATABASE_URL is not set"); + process.exit(1); +} + +const BASE_DIR = path.resolve(__dirname, ".."); +const JOURNALS_DIR = path.join(BASE_DIR, "lib", "jsons", "journals"); + +type IntroJson = { + book: string; + title: string; + author: string; + year: number | null; + description: string | null; + totalChapters: number; + chapters: unknown[]; +}; +type ChapterJson = { + book: string; + chapter: number; + title: string; + type: string; + slug?: string; + paragraphs: string[]; +}; + +function readJson(filePath: string): T { + return JSON.parse(fs.readFileSync(filePath, "utf-8")); +} + +function sanitizeDoc(obj: unknown): unknown { + if (typeof obj === "string") { + let s = obj.normalize("NFC"); + s = s.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, ""); + s = s.replace(/[\uD800-\uDFFF]/g, ""); + s = s.replace(/[\u200B-\u200F\u2028-\u202F\uFEFF]/g, ""); + s = Buffer.from(s, "utf-8").toString("utf-8"); + return s; + } + if (Array.isArray(obj)) return obj.map(sanitizeDoc); + if (obj !== null && typeof obj === "object") { + const result: Record = {}; + for (const [key, value] of Object.entries(obj as Record)) + result[key] = sanitizeDoc(value); + return result; + } + return obj; +} + +async function seedOneJournal( + db: ReturnType, + docDir: string, +): Promise<{ bookCode: string; chapters: number }> { + const introPath = path.join(docDir, "intro.json"); + if (!fs.existsSync(introPath)) throw new Error(`No intro.json in ${docDir}`); + const intro = sanitizeDoc(readJson(introPath)) as IntroJson; + const bookCode = intro.book; + + const metaCol = db.collection("journal_meta"); + await metaCol.deleteMany({ book: bookCode }); + await metaCol.insertOne({ book: bookCode, data: intro }); + console.log(` 📖 Inserted journal_meta for "${intro.title}" (${bookCode})`); + + const chaptersCol = db.collection("journal_chapters"); + await chaptersCol.deleteMany({ book: bookCode }); + + const chaptersDir = path.join(docDir, "chapters"); + let inserted = 0; + if (fs.existsSync(chaptersDir)) { + const files = fs + .readdirSync(chaptersDir) + .filter((f) => f.endsWith(".json")) + .sort(); + for (const file of files) { + const ch = sanitizeDoc( + readJson(path.join(chaptersDir, file)), + ) as ChapterJson; + const chNum = ch.chapter ?? parseInt(file.replace(".json", ""), 10); + const slug = ch.slug ?? String(chNum); + await chaptersCol.insertOne({ + book: bookCode, + chapter: chNum, + title: ch.title, + type: ch.type || "chapter", + slug, + order: 100 + chNum, + paragraphs: ch.paragraphs, + }); + inserted++; + } + console.log(` 📄 Chapters: ${inserted}`); + } + + const total = await chaptersCol.countDocuments({ book: bookCode }); + console.log(` ✅ ${bookCode}: ${total} chapter entries`); + return { bookCode, chapters: inserted }; +} + +async function main() { + console.log("🌱 Seeding Journals...\n"); + const client = new MongoClient(DATABASE_URL!); + await client.connect(); + console.log("📡 Connected to MongoDB\n"); + const db = client.db(); + const start = Date.now(); + + if (!fs.existsSync(JOURNALS_DIR)) { + console.error(`❌ ${JOURNALS_DIR} not found`); + process.exit(1); + } + + const subdirs = fs + .readdirSync(JOURNALS_DIR) + .filter((e) => { + const p = path.join(JOURNALS_DIR, e); + return ( + fs.statSync(p).isDirectory() && + fs.existsSync(path.join(p, "intro.json")) + ); + }) + .sort(); + + if (subdirs.length === 0) { + console.error("❌ No journal directories found"); + process.exit(1); + } + console.log(`📂 Found ${subdirs.length} journal(s)\n`); + + let totalChapters = 0; + const bookCodes: string[] = []; + for (const subdir of subdirs) { + console.log(`\n📝 Processing: ${subdir}`); + const result = await seedOneJournal(db, path.join(JOURNALS_DIR, subdir)); + bookCodes.push(result.bookCode); + totalChapters += result.chapters; + } + + const elapsed = ((Date.now() - start) / 1000).toFixed(1); + console.log("\n" + "─".repeat(50)); + console.log( + `📊 Summary: ${subdirs.length} journals, ${totalChapters} chapters`, + ); + console.log(` Book codes: ${bookCodes.join(", ")}`); + console.log(`\n🎉 Done in ${elapsed}s`); + await client.close(); +} + +main().catch((err) => { + console.error("❌ Seed failed:", err); + process.exit(1); +}); diff --git a/scripts/seed-mog.ts b/scripts/seed-mog.ts new file mode 100644 index 00000000..cf48cf85 --- /dev/null +++ b/scripts/seed-mog.ts @@ -0,0 +1,204 @@ +/** + * seed-mog.ts — Push Message of Godhead JSON data into MongoDB. + * + * Uses the native MongoDB driver (same pattern as seed-mg.ts) to avoid + * Prisma replica-set write requirements. + * + * MOG is a prose book with no volume structure. + * We use volume=0 as the sentinel for "no volumes". + * + * Collections written: + * - book_meta → one doc { book: "mog", data: } + * - prose_chapters → 1 intro segment + 3 chapters = 4 docs + * + * Usage: + * npx tsx scripts/seed-mog.ts + */ + +import dotenv from "dotenv"; +import fs from "fs"; +import path from "path"; +import { MongoClient } from "mongodb"; + +// Load env +dotenv.config({ path: ".env.local" }); +dotenv.config({ path: ".env" }); + +const DATABASE_URL = process.env.DATABASE_URL; +if (!DATABASE_URL) { + console.error("❌ DATABASE_URL is not set in .env"); + process.exit(1); +} + +const BASE_DIR = path.resolve(__dirname, ".."); +const MOG_DIR = path.join(BASE_DIR, "lib", "jsons", "books", "mog"); + +/** Volume sentinel for books without volumes. */ +const NO_VOLUME = 0; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function readJson(filePath: string): T { + return JSON.parse(fs.readFileSync(filePath, "utf-8")); +} + +/** + * Insert documents one at a time with a small delay every N docs. + * Uses insertOne to avoid BSON wire-protocol corruption. + */ +async function insertBatched( + col: ReturnType["collection"]>, + docs: Record[], +) { + for (let i = 0; i < docs.length; i++) { + await col.insertOne(docs[i]); + if ((i + 1) % 100 === 0) { + await new Promise((r) => setTimeout(r, 100)); + } + } +} + +// --------------------------------------------------------------------------- +// Types for the JSON files +// --------------------------------------------------------------------------- + +type MogIntroJson = { + book: string; + title: string; + author: string; + year: number; + description: string; + totalChapters: number; + chapters: { number: number; title: string }[]; +}; + +type MogIntroSegmentJson = { + slug: string; + title: string; + type: string; + book: string; + order: number; + paragraphs: string[]; +}; + +type MogChapterJson = { + book: string; + chapter: number; + title: string; + type: string; + paragraphs: string[]; +}; + +// --------------------------------------------------------------------------- +// Seed function +// --------------------------------------------------------------------------- + +async function seedMog(db: ReturnType) { + const introPath = path.join(MOG_DIR, "intro.json"); + if (!fs.existsSync(introPath)) { + console.error("❌ MOG intro.json not found at", introPath); + process.exit(1); + } + + const intro = readJson(introPath); + + // ---- 1. Book meta ---- + const metaCol = db.collection("book_meta"); + await metaCol.deleteMany({ book: "mog" }); + await metaCol.insertOne({ book: "mog", data: intro }); + console.log(" 📖 MOG book meta inserted"); + + // ---- 2. Prose chapters collection ---- + const proseCol = db.collection("prose_chapters"); + await proseCol.deleteMany({ book: "mog" }); + + const docs: Record[] = []; + + // Intro segments (front-pages) + const introSegmentFiles = ["front-pages.json"]; + + const introDir = path.join(MOG_DIR, "intro"); + for (const file of introSegmentFiles) { + const filePath = path.join(introDir, file); + if (!fs.existsSync(filePath)) { + console.warn(` ⚠️ Intro segment not found, skipping: ${file}`); + continue; + } + + const seg = readJson(filePath); + + docs.push({ + book: "mog", + volume: NO_VOLUME, + chapter: null, + title: seg.title, + type: "intro-segment", + slug: `intro-${seg.slug}`, + order: seg.order, + paragraphs: seg.paragraphs, + }); + } + console.log(` 📄 Intro segments: ${docs.length}`); + + // Chapters (01.json through 03.json) + const chaptersDir = path.join(MOG_DIR, "chapters"); + if (fs.existsSync(chaptersDir)) { + const chapterFiles = fs + .readdirSync(chaptersDir) + .filter((f) => f.endsWith(".json")) + .sort(); + + for (const file of chapterFiles) { + const ch = readJson(path.join(chaptersDir, file)); + const chNum = ch.chapter ?? parseInt(file.replace(".json", ""), 10); + docs.push({ + book: "mog", + volume: NO_VOLUME, + chapter: chNum, + title: ch.title, + type: "chapter", + slug: String(chNum), + order: 100 + chNum, + paragraphs: ch.paragraphs, + }); + } + console.log(` 📖 Chapters: ${chapterFiles.length}`); + } + + // Insert all docs + console.log(`\n Inserting ${docs.length} documents...`); + await insertBatched(proseCol, docs); + + const totalDocs = await proseCol.countDocuments({ book: "mog" }); + console.log(` ✅ MOG: ${totalDocs} prose entries in MongoDB`); +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function main() { + console.log("🌱 Seeding Message of Godhead...\n"); + + const client = new MongoClient(DATABASE_URL!); + await client.connect(); + console.log("📡 Connected to MongoDB\n"); + + const db = client.db(); + + const start = Date.now(); + + await seedMog(db); + + const elapsed = ((Date.now() - start) / 1000).toFixed(1); + console.log(`\n🎉 Done in ${elapsed}s`); + + await client.close(); +} + +main().catch((err) => { + console.error("❌ Seed failed:", err); + process.exit(1); +}); diff --git a/scripts/seed-other-documents.ts b/scripts/seed-other-documents.ts new file mode 100644 index 00000000..7ebee9a5 --- /dev/null +++ b/scripts/seed-other-documents.ts @@ -0,0 +1,274 @@ +/** + * seed-other-documents.ts — Push "Other Documents" into MongoDB. + * + * Uses dedicated collections for other documents (not shared with books): + * - other_document_meta → one doc per document { book: "", data: } + * - other_document_chapters → N chapter docs per document + * + * Each "other document" is a subdirectory under lib/jsons/other-documents/ + * with an intro.json and chapters/*.json. + * + * Book codes have NO prefix (e.g. "analysis-bg", "real-peace-formula"). + * + * Usage: + * npx tsx scripts/seed-other-documents.ts + */ + +import dotenv from "dotenv"; +import fs from "fs"; +import path from "path"; +import { MongoClient } from "mongodb"; + +// Load env +dotenv.config({ path: ".env.local" }); +dotenv.config({ path: ".env" }); + +const DATABASE_URL = process.env.DATABASE_URL; +if (!DATABASE_URL) { + console.error("❌ DATABASE_URL is not set in .env"); + process.exit(1); +} + +const BASE_DIR = path.resolve(__dirname, ".."); +const OTHER_DOCS_DIR = path.join(BASE_DIR, "lib", "jsons", "other-documents"); + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +type IntroJson = { + book: string; + title: string; + author: string; + year: number | null; + description: string | null; + totalChapters: number; + chapters: { number: number; title: string }[]; +}; + +type ChapterJson = { + book: string; + chapter: number; + title: string; + type: string; + paragraphs: string[]; +}; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function readJson(filePath: string): T { + return JSON.parse(fs.readFileSync(filePath, "utf-8")); +} + +/** + * Recursively sanitize all strings in a document to prevent BSON encoding errors. + */ +function sanitizeDoc(obj: unknown): unknown { + if (typeof obj === "string") { + let s = obj.normalize("NFC"); + s = s.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, ""); + s = s.replace(/[\uD800-\uDFFF]/g, ""); + s = s.replace(/[\u200B-\u200F\u2028-\u202F\uFEFF]/g, ""); + s = Buffer.from(s, "utf-8").toString("utf-8"); + return s; + } + if (Array.isArray(obj)) { + return obj.map(sanitizeDoc); + } + if (obj !== null && typeof obj === "object") { + const result: Record = {}; + for (const [key, value] of Object.entries(obj as Record)) { + result[key] = sanitizeDoc(value); + } + return result; + } + return obj; +} + +// --------------------------------------------------------------------------- +// Seed one document +// --------------------------------------------------------------------------- + +async function seedOneDocument( + db: ReturnType, + docDir: string, +): Promise<{ bookCode: string; chapters: number }> { + const introPath = path.join(docDir, "intro.json"); + if (!fs.existsSync(introPath)) { + throw new Error(`No intro.json found in ${docDir}`); + } + + const intro = sanitizeDoc(readJson(introPath)) as IntroJson; + const bookCode = intro.book; + + // ---- Document meta ---- + const metaCol = db.collection("other_document_meta"); + await metaCol.deleteMany({ book: bookCode }); + await metaCol.insertOne({ book: bookCode, data: intro }); + console.log( + ` 📖 Inserted other_document_meta for "${intro.title}" (${bookCode})`, + ); + + // ---- Document chapters ---- + const chaptersCol = db.collection("other_document_chapters"); + await chaptersCol.deleteMany({ book: bookCode }); + + const docs: Record[] = []; + + const chaptersDir = path.join(docDir, "chapters"); + if (fs.existsSync(chaptersDir)) { + const chapterFiles = fs + .readdirSync(chaptersDir) + .filter((f) => f.endsWith(".json")) + .sort(); + + for (const file of chapterFiles) { + const ch = sanitizeDoc( + readJson(path.join(chaptersDir, file)), + ) as ChapterJson; + + const chNum = ch.chapter ?? parseInt(file.replace(".json", ""), 10); + docs.push({ + book: bookCode, + chapter: chNum, + title: ch.title, + type: ch.type || "chapter", + slug: String(chNum), + order: 100 + chNum, + paragraphs: ch.paragraphs, + }); + } + console.log(` 📄 Chapters: ${chapterFiles.length}`); + } + + // Insert all chapter docs + if (docs.length > 0) { + console.log( + ` 💾 Inserting ${docs.length} other_document_chapters docs...`, + ); + for (const doc of docs) { + await chaptersCol.insertOne(doc); + } + } + + const totalDocs = await chaptersCol.countDocuments({ book: bookCode }); + console.log(` ✅ ${bookCode}: ${totalDocs} chapter entries in MongoDB`); + + return { bookCode, chapters: docs.length }; +} + +// --------------------------------------------------------------------------- +// Clean up old data from shared collections +// --------------------------------------------------------------------------- + +async function cleanupOldData(db: ReturnType) { + // Remove od- prefixed entries from book_meta (old location) + const bookMetaCol = db.collection("book_meta"); + const deletedMeta = await bookMetaCol.deleteMany({ + book: { $regex: /^od-/ }, + }); + if (deletedMeta.deletedCount > 0) { + console.log( + ` 🗑️ Deleted ${deletedMeta.deletedCount} od-* doc(s) from book_meta`, + ); + } + + // Remove od- prefixed entries from prose_chapters (old location) + const proseCol = db.collection("prose_chapters"); + const deletedProse = await proseCol.deleteMany({ + book: { $regex: /^od-/ }, + }); + if (deletedProse.deletedCount > 0) { + console.log( + ` 🗑️ Deleted ${deletedProse.deletedCount} od-* doc(s) from prose_chapters`, + ); + } + + // Clean up legacy other_documents collection + const collections = await db + .listCollections({ name: "other_documents" }) + .toArray(); + if (collections.length > 0) { + const col = db.collection("other_documents"); + const count = await col.countDocuments(); + if (count > 0) { + await col.deleteMany({}); + console.log( + ` 🗑️ Deleted ${count} doc(s) from legacy other_documents collection`, + ); + } + } +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function main() { + console.log("🌱 Seeding Other Documents...\n"); + + const client = new MongoClient(DATABASE_URL!); + await client.connect(); + console.log("📡 Connected to MongoDB\n"); + + const db = client.db(); + const start = Date.now(); + + // Discover all subdirectories + if (!fs.existsSync(OTHER_DOCS_DIR)) { + console.error(`❌ Directory not found: ${OTHER_DOCS_DIR}`); + process.exit(1); + } + + const subdirs = fs + .readdirSync(OTHER_DOCS_DIR) + .filter((entry) => { + const fullPath = path.join(OTHER_DOCS_DIR, entry); + return ( + fs.statSync(fullPath).isDirectory() && + fs.existsSync(path.join(fullPath, "intro.json")) + ); + }) + .sort(); + + if (subdirs.length === 0) { + console.error(`❌ No document directories found in ${OTHER_DOCS_DIR}`); + process.exit(1); + } + + console.log(`📂 Found ${subdirs.length} document(s)\n`); + + // Seed each document + let totalChapters = 0; + const bookCodes: string[] = []; + + for (const subdir of subdirs) { + console.log(`\n📝 Processing: ${subdir}`); + const docDir = path.join(OTHER_DOCS_DIR, subdir); + const result = await seedOneDocument(db, docDir); + bookCodes.push(result.bookCode); + totalChapters += result.chapters; + } + + // Clean up old data from shared collections + console.log("\n🧹 Cleaning up old data from shared collections..."); + await cleanupOldData(db); + + // Summary + const elapsed = ((Date.now() - start) / 1000).toFixed(1); + console.log("\n" + "─".repeat(50)); + console.log(`📊 Summary:`); + console.log(` Documents: ${subdirs.length}`); + console.log(` Book codes: ${bookCodes.join(", ")}`); + console.log(` Total chapters: ${totalChapters}`); + console.log(`\n🎉 Done in ${elapsed}s`); + + await client.close(); +} + +main().catch((err) => { + console.error("❌ Seed failed:", err); + process.exit(1); +}); diff --git a/scripts/seed-pk.ts b/scripts/seed-pk.ts new file mode 100644 index 00000000..350db624 --- /dev/null +++ b/scripts/seed-pk.ts @@ -0,0 +1,169 @@ +/** + * seed-pk.ts — Push Prayers of King Kulaśekhara JSON data into MongoDB. + * + * Uses the native MongoDB driver (same pattern as seed-mog.ts) to avoid + * Prisma replica-set write requirements. + * + * PK is a prose book with no volume structure and no intro segments. + * We use volume=0 as the sentinel for "no volumes". + * + * Collections written: + * - book_meta → one doc { book: "pk", data: } + * - prose_chapters → 6 chapters (one per verse/prayer) + * + * Usage: + * npx tsx scripts/seed-pk.ts + */ + +import dotenv from "dotenv"; +import fs from "fs"; +import path from "path"; +import { MongoClient } from "mongodb"; + +// Load env +dotenv.config({ path: ".env.local" }); +dotenv.config({ path: ".env" }); + +const DATABASE_URL = process.env.DATABASE_URL; +if (!DATABASE_URL) { + console.error("❌ DATABASE_URL is not set in .env"); + process.exit(1); +} + +const BASE_DIR = path.resolve(__dirname, ".."); +const PK_DIR = path.join(BASE_DIR, "lib", "jsons", "books", "pk"); + +/** Volume sentinel for books without volumes. */ +const NO_VOLUME = 0; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function readJson(filePath: string): T { + return JSON.parse(fs.readFileSync(filePath, "utf-8")); +} + +/** + * Insert documents one at a time with a small delay every N docs. + * Uses insertOne to avoid BSON wire-protocol corruption. + */ +async function insertBatched( + col: ReturnType["collection"]>, + docs: Record[], +) { + for (let i = 0; i < docs.length; i++) { + await col.insertOne(docs[i]); + if ((i + 1) % 100 === 0) { + await new Promise((r) => setTimeout(r, 100)); + } + } +} + +// --------------------------------------------------------------------------- +// Types for the JSON files +// --------------------------------------------------------------------------- + +type PkIntroJson = { + book: string; + title: string; + author: string; + year: number; + description: string; + totalChapters: number; + chapters: { number: number; title: string }[]; +}; + +type PkChapterJson = { + book: string; + chapter: number; + title: string; + type: string; + paragraphs: string[]; +}; + +// --------------------------------------------------------------------------- +// Seed function +// --------------------------------------------------------------------------- + +async function seedPk(db: ReturnType) { + const introPath = path.join(PK_DIR, "intro.json"); + if (!fs.existsSync(introPath)) { + console.error("❌ PK intro.json not found at", introPath); + process.exit(1); + } + + const intro = readJson(introPath); + + // ---- 1. Book meta ---- + const metaCol = db.collection("book_meta"); + await metaCol.deleteMany({ book: "pk" }); + await metaCol.insertOne({ book: "pk", data: intro }); + console.log(" 📖 PK book meta inserted"); + + // ---- 2. Prose chapters collection ---- + const proseCol = db.collection("prose_chapters"); + await proseCol.deleteMany({ book: "pk" }); + + const docs: Record[] = []; + + // Chapters (01.json through 06.json) + const chaptersDir = path.join(PK_DIR, "chapters"); + if (fs.existsSync(chaptersDir)) { + const chapterFiles = fs + .readdirSync(chaptersDir) + .filter((f) => f.endsWith(".json")) + .sort(); + + for (const file of chapterFiles) { + const ch = readJson(path.join(chaptersDir, file)); + const chNum = ch.chapter ?? parseInt(file.replace(".json", ""), 10); + docs.push({ + book: "pk", + volume: NO_VOLUME, + chapter: chNum, + title: ch.title, + type: "chapter", + slug: String(chNum), + order: 100 + chNum, + paragraphs: ch.paragraphs, + }); + } + console.log(` 📖 Chapters: ${chapterFiles.length}`); + } + + // Insert all docs + console.log(`\n Inserting ${docs.length} documents...`); + await insertBatched(proseCol, docs); + + const totalDocs = await proseCol.countDocuments({ book: "pk" }); + console.log(` ✅ PK: ${totalDocs} prose entries in MongoDB`); +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function main() { + console.log("🌱 Seeding Prayers of King Kulaśekhara (1958)...\n"); + + const client = new MongoClient(DATABASE_URL!); + await client.connect(); + console.log("📡 Connected to MongoDB\n"); + + const db = client.db(); + + const start = Date.now(); + + await seedPk(db); + + const elapsed = ((Date.now() - start) / 1000).toFixed(1); + console.log(`\n🎉 Done in ${elapsed}s`); + + await client.close(); +} + +main().catch((err) => { + console.error("❌ Seed failed:", err); + process.exit(1); +}); diff --git a/scripts/seed-pre-1977-essays.ts b/scripts/seed-pre-1977-essays.ts new file mode 100644 index 00000000..08e2de53 --- /dev/null +++ b/scripts/seed-pre-1977-essays.ts @@ -0,0 +1,179 @@ +/** + * seed-pre-1977-essays.ts — Push thirteen pre-1977 essays into MongoDB. + * + * Uses the native MongoDB driver (same pattern as seed-cc-essays.ts) to avoid + * Prisma replica-set write requirements. + * + * Collections written: + * - essays → 9 essay docs (upsert by slug) + * + * JSON sources: + * - lib/jsons/essays/man-is-the-architect-of-his-own-fortune.json + * - lib/jsons/essays/what-is-the-matter-with-the-world.json + * - lib/jsons/essays/religion-is-godly-life.json + * - lib/jsons/essays/truth-and-beauty.json + * - lib/jsons/essays/misuse-of-public-fund-in-the-name-of-religion.json + * - lib/jsons/essays/perfection-at-home.json + * - lib/jsons/essays/how-should-one-cultivate-human-spirit.json + * - lib/jsons/essays/viswa-dharma.json + * - lib/jsons/essays/glorification-of-srila-bhaktisiddhanta-sarasvati-thakura.json + * - lib/jsons/essays/who-is-crazy-btg66.json + * - lib/jsons/essays/manifestation-of-dharma-in-human-being.json + * - lib/jsons/essays/message-of-krishna-consciousness.json + * - lib/jsons/essays/a-wrong-civilization.json + * + * Usage: + * npx tsx scripts/seed-pre-1977-essays.ts + */ + +import dotenv from "dotenv"; +import fs from "fs"; +import path from "path"; +import { MongoClient } from "mongodb"; + +// Load env +dotenv.config({ path: ".env.local" }); +dotenv.config({ path: ".env" }); + +const DATABASE_URL = process.env.DATABASE_URL; +if (!DATABASE_URL) { + console.error("❌ DATABASE_URL is not set in .env"); + process.exit(1); +} + +const BASE_DIR = path.resolve(__dirname, ".."); +const ESSAYS_DIR = path.join(BASE_DIR, "lib", "jsons", "essays"); + +const ESSAY_FILES = [ + "man-is-the-architect-of-his-own-fortune.json", + "what-is-the-matter-with-the-world.json", + "religion-is-godly-life.json", + "truth-and-beauty.json", + "misuse-of-public-fund-in-the-name-of-religion.json", + "perfection-at-home.json", + "how-should-one-cultivate-human-spirit.json", + "viswa-dharma.json", + "glorification-of-srila-bhaktisiddhanta-sarasvati-thakura.json", + "who-is-crazy-btg66.json", + "manifestation-of-dharma-in-human-being.json", + "message-of-krishna-consciousness.json", + "a-wrong-civilization.json", +]; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +type EssayJson = { + slug: string; + title: string; + author: string; + year: number | null; + group: string; + order: number; + paragraphs: string[]; +}; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function readJson(filePath: string): T { + return JSON.parse(fs.readFileSync(filePath, "utf-8")); +} + +// --------------------------------------------------------------------------- +// Seed function +// --------------------------------------------------------------------------- + +async function seedPre1977Essays(db: ReturnType) { + const essaysCol = db.collection("essays"); + + let inserted = 0; + let updated = 0; + let unchanged = 0; + + for (const filename of ESSAY_FILES) { + const jsonPath = path.join(ESSAYS_DIR, filename); + + if (!fs.existsSync(jsonPath)) { + console.error(` ❌ Missing: ${jsonPath}`); + console.error(" Ensure the JSON file exists in lib/jsons/essays/."); + process.exit(1); + } + + const essay = readJson(jsonPath); + + const result = await essaysCol.updateOne( + { slug: essay.slug }, + { + $set: { + slug: essay.slug, + title: essay.title, + author: essay.author, + year: essay.year, + group: essay.group, + order: essay.order, + paragraphs: essay.paragraphs, + }, + }, + { upsert: true }, + ); + + if (result.upsertedCount > 0) { + console.log( + ` 📝 Inserted "${essay.title}" (${essay.paragraphs.length} paragraphs)`, + ); + inserted++; + } else if (result.modifiedCount > 0) { + console.log( + ` 📝 Updated "${essay.title}" (${essay.paragraphs.length} paragraphs)`, + ); + updated++; + } else { + console.log(` 📝 "${essay.title}" already up to date`); + unchanged++; + } + } + + // Verify + const totalDocs = await essaysCol.countDocuments({ + group: "pre-1977-essays", + }); + console.log( + `\n ✅ Verified: ${totalDocs} document(s) in essays collection with group "pre-1977-essays"`, + ); + + // Summary + console.log( + `\n Summary: ${inserted} inserted, ${updated} updated, ${unchanged} unchanged`, + ); +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function main() { + console.log("🌱 Seeding Pre-1977 Essays (13 essays)...\n"); + + const client = new MongoClient(DATABASE_URL!); + await client.connect(); + console.log("📡 Connected to MongoDB\n"); + + const db = client.db(); + + const start = Date.now(); + + await seedPre1977Essays(db); + + const elapsed = ((Date.now() - start) / 1000).toFixed(1); + console.log(`\n🎉 Done in ${elapsed}s`); + + await client.close(); +} + +main().catch((err) => { + console.error("❌ Seed failed:", err); + process.exit(1); +}); diff --git a/scripts/seed-va.ts b/scripts/seed-va.ts new file mode 100644 index 00000000..57611cad --- /dev/null +++ b/scripts/seed-va.ts @@ -0,0 +1,139 @@ +/** + * seed-va.ts — Push Vaiśiṣṭyāṣṭakam (VA) into MongoDB. + * + * Collections written: + * - book_meta → 1 doc { book: "va", data: } + * - prose_chapters → 8 chapter docs + * + * JSON sources: + * - lib/jsons/books/va/intro.json + * - lib/jsons/books/va/chapters/01.json through 08.json + * + * Usage: + * npx tsx scripts/seed-va.ts + */ + +import dotenv from "dotenv"; +import fs from "fs"; +import path from "path"; +import { MongoClient } from "mongodb"; + +// Load env +dotenv.config({ path: ".env.local" }); +dotenv.config({ path: ".env" }); + +const DATABASE_URL = process.env.DATABASE_URL; +if (!DATABASE_URL) { + console.error("❌ DATABASE_URL is not set in .env"); + process.exit(1); +} + +const BASE_DIR = path.resolve(__dirname, ".."); +const BOOK_DIR = path.join(BASE_DIR, "lib", "jsons", "books", "va"); +const BOOK = "va"; +const NO_VOLUME = 0; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +type IntroJson = { + book: string; + title: string; + author: string; + year: number | null; + description: string; + totalChapters: number; + chapters: { number: number; title: string }[]; +}; + +type ChapterJson = { + book: string; + chapter: number; + title: string; + type: string; + paragraphs: string[]; +}; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function readJson(filePath: string): T { + return JSON.parse(fs.readFileSync(filePath, "utf-8")); +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function main() { + console.log("🌱 Seeding Vaiśiṣṭyāṣṭakam (VA)...\n"); + + const client = new MongoClient(DATABASE_URL!); + await client.connect(); + console.log("📡 Connected to MongoDB\n"); + + const db = client.db(); + const start = Date.now(); + + // ---- Read intro ---- + const introPath = path.join(BOOK_DIR, "intro.json"); + if (!fs.existsSync(introPath)) { + console.error(`❌ Missing: ${introPath}`); + process.exit(1); + } + const intro = readJson(introPath); + + // ---- Book meta ---- + const metaCol = db.collection("book_meta"); + await metaCol.deleteMany({ book: BOOK }); + await metaCol.insertOne({ book: BOOK, data: intro }); + console.log(`📖 Inserted book_meta for "${intro.title}" (${BOOK})`); + + // ---- Chapters ---- + const proseCol = db.collection("prose_chapters"); + await proseCol.deleteMany({ book: BOOK }); + + const chaptersDir = path.join(BOOK_DIR, "chapters"); + const chapterFiles = fs + .readdirSync(chaptersDir) + .filter((f) => f.endsWith(".json")) + .sort(); + + let inserted = 0; + for (const file of chapterFiles) { + const ch = readJson(path.join(chaptersDir, file)); + const chNum = ch.chapter; + + await proseCol.insertOne({ + book: BOOK, + volume: NO_VOLUME, + chapter: chNum, + title: ch.title, + type: ch.type || "chapter", + slug: String(chNum), + order: 100 + chNum, + paragraphs: ch.paragraphs, + }); + + console.log( + ` 📄 Chapter ${chNum}: "${ch.title}" (${ch.paragraphs.length} paragraphs)`, + ); + inserted++; + } + + // Verify + const totalDocs = await proseCol.countDocuments({ book: BOOK }); + console.log(`\n✅ Verified: ${totalDocs} prose_chapters docs for ${BOOK}`); + + const elapsed = ((Date.now() - start) / 1000).toFixed(1); + console.log(`\n🎉 Done in ${elapsed}s (${inserted} chapters inserted)`); + + await client.close(); +} + +main().catch((err) => { + console.error("❌ Seed failed:", err); + process.exit(1); +}); diff --git a/scripts/the_beginning_extract.py b/scripts/the_beginning_extract.py new file mode 100644 index 00000000..36e933f3 --- /dev/null +++ b/scripts/the_beginning_extract.py @@ -0,0 +1,1017 @@ +#!/usr/bin/env python3 +"""Extract "The Beginning" — The 1966 New York Journal from others-before-1977.html. + +Each diary day entry becomes its own chapter file. +Chapters are grouped hierarchically: + - Introduction (standalone) + - Chapter 1: The Upper West Side (summary + day entries grouped by month) + - Chapter 2: The Bowery (summary + day entries grouped by month) + - Chapter 3: 26 Second Avenue (summary + day entries grouped by month + Notice) + - Memoranda and Accounts (standalone) + - Epilogue (standalone) + - Excerpts (each excerpt as its own chapter) + +Output: + lib/jsons/journals/the-beginning/intro.json + lib/jsons/journals/the-beginning/chapters/introduction.json + lib/jsons/journals/the-beginning/chapters/upper-west-side-summary.json + lib/jsons/journals/the-beginning/chapters/jan-19.json + ... + lib/jsons/journals/the-beginning/chapters/memoranda.json + lib/jsons/journals/the-beginning/chapters/epilogue.json + lib/jsons/journals/the-beginning/chapters/excerpt-*.json + +Usage: + python3 scripts/the_beginning_extract.py +""" + +import json +import os +import re +import sys + +from bs4 import BeautifulSoup, Tag + +# --------------------------------------------------------------------------- +# ScaGoudy → IAST Unicode mapping +# --------------------------------------------------------------------------- +SCAGOUDY_TO_IAST = { + "ä": "ā", + "Ä": "Ā", + "å": "ṛ", + "Å": "Ṛ", + "ñ": "ṣ", + "Ñ": "Ṣ", + "ö": "ṭ", + "Ö": "Ṭ", + "ë": "ṇ", + "Ë": "Ṇ", + "ò": "ḍ", + "Ò": "Ḍ", + "ç": "ś", + "Ç": "Ś", + "ì": "ṅ", + "Ì": "Ṅ", + "ï": "ñ", + "Ï": "Ñ", + "ü": "ū", + "Ü": "Ū", + "é": "ī", + "É": "Ī", + "à": "ṁ", + "ù": "ḥ", + "è": "ṝ", + "ÿ": "ḷ", +} + + +def decode_scagoudy(text: str) -> str: + for src, dst in SCAGOUDY_TO_IAST.items(): + text = text.replace(src, dst) + return text + + +def get_css_class(tag: Tag) -> str: + classes = tag.get("class", []) + return classes[0] if classes else "" + + +def inner_html(tag: Tag) -> str: + """Get inner HTML of a tag, decode ScaGoudy, clean up spans.""" + html = "".join(str(child) for child in tag.children) + html = decode_scagoudy(html) + # text → text + html = re.sub( + r']*class="s3"[^>]*>(.*?)', r"\1", html, flags=re.DOTALL + ) + # ... → keep inner text + html = re.sub( + r']*class="s5"[^>]*>(.*?)', r"\1", html, flags=re.DOTALL + ) + # ... → keep inner text + html = re.sub( + r']*class="s7"[^>]*>(.*?)', r"\1", html, flags=re.DOTALL + ) + # ... → keep inner text + html = re.sub( + r']*class="s\d+"[^>]*>(.*?)', r"\1", html, flags=re.DOTALL + ) + # Remove Apple-tab-span + html = re.sub(r'\t', " ", html) + # Remove any remaining spans but keep content + html = re.sub(r"]*>(.*?)", r"\1", html, flags=re.DOTALL) + # Clean up tab characters + html = html.replace("\t", " ") + # Collapse multiple spaces + html = re.sub(r"[ ]+", " ", html) + html = html.strip() + # Remove leading
tags + html = re.sub(r"^([\s]*)+", "", html) + return html + + +def write_json(data: dict, path: str): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + f.write("\n") + + +# --------------------------------------------------------------------------- +# Month helpers +# --------------------------------------------------------------------------- +MONTH_MAP = { + "january": "jan", + "february": "feb", + "march": "mar", + "april": "apr", + "may": "may", + "june": "jun", + "july": "jul", + "august": "aug", + "september": "sep", + "october": "oct", + "november": "nov", + "december": "dec", +} + +MONTH_FULL = { + "jan": "January 1966", + "feb": "February 1966", + "mar": "March 1966", + "apr": "April 1966", + "may": "May 1966", + "jun": "June 1966", + "jul": "July 1966", + "aug": "August 1966", + "sep": "September 1966", + "oct": "October 1966", + "nov": "November 1966", +} + +DAY_NAMES = { + "monday", + "tuesday", + "wednesday", + "thursday", + "friday", + "saturday", + "sunday", +} + + +def parse_day_month(text: str): + """Parse 'Wednesday 19 January' → ('Wednesday', 19, 'jan') + Also handles 'Notice' and other non-date entries. + Returns (day_name, date_num, month_abbr) or (None, None, None). + """ + text = text.strip() + # Try "DayName Number MonthName" pattern + m = re.match(r"([A-Za-z]+)\s+(\d+)\s+([A-Za-z]+)", text) + if m: + day = m.group(1).capitalize() + num = int(m.group(2)) + month = m.group(3).lower() + if day.lower() in DAY_NAMES and month in MONTH_MAP: + return (day, num, MONTH_MAP[month]) + + return (None, None, None) + + +def slugify_excerpt(title: str) -> str: + """Create a URL-friendly slug from an excerpt title.""" + # Remove HTML tags + title = re.sub(r"<[^>]+>", "", title) + title = decode_scagoudy(title) + # Take first 60 chars + title = title[:80].lower().strip() + # Replace non-alnum with hyphens + title = re.sub(r"[^a-z0-9]+", "-", title) + title = title.strip("-") + # Truncate to reasonable length + if len(title) > 60: + title = title[:60].rsplit("-", 1)[0] + return f"excerpt-{title}" + + +# --------------------------------------------------------------------------- +# Section boundaries detector +# --------------------------------------------------------------------------- + + +def find_section_boundaries(all_tags): + """Find the start/end tag indices for each major section.""" + sections = {} + beg_start = None + beg_end = None + + for i, tag in enumerate(all_tags): + cls = get_css_class(tag) + text = tag.get_text().strip() + + # Find "The Beginning" start + if cls == "p170" and "The Beginning" in text: + beg_start = i + + if beg_start is not None and i > beg_start: + # Find section markers by p4 bookmarks + if cls == "p4" and text.startswith("BEG"): + if "Introduction" in text: + sections["introduction_start"] = i + elif "BEG 1" in text: + sections["ch1_start"] = i + elif "BEG 2" in text: + sections["ch2_start"] = i + elif "BEG 3" in text: + sections["ch3_start"] = i + elif "BEG 4" in text: + sections["ch4_start"] = i + elif "BEG 5" in text: + sections["ch5_start"] = i + elif "BEG 6" in text: + sections["ch6_start"] = i + + # Detect end of The Beginning (next non-BEG p4 or end of file) + if ( + cls == "p4" + and not text.startswith("BEG") + and beg_start + and "ch6_start" in sections + and i > sections["ch6_start"] + 5 + ): + beg_end = i + break + + # If no explicit end found, use end of conversation content + if beg_end is None: + # Search for the last p74 conversation line + for i in range(len(all_tags) - 1, sections.get("ch6_start", 0), -1): + cls = get_css_class(all_tags[i]) + if cls in ("p74", "p6", "p77"): + beg_end = i + 1 + break + + sections["beg_start"] = beg_start + sections["beg_end"] = beg_end + + return sections + + +# --------------------------------------------------------------------------- +# Extract prose section (Introduction, Epilogue, etc.) +# --------------------------------------------------------------------------- + + +def extract_prose_section(tags, start_idx, end_idx, skip_classes=None): + """Extract paragraphs from a prose section.""" + if skip_classes is None: + skip_classes = {"p4", "p11", "p3", "p5", "p39", "p24", "p153"} + + paragraphs = [] + for i in range(start_idx, end_idx): + tag = tags[i] + cls = get_css_class(tag) + + if cls in skip_classes: + continue + + html = inner_html(tag) + if html and html != "
": + paragraphs.append(html) + + return paragraphs + + +# --------------------------------------------------------------------------- +# Extract day entries from a chapter section +# --------------------------------------------------------------------------- + + +def extract_day_entries(tags, start_idx, end_idx): + """Extract day entries from a chapter section. + Returns (summary_paragraphs, day_entries_list). + """ + summary_paragraphs = [] + day_entries = [] + current_entry = None + in_summary = True # Start in summary mode until first p39 day entry + + skip_classes = {"p4", "p11", "p3", "p5", "p153"} + + def finalize_entry(): + nonlocal current_entry + if current_entry and current_entry["paragraphs"]: + day_entries.append(current_entry) + current_entry = None + + for i in range(start_idx, end_idx): + tag = tags[i] + cls = get_css_class(tag) + text = tag.get_text().strip() + + if cls in skip_classes: + continue + + # p39 = day bookmark + if cls == "p39": + finalize_entry() + in_summary = False + + # Check if it's a special entry like "Notice" or "Memoranda" + if text.lower() == "notice": + current_entry = { + "day_name": None, + "date": None, + "month": None, + "slug": "notice", + "title": "Notice", + "paragraphs": [], + } + continue + elif text.lower() == "memoranda": + current_entry = { + "day_name": None, + "date": None, + "month": None, + "slug": "memoranda", + "title": "Memoranda", + "paragraphs": [], + } + continue + + day_name, date_num, month_abbr = parse_day_month(text) + if day_name and date_num and month_abbr: + slug = f"{month_abbr}-{date_num}" + title = f"{day_name} {date_num}" + current_entry = { + "day_name": day_name, + "date": date_num, + "month": month_abbr, + "slug": slug, + "title": title, + "paragraphs": [], + } + else: + # Can't parse — might be an excerpt title or special entry + # Treat as a titled entry + slug_text = re.sub(r"[^a-z0-9]+", "-", text.lower().strip())[:50].strip( + "-" + ) + current_entry = { + "day_name": None, + "date": None, + "month": None, + "slug": slug_text or "unknown", + "title": text, + "paragraphs": [], + } + continue + + # p8 = bold day header (display version of p39) + if cls == "p8": + # This is the bold display version — skip if we already have the entry from p39 + if current_entry is not None: + continue + # If we're in summary mode, this might be the chapter date range or summary header + if in_summary: + html = inner_html(tag) + if html: + summary_paragraphs.append(html) + continue + continue + + # p24 = empty line (skip) + if cls == "p24": + continue + + # p29 = bold subheading (like "NOTICE") + if cls == "p29": + if current_entry is not None: + html = inner_html(tag) + if html: + current_entry["paragraphs"].append(html) + continue + + # Collect body paragraphs + html = inner_html(tag) + if not html or html == "
": + continue + + if in_summary: + summary_paragraphs.append(html) + elif current_entry is not None: + current_entry["paragraphs"].append(html) + + finalize_entry() + return summary_paragraphs, day_entries + + +# --------------------------------------------------------------------------- +# Extract excerpts +# --------------------------------------------------------------------------- + + +def extract_excerpts(tags, start_idx, end_idx): + """Extract individual excerpts from the Excerpts section. + Each excerpt starts with a p39 bookmark. + Returns a list of { slug, title, paragraphs }. + """ + excerpts = [] + current_excerpt = None + in_preamble = True # Before first p39 excerpt + + preamble_paragraphs = [] + skip_classes = {"p4", "p11", "p3", "p5", "p153"} + seen_slugs = set() + + def finalize_excerpt(): + nonlocal current_excerpt + if current_excerpt and current_excerpt["paragraphs"]: + excerpts.append(current_excerpt) + current_excerpt = None + + for i in range(start_idx, end_idx): + tag = tags[i] + cls = get_css_class(tag) + text = tag.get_text().strip() + + if cls in skip_classes: + continue + + # p39 = excerpt title bookmark + if cls == "p39": + finalize_excerpt() + in_preamble = False + + title = decode_scagoudy(text) + slug = slugify_excerpt(title) + + # Ensure unique slug + if slug in seen_slugs: + suffix = 2 + while f"{slug}-{suffix}" in seen_slugs: + suffix += 1 + slug = f"{slug}-{suffix}" + seen_slugs.add(slug) + + current_excerpt = { + "slug": slug, + "title": title, + "paragraphs": [], + } + continue + + # p8 = bold excerpt header (display version) — skip if we have the excerpt + if cls == "p8": + if current_excerpt is not None: + # Skip the bold header — we already have the title from p39 + continue + if in_preamble: + html = inner_html(tag) + if html: + preamble_paragraphs.append(html) + continue + + if cls == "p24": + continue + + html = inner_html(tag) + if not html or html == "
": + continue + + if in_preamble: + preamble_paragraphs.append(html) + elif current_excerpt is not None: + # For conversation lines (p74), format with bold speaker + # The inner_html already preserves tags + current_excerpt["paragraphs"].append(html) + + finalize_excerpt() + + return preamble_paragraphs, excerpts + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def main(): + base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html") + output_dir = os.path.join(base_dir, "lib", "jsons", "journals", "the-beginning") + chapters_dir = os.path.join(output_dir, "chapters") + + if not os.path.exists(html_path): + print(f"ERROR: {html_path} not found!") + sys.exit(1) + + print(f"Reading {html_path}...") + with open(html_path, "r", encoding="utf-8") as f: + content = f.read() + + soup = BeautifulSoup(content, "html.parser") + all_tags = soup.find_all("p") + print(f" Parsed {len(all_tags)}

tags total") + + # Find section boundaries + sections = find_section_boundaries(all_tags) + print( + f" The Beginning: tags[{sections['beg_start']}] → tags[{sections['beg_end']}]" + ) + for key, val in sorted(sections.items()): + print(f" {key}: {val}") + + # --------------------------------------------------------------------------- + # Extract Introduction + # --------------------------------------------------------------------------- + intro_start = sections["introduction_start"] + intro_end = sections["ch1_start"] + intro_paragraphs = extract_prose_section(all_tags, intro_start, intro_end) + print(f"\n Introduction: {len(intro_paragraphs)} paragraphs") + + # --------------------------------------------------------------------------- + # Extract Chapter 1: The Upper West Side + # --------------------------------------------------------------------------- + ch1_start = sections["ch1_start"] + ch1_end = sections["ch2_start"] + ch1_summary, ch1_entries = extract_day_entries(all_tags, ch1_start, ch1_end) + print( + f" Ch 1 (Upper West Side): {len(ch1_summary)} summary paras, {len(ch1_entries)} day entries" + ) + + # --------------------------------------------------------------------------- + # Extract Chapter 2: The Bowery + # --------------------------------------------------------------------------- + ch2_start = sections["ch2_start"] + ch2_end = sections["ch3_start"] + ch2_summary, ch2_entries = extract_day_entries(all_tags, ch2_start, ch2_end) + print( + f" Ch 2 (The Bowery): {len(ch2_summary)} summary paras, {len(ch2_entries)} day entries" + ) + + # --------------------------------------------------------------------------- + # Extract Chapter 3: 26 Second Avenue + # --------------------------------------------------------------------------- + ch3_start = sections["ch3_start"] + ch3_end = sections["ch4_start"] + ch3_summary, ch3_entries = extract_day_entries(all_tags, ch3_start, ch3_end) + print( + f" Ch 3 (26 Second Ave): {len(ch3_summary)} summary paras, {len(ch3_entries)} day entries" + ) + + # --------------------------------------------------------------------------- + # Extract Memoranda and Accounts + # --------------------------------------------------------------------------- + ch4_start = sections["ch4_start"] + ch4_end = sections["ch5_start"] + memo_paragraphs = extract_prose_section(all_tags, ch4_start, ch4_end) + print(f" Memoranda: {len(memo_paragraphs)} paragraphs") + + # --------------------------------------------------------------------------- + # Extract Epilogue + # --------------------------------------------------------------------------- + ch5_start = sections["ch5_start"] + ch5_end = sections["ch6_start"] + epilogue_paragraphs = extract_prose_section( + all_tags, + ch5_start, + ch5_end, + skip_classes={"p4", "p11", "p3", "p5", "p153", "p24"}, + ) + print(f" Epilogue: {len(epilogue_paragraphs)} paragraphs") + + # --------------------------------------------------------------------------- + # Extract Excerpts + # --------------------------------------------------------------------------- + ch6_start = sections["ch6_start"] + ch6_end = sections["beg_end"] + excerpt_preamble, excerpt_list = extract_excerpts(all_tags, ch6_start, ch6_end) + print( + f" Excerpts: {len(excerpt_preamble)} preamble paras, {len(excerpt_list)} excerpts" + ) + for ex in excerpt_list: + print( + f" - {ex['slug']}: {ex['title'][:60]}... ({len(ex['paragraphs'])} paras)" + ) + + # --------------------------------------------------------------------------- + # Group day entries by month within each chapter + # --------------------------------------------------------------------------- + + def group_by_month(entries): + """Group day entries by month, preserving order.""" + groups = [] # List of (month_abbr, month_full, entries_list) + current_month = None + current_group = None + + for entry in entries: + m = entry.get("month") + if m and m != current_month: + current_month = m + current_group = { + "month": m, + "month_full": MONTH_FULL.get(m, m), + "entries": [], + } + groups.append(current_group) + if current_group is not None: + current_group["entries"].append(entry) + else: + # Entry without month (like "Notice") — create special group + current_group = { + "month": None, + "month_full": "Other", + "entries": [entry], + } + groups.append(current_group) + current_group = None + current_month = None + + return groups + + ch1_months = group_by_month(ch1_entries) + ch2_months = group_by_month(ch2_entries) + ch3_months = group_by_month(ch3_entries) + + # --------------------------------------------------------------------------- + # Ensure unique slugs across all entries + # --------------------------------------------------------------------------- + all_slugs = set() + + def ensure_unique_slug(slug): + if slug in all_slugs: + suffix = 2 + while f"{slug}-{suffix}" in all_slugs: + suffix += 1 + slug = f"{slug}-{suffix}" + all_slugs.add(slug) + return slug + + # Reserve standalone slugs + for s in [ + "introduction", + "upper-west-side-summary", + "bowery-summary", + "second-avenue-summary", + "memoranda", + "epilogue", + "excerpts-preamble", + ]: + all_slugs.add(s) + + # Ensure unique slugs for day entries + for entries in [ch1_entries, ch2_entries, ch3_entries]: + for entry in entries: + entry["slug"] = ensure_unique_slug(entry["slug"]) + + # Ensure unique slugs for excerpts + for ex in excerpt_list: + ex["slug"] = ensure_unique_slug(ex["slug"]) + + # --------------------------------------------------------------------------- + # Build intro.json with hierarchical chapters + # --------------------------------------------------------------------------- + + intro_chapters = [] + + # Introduction (standalone) + intro_chapters.append({"slug": "introduction", "title": "Introduction"}) + + # Chapter 1: The Upper West Side + ch1_sub = [{"slug": "upper-west-side-summary", "title": "Summary"}] + for mg in ch1_months: + ch1_sub.append( + { + "title": mg["month_full"], + "subChapters": [ + {"slug": e["slug"], "title": e["title"]} for e in mg["entries"] + ], + } + ) + intro_chapters.append( + { + "title": "The Upper West Side (Jan–Mar 1966)", + "subChapters": ch1_sub, + } + ) + + # Chapter 2: The Bowery + ch2_sub = [{"slug": "bowery-summary", "title": "Summary"}] + for mg in ch2_months: + ch2_sub.append( + { + "title": mg["month_full"], + "subChapters": [ + {"slug": e["slug"], "title": e["title"]} for e in mg["entries"] + ], + } + ) + intro_chapters.append( + { + "title": "The Bowery (Apr–Jun 1966)", + "subChapters": ch2_sub, + } + ) + + # Chapter 3: 26 Second Avenue + ch3_sub = [{"slug": "second-avenue-summary", "title": "Summary"}] + for mg in ch3_months: + if mg["month"] is not None: + ch3_sub.append( + { + "title": mg["month_full"], + "subChapters": [ + {"slug": e["slug"], "title": e["title"]} for e in mg["entries"] + ], + } + ) + else: + # Non-month entries like "Notice" + for e in mg["entries"]: + ch3_sub.append({"slug": e["slug"], "title": e["title"]}) + intro_chapters.append( + { + "title": "26 Second Avenue (Jul–Nov 1966)", + "subChapters": ch3_sub, + } + ) + + # Memoranda + intro_chapters.append({"slug": "memoranda", "title": "Memoranda and Accounts"}) + + # Epilogue + intro_chapters.append({"slug": "epilogue", "title": "Epilogue"}) + + # Excerpts — group under a parent heading + excerpt_sub = [] + if excerpt_preamble: + excerpt_sub.append({"slug": "excerpts-preamble", "title": "Preamble"}) + for ex in excerpt_list: + # Shorten title for sidebar + short_title = ex["title"] + if len(short_title) > 80: + short_title = short_title[:77] + "..." + excerpt_sub.append({"slug": ex["slug"], "title": short_title}) + intro_chapters.append( + { + "title": "Excerpts", + "subChapters": excerpt_sub, + } + ) + + # Count total chapters + total_chapters = 0 + for ch in intro_chapters: + if "slug" in ch: + total_chapters += 1 + if "subChapters" in ch: + for sc in ch["subChapters"]: + if "slug" in sc: + total_chapters += 1 + if "subChapters" in sc: + total_chapters += len(sc["subChapters"]) + + intro_json = { + "book": "the-beginning", + "title": "The Beginning — The 1966 New York Journal", + "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda", + "year": 1966, + "description": "Śrīla Prabhupāda's personal journal documenting the founding of the Hare Kṛṣṇa movement in New York City, from January to November 1966.", + "totalChapters": total_chapters, + "chapters": intro_chapters, + } + + # --------------------------------------------------------------------------- + # Write all JSON files + # --------------------------------------------------------------------------- + + # Clean output directory + if os.path.exists(chapters_dir): + for f in os.listdir(chapters_dir): + os.remove(os.path.join(chapters_dir, f)) + + # intro.json + write_json(intro_json, os.path.join(output_dir, "intro.json")) + print(f"\n Written: intro.json ({total_chapters} total chapters)") + + chapter_num = 1 + files_written = 0 + + # Introduction + write_json( + { + "book": "the-beginning", + "chapter": chapter_num, + "title": "Introduction", + "type": "introduction", + "slug": "introduction", + "paragraphs": intro_paragraphs, + }, + os.path.join(chapters_dir, "introduction.json"), + ) + print(f" Written: introduction.json ({len(intro_paragraphs)} paras)") + chapter_num += 1 + files_written += 1 + + # Chapter 1 summary + write_json( + { + "book": "the-beginning", + "chapter": chapter_num, + "title": "The Upper West Side — Summary", + "type": "summary", + "slug": "upper-west-side-summary", + "paragraphs": ch1_summary, + }, + os.path.join(chapters_dir, "upper-west-side-summary.json"), + ) + print(f" Written: upper-west-side-summary.json ({len(ch1_summary)} paras)") + chapter_num += 1 + files_written += 1 + + # Chapter 1 day entries + for entry in ch1_entries: + write_json( + { + "book": "the-beginning", + "chapter": chapter_num, + "title": entry["title"], + "type": "chapter", + "slug": entry["slug"], + "paragraphs": entry["paragraphs"], + }, + os.path.join(chapters_dir, f"{entry['slug']}.json"), + ) + chapter_num += 1 + files_written += 1 + + print(f" Written: {len(ch1_entries)} Chapter 1 day entry files") + + # Chapter 2 summary + write_json( + { + "book": "the-beginning", + "chapter": chapter_num, + "title": "The Bowery — Summary", + "type": "summary", + "slug": "bowery-summary", + "paragraphs": ch2_summary, + }, + os.path.join(chapters_dir, "bowery-summary.json"), + ) + print(f" Written: bowery-summary.json ({len(ch2_summary)} paras)") + chapter_num += 1 + files_written += 1 + + # Chapter 2 day entries + for entry in ch2_entries: + write_json( + { + "book": "the-beginning", + "chapter": chapter_num, + "title": entry["title"], + "type": "chapter", + "slug": entry["slug"], + "paragraphs": entry["paragraphs"], + }, + os.path.join(chapters_dir, f"{entry['slug']}.json"), + ) + chapter_num += 1 + files_written += 1 + + print(f" Written: {len(ch2_entries)} Chapter 2 day entry files") + + # Chapter 3 summary + write_json( + { + "book": "the-beginning", + "chapter": chapter_num, + "title": "26 Second Avenue — Summary", + "type": "summary", + "slug": "second-avenue-summary", + "paragraphs": ch3_summary, + }, + os.path.join(chapters_dir, "second-avenue-summary.json"), + ) + print(f" Written: second-avenue-summary.json ({len(ch3_summary)} paras)") + chapter_num += 1 + files_written += 1 + + # Chapter 3 day entries + for entry in ch3_entries: + write_json( + { + "book": "the-beginning", + "chapter": chapter_num, + "title": entry["title"], + "type": "chapter", + "slug": entry["slug"], + "paragraphs": entry["paragraphs"], + }, + os.path.join(chapters_dir, f"{entry['slug']}.json"), + ) + chapter_num += 1 + files_written += 1 + + print(f" Written: {len(ch3_entries)} Chapter 3 day entry files") + + # Memoranda + write_json( + { + "book": "the-beginning", + "chapter": chapter_num, + "title": "Memoranda and Accounts", + "type": "chapter", + "slug": "memoranda", + "paragraphs": memo_paragraphs, + }, + os.path.join(chapters_dir, "memoranda.json"), + ) + print(f" Written: memoranda.json ({len(memo_paragraphs)} paras)") + chapter_num += 1 + files_written += 1 + + # Epilogue + write_json( + { + "book": "the-beginning", + "chapter": chapter_num, + "title": "Epilogue", + "type": "chapter", + "slug": "epilogue", + "paragraphs": epilogue_paragraphs, + }, + os.path.join(chapters_dir, "epilogue.json"), + ) + print(f" Written: epilogue.json ({len(epilogue_paragraphs)} paras)") + chapter_num += 1 + files_written += 1 + + # Excerpts preamble + if excerpt_preamble: + write_json( + { + "book": "the-beginning", + "chapter": chapter_num, + "title": "Excerpts — Preamble", + "type": "chapter", + "slug": "excerpts-preamble", + "paragraphs": excerpt_preamble, + }, + os.path.join(chapters_dir, "excerpts-preamble.json"), + ) + print(f" Written: excerpts-preamble.json ({len(excerpt_preamble)} paras)") + chapter_num += 1 + files_written += 1 + + # Excerpt entries + for ex in excerpt_list: + write_json( + { + "book": "the-beginning", + "chapter": chapter_num, + "title": ex["title"], + "type": "excerpt", + "slug": ex["slug"], + "paragraphs": ex["paragraphs"], + }, + os.path.join(chapters_dir, f"{ex['slug']}.json"), + ) + chapter_num += 1 + files_written += 1 + + print(f" Written: {len(excerpt_list)} excerpt files") + + # --------------------------------------------------------------------------- + # Summary + # --------------------------------------------------------------------------- + print(f"\n{'=' * 60}") + print(f" EXTRACTION SUMMARY") + print(f"{'=' * 60}") + print(f" Introduction: {len(intro_paragraphs)} paragraphs") + print( + f" Ch 1 (Upper West Side): {len(ch1_summary)} summary + {len(ch1_entries)} days" + ) + for mg in ch1_months: + print(f" {mg['month_full']}: {len(mg['entries'])} entries") + print(f" Ch 2 (The Bowery): {len(ch2_summary)} summary + {len(ch2_entries)} days") + for mg in ch2_months: + print(f" {mg['month_full']}: {len(mg['entries'])} entries") + print( + f" Ch 3 (26 Second Ave): {len(ch3_summary)} summary + {len(ch3_entries)} days" + ) + for mg in ch3_months: + mname = mg["month_full"] if mg["month"] else "Other" + print(f" {mname}: {len(mg['entries'])} entries") + print(f" Memoranda: {len(memo_paragraphs)} paragraphs") + print(f" Epilogue: {len(epilogue_paragraphs)} paragraphs") + print(f" Excerpts: {len(excerpt_list)} excerpts") + print(f" Total chapter files: {files_written}") + print(f" Total chapters (for intro.json): {total_chapters}") + print(f"{'=' * 60}") + print(f"\nDone! ✨") + + +if __name__ == "__main__": + main()