From 401766928a340e5a65368b5f098ce8822bbeda27 Mon Sep 17 00:00:00 2001 From: gosangam Date: Mon, 4 May 2026 00:08:31 +0530 Subject: [PATCH] feat: letters added --- PROJECT_CONTEXT.md | 187 ++++++- app/letters/[slug]/page.tsx | 117 +++++ app/letters/letters-browser.tsx | 285 ++++++++++ app/letters/letters-nav.tsx | 37 ++ app/letters/location/[name]/page.tsx | 64 +++ app/letters/page.tsx | 55 ++ app/letters/recipient/[name]/page.tsx | 65 +++ app/letters/year/[year]/page.tsx | 66 +++ app/magazines/[slug]/[issue]/page.tsx | 223 ++++++++ app/magazines/[slug]/page.tsx | 96 ++++ app/magazines/page.tsx | 110 ++++ app/page.tsx | 85 ++- lib/letters.ts | 249 +++++++++ lib/magazines.ts | 282 ++++++++++ prisma/schema.prisma | 78 +++ scripts/btg_1978_extract.py | 717 ++++++++++++++++++++++++++ scripts/btg_extract.py | 546 ++++++++++++++++++++ scripts/letters_extract.py | 689 +++++++++++++++++++++++++ scripts/seed-letters.ts | 158 ++++++ scripts/seed-magazines.ts | 189 +++++++ 20 files changed, 4272 insertions(+), 26 deletions(-) create mode 100644 app/letters/[slug]/page.tsx create mode 100644 app/letters/letters-browser.tsx create mode 100644 app/letters/letters-nav.tsx create mode 100644 app/letters/location/[name]/page.tsx create mode 100644 app/letters/page.tsx create mode 100644 app/letters/recipient/[name]/page.tsx create mode 100644 app/letters/year/[year]/page.tsx create mode 100644 app/magazines/[slug]/[issue]/page.tsx create mode 100644 app/magazines/[slug]/page.tsx create mode 100644 app/magazines/page.tsx create mode 100644 lib/letters.ts create mode 100644 lib/magazines.ts create mode 100644 scripts/btg_1978_extract.py create mode 100644 scripts/btg_extract.py create mode 100644 scripts/letters_extract.py create mode 100644 scripts/seed-letters.ts create mode 100644 scripts/seed-magazines.ts diff --git a/PROJECT_CONTEXT.md b/PROJECT_CONTEXT.md index 2731b12f..3aa1c85a 100644 --- a/PROJECT_CONTEXT.md +++ b/PROJECT_CONTEXT.md @@ -27,7 +27,7 @@ The original RTF files use **custom font encodings** (Indevr, ScaGoudy, Inbenr/I ``` spsays-training/ ├── app/ -│ ├── page.tsx → Landing page with Logo, links to Books/Essays/Glossary/RTFs +│ ├── page.tsx → Landing page with Logo, links to Books/Essays/Journals/Glossary/Other Documents/RTFs │ ├── layout.tsx → ThemeProvider + TooltipProvider + PasswordGate wrapper │ ├── globals.css → Vintage color palette + reading settings CSS │ ├── books/ @@ -56,10 +56,28 @@ spsays-training/ │ │ ├── ochkm/ → On Chanting the Hare Kṛṣṇa Mantra (4 chapters) │ │ ├── hkh/ → The Kṛṣṇa Consciousness Handbook (19 chapters) │ │ ├── ssr/ → The Science of Self-Realization (8 chapters, 50 sub-chapters) -│ │ └── ds/ → Dialectic Spiritualism (13 chapters, 36 sub-chapters) +│ │ ├── ds/ → Dialectic Spiritualism (13 chapters, 36 sub-chapters) +│ │ └── va/ → Vaiśiṣṭyāṣṭakam (8 chapters) +│ ├── magazines/ +│ │ ├── page.tsx → Magazines index page (grouped by flag: "Pre 1978") +│ │ └── [slug]/ +│ │ ├── page.tsx → Magazine issue listing (grouped by volume) +│ │ └── [issue]/page.tsx → Individual issue reader with sidebar │ ├── essays/ -│ │ ├── page.tsx → Essays listing page +│ │ ├── page.tsx → Essays grid page (searchable) +│ │ ├── essays-grid.tsx → Client component for searchable essay grid │ │ └── [slug]/page.tsx → Individual essay reader +│ ├── journals/ +│ │ ├── page.tsx → Journals index page (grid) +│ │ └── [slug]/ +│ │ ├── page.tsx → Journal chapter listing (hierarchical: months → days) +│ │ └── [chapter]/page.tsx → Individual journal entry reader +│ ├── other-documents/ +│ │ ├── page.tsx → Other documents grid page (searchable) +│ │ ├── other-documents-grid.tsx → Client component for searchable grid +│ │ └── [slug]/ +│ │ ├── page.tsx → Document chapter listing +│ │ └── [chapter]/page.tsx → Individual chapter reader │ ├── glossary/ │ │ ├── page.tsx → Server component wrapper │ │ └── glossary-client.tsx → Searchable client component (2749 entries) @@ -104,6 +122,10 @@ spsays-training/ │ ├── ssr.ts → Science of Self-Realization data access (Prisma, hierarchical) │ ├── ds.ts → Dialectic Spiritualism data access (Prisma, hierarchical) │ ├── te.ts → Two Essays data access (Prisma, uses essays collection) +│ ├── va.ts → Vaiśiṣṭyāṣṭakam data access (Prisma) +│ ├── other-documents.ts → Other Documents data access (Prisma, dedicated collections) +│ ├── journals.ts → Journals data access (Prisma, dedicated collections) +│ ├── magazines.ts → Magazines data access (Prisma, dedicated collections) │ └── jsons/books/ → Extracted JSON data (all books) │ ├── bg/, sb/, cc/, sb-1962/, kb/ → Original verse/prose books │ ├── nod/, tlc/, iso/, noi/ → Extracted from other.html @@ -111,9 +133,16 @@ spsays-training/ │ ├── rv/, poy/, pqpa/, ttp/, ochkm/ → More prose books │ ├── hkh/, ssr/, ds/ → Large structured books │ ├── te/ → Two Essays -│ └── glossary/index.json → 2,749 glossary entries +│ ├── va/ → Vaiśiṣṭyāṣṭakam +│ ├── glossary/index.json → 2,749 glossary entries +│ ├── essays/ → Essay JSON files (pre-1977, cc-essays, standalone, etc.) +│ ├── other-documents/ → Other document JSON files (12 documents) +│ ├── journals/ → Journal JSON files (2 journals, 300 chapters) +│ └── magazines/ → Magazine JSON files (2 magazines, 59 issues) +│ ├── btg/ → Back to Godhead (1944–1960), 29 issues +│ └── btg-1978/ → Back to Godhead (1966–1973), 30 issues ├── prisma/ -│ └── schema.prisma → Verse, BookMeta, ProseChapter, Essay, GlossaryEntry models +│ └── schema.prisma → Verse, BookMeta, ProseChapter, Essay, GlossaryEntry, OtherDocumentMeta, OtherDocumentChapter, JournalMeta, JournalChapter, MagazineMeta, MagazineIssue models ├── scripts/ │ ├── bg_extract.py → BG extraction from bg.html │ ├── sb_extract.py → SB extraction from sb.html @@ -141,6 +170,15 @@ spsays-training/ │ ├── seed-db.ts → Main MongoDB seeder (BG, SB, CC, SB1962, KB, glossary) │ ├── seed-nod.ts … seed-ds.ts → Individual book seeders │ ├── seed-te.ts → Two Essays seeder (uses essays collection) +│ ├── seed-va.ts → Vaiśiṣṭyāṣṭakam seeder +│ ├── seed-pre-1977-essays.ts → Pre-1977 essays seeder (13 essays) +│ ├── seed-other-documents.ts → Other documents seeder (dedicated collections) +│ ├── seed-journals.ts → Journals seeder (dedicated collections) +│ ├── jaladuta_diary_extract.py → Jaladuta Diary extraction (83 day entries) +│ ├── the_beginning_extract.py → The Beginning extraction (189 day entries + 20 excerpts) +│ ├── btg_extract.py → BTG (1944–1960) extraction from btg-1960.html +│ ├── btg_1978_extract.py → BTG (1966–1973) extraction from btg-1978.html +│ ├── seed-magazines.ts → Magazines seeder (dedicated collections) │ ├── kb_extract_*.py → KB extraction scripts (intro, vol1, vol2) │ ├── inject-fonts.py → Inject @font-face rules into converted HTML │ └── fix-bengali-encoding.py → Fix Bengali font encoding in cc.html @@ -226,6 +264,77 @@ model Essay { @@map("essays") } +model OtherDocumentMeta { + id String @id @default(auto()) @map("_id") @db.ObjectId + book String @unique + data Json + + @@map("other_document_meta") +} + +model OtherDocumentChapter { + id String @id @default(auto()) @map("_id") @db.ObjectId + book String + chapter Int + title String + type String + slug String + order Int + paragraphs String[] + + @@unique([book, slug]) + @@index([book]) + @@map("other_document_chapters") +} + +model JournalMeta { + id String @id @default(auto()) @map("_id") @db.ObjectId + book String @unique + data Json + + @@map("journal_meta") +} + +model JournalChapter { + id String @id @default(auto()) @map("_id") @db.ObjectId + book String + chapter Int + title String + type String + slug String + order Int + paragraphs String[] + + @@unique([book, slug]) + @@index([book]) + @@map("journal_chapters") +} + +model MagazineMeta { + id String @id @default(auto()) @map("_id") @db.ObjectId + book String @unique + data Json + + @@map("magazine_meta") +} + +model MagazineIssue { + id String @id @default(auto()) @map("_id") @db.ObjectId + book String + volume Int + part Int + title String + slug String + order Int + paragraphs String[] + paragraphsJson String? // JSON string fallback for large issues (BSON wire protocol workaround) + + @@unique([book, slug]) + @@index([book]) + @@index([book, volume]) + @@map("magazine_issues") +} + model GlossaryEntry { id String @id @default(auto()) @map("_id") @db.ObjectId term String @@ -285,6 +394,7 @@ model GlossaryEntry { | Śrī Caitanya-bhāgavata Commentary | `cbhc` | 4 | others-before-1977.html | | Bhagavāner Kathā | `bk` | 2 | others-before-1977.html (ScaGoudy) | | Prayers of King Kulaśekhara (1958) | `pk` | 6 | others-before-1977.html | +| Vaiśiṣṭyāṣṭakam | `va` | 8 | others-before-1977.html | ### Essays (stored in `essays` collection) @@ -297,18 +407,53 @@ model GlossaryEntry { | The Causes of Lord Caitanya's Incarnation | `causes-of-lord-caitanyas-incarnation` | `cc-essays` | 1957 | | The Confidential Reasons for Lord Caitanya's Appearance | `confidential-reasons-for-lord-caitanyas-appearance` | `cc-essays` | 1957 | | The Glories of Lord Nityānanda | `glories-of-lord-nityananda` | `cc-essays` | 1957 | +| Man Is the Architect of His Own Fortune | `man-is-the-architect-of-his-own-fortune` | `pre-1977-essays` | — | +| What Is the Matter with the World? | `what-is-the-matter-with-the-world` | `pre-1977-essays` | 1957 | +| Religion Is Godly Life | `religion-is-godly-life` | `pre-1977-essays` | — | +| Truth and Beauty | `truth-and-beauty` | `pre-1977-essays` | 1958 | +| Misuse of Public Fund in the Name of Religion | `misuse-of-public-fund-in-the-name-of-religion` | `pre-1977-essays` | 1958 | +| Perfection at Home | `perfection-at-home` | `pre-1977-essays` | — | +| How Should One Cultivate Human Spirit | `how-should-one-cultivate-human-spirit` | `pre-1977-essays` | 1961 | +| Viswa Dharma | `viswa-dharma` | `pre-1977-essays` | — | +| Glorification of Śrīla Bhaktisiddhānta Sarasvatī Ṭhākura | `glorification-of-srila-bhaktisiddhanta-sarasvati-thakura` | `pre-1977-essays` | 1952 | +| Who Is Crazy? (Back to Godhead #66) | `who-is-crazy-btg66` | `pre-1977-essays` | — | +| Manifestation of Dharma in Human Being | `manifestation-of-dharma-in-human-being` | `pre-1977-essays` | 1972 | +| Message of Krishna Consciousness | `message-of-krishna-consciousness` | `pre-1977-essays` | 1972 | +| A Wrong Civilization | `a-wrong-civilization` | `pre-1977-essays` | 1936 | | Vedānta-sūtra (Typed Transcript) | `vedanta-sutra-typed` | `typed-transcripts` | 1969 | | Shaktipujah | `shaktipujah` | `reviews` | — | | Devotee and the Divinity | `devotee-and-the-divinity` | `standalone` | — | -### Other Documents (stored in `book_meta` + `prose_chapters` with `od-` prefix) +### Other Documents (stored in `other_document_meta` + `other_document_chapters` collections) | Document | Code | Chapters | |----------|------|----------| -| An Analysis of Bhagwat Geeta | `od-analysis-bg` | 3 | -| Solution of Present Crisis by Bhagwat Geeta | `od-solution-crisis` | 4 | -| Śrī Caitanya Mahāprabhu | `od-sri-caitanya` | 3 | -| Conception of Geeta Nagari on the Principles of Gandhi | `od-geeta-nagari` | 1 | +| An Analysis of Bhagwat Geeta | `analysis-bg` | 3 | +| Solution of Present Crisis by Bhagwat Geeta | `solution-crisis` | 4 | +| Śrī Caitanya Mahāprabhu | `sri-caitanya` | 3 | +| Conception of Geeta Nagari on the Principles of Gandhi | `geeta-nagari` | 1 | +| The Real Peace Formula | `real-peace-formula` | 3 | +| International Institution for God Consciousness | `intl-institution-god-consciousness` | 1 | +| "Srimad Bhawatam" India's Message of Peace and Goodwill | `srimad-bhawatam-message` | 1 | +| Viraha Aṣṭaka | `viraha-astaka` | 8 | +| Vṛndāvana Bhajana | `vrndavana-bhajana` | 1 | +| Prayer unto the Lotus Feet of Kṛṣṇa | `prayer-unto-lotus-feet-of-krsna` | 1 | +| Märkine Bhägavata-dharma | `markine-bhagavata-dharma` | 1 | +| Jaladuta Pamphlet | `jaladuta-pamphlet` | 1 | + +### Journals (stored in `journal_meta` + `journal_chapters` collections) + +| Journal | Code | Chapters | Structure | +|---------|------|----------|-----------| +| The Jaladuta Diary, 1965 | `jaladuta-diary` | 85 | Preface, Introduction, 83 day entries grouped by month (Apr–Nov 1965) | +| The Beginning — The 1966 New York Journal | `the-beginning` | 215 | Introduction, 3 place-based chapters with day entries, Memoranda, Epilogue, 20 Excerpts | + +### Magazines (stored in `magazine_meta` + `magazine_issues` collections) + +| Magazine | Code | Issues | Articles | Flag | Source | +|----------|------|--------|----------|------|--------| +| Back to Godhead (1944–1960) | `btg` | 29 | 89 | Pre 1978 | btg-1960.html | +| Back to Godhead (1966–1973) | `btg-1978` | 30 | 1,035 | Pre 1978 | btg-1978.html | ### Other @@ -389,17 +534,33 @@ IAST transliteration → Devanagari via `indic-transliteration` library. Key fix | Route | Content | |-------|---------| -| `/essays` | All essays listing (10 essays across 4 groups) | +| `/essays` | Searchable essay grid (23 essays across 6 groups) | | `/essays/[slug]` | Individual essay reader | ### Other Documents | Route | Content | |-------|---------| -| `/other-documents` | All other documents listing | +| `/other-documents` | Searchable other documents grid (12 documents) | | `/other-documents/[slug]` | Document index (chapter listing) | | `/other-documents/[slug]/[chapter]` | Individual chapter reader | +### Journals + +| Route | Content | +|-------|---------| +| `/journals` | Journals index (grid) | +| `/journals/[slug]` | Journal chapter listing (hierarchical: place → month → day) | +| `/journals/[slug]/[chapter]` | Individual journal entry reader (slug-based, e.g., `apr-3`, `sep-17`, `excerpt-...`) | + +### Magazines + +| Route | Content | +|-------|---------| +| `/magazines` | Magazines index (grouped by flag: "Pre 1978 Editions") | +| `/magazines/[slug]` | Magazine issue listing (grouped by volume) | +| `/magazines/[slug]/[issue]` | Individual issue reader (slug-based, e.g., `v1-p1`, `no-17`) | + ### Other routes | Route | Content | @@ -407,6 +568,8 @@ IAST transliteration → Devanagari via `indic-transliteration` library. Key fix | `/` | Landing page | | `/books` | Searchable book grid | | `/glossary` | Searchable glossary (2,749 entries) | +| `/journals` | Journals (2 journals, 300 chapters) | +| `/magazines` | Magazines (2 magazines, 59 issues, Pre 1978) | | `/rtfs` | RTF viewer (gated by ALLOW_RTF=true) | | `/api/verify-access` | Password protection endpoint | diff --git a/app/letters/[slug]/page.tsx b/app/letters/[slug]/page.tsx new file mode 100644 index 00000000..3b55b82b --- /dev/null +++ b/app/letters/[slug]/page.tsx @@ -0,0 +1,117 @@ +export const dynamic = "force-dynamic"; + +import Link from "next/link"; +import { notFound } from "next/navigation"; +import { getLetter, getAdjacentLetters } from "@/lib/letters"; +import { PageHeader } from "@/components/page-header"; +import { Card, CardContent } from "@/components/ui/card"; +import { buttonVariants } from "@/components/ui/button"; +import { ChevronLeft, ChevronRight } from "lucide-react"; +import { cn } from "@/lib/utils"; + +export default async function LetterPage({ + params, +}: { + params: Promise<{ slug: string }>; +}) { + const { slug } = await params; + + const letter = await getLetter(slug); + if (!letter) notFound(); + + const { prev, next } = await getAdjacentLetters(slug, letter.year); + + const backHref = letter.year ? `/letters/year/${letter.year}` : "/letters"; + const backLabel = letter.year ? `Letters from ${letter.year}` : "Letters"; + + return ( +
+ + + ← {backLabel} + +

+ Letter to {letter.recipientDisplay} +

+

+ {letter.date} + {letter.location ? ` · ${letter.location}` : ""} +

+ + +
+ {/* Top nav */} + + + {/* Letter content */} + + +
+ {letter.paragraphs.map((paragraph, i) => ( +
+ ))} +
+ + + + {/* Bottom nav */} + +
+
+ ); +} + +function LetterNav({ + prev, + next, +}: { + prev: { slug: string; recipientDisplay: string } | null; + next: { slug: string; recipientDisplay: string } | null; +}) { + return ( +
+ {prev ? ( + + + + {prev.recipientDisplay} + + Previous + + ) : ( +
+ )} + + {next ? ( + + + {next.recipientDisplay} + + Next + + + ) : ( +
+ )} +
+ ); +} diff --git a/app/letters/letters-browser.tsx b/app/letters/letters-browser.tsx new file mode 100644 index 00000000..71846dac --- /dev/null +++ b/app/letters/letters-browser.tsx @@ -0,0 +1,285 @@ +"use client"; + +import { useState, useMemo } from "react"; +import Link from "next/link"; +import { + Card, + CardContent, +} from "@/components/ui/card"; +import { Badge } from "@/components/ui/badge"; +import { Input } from "@/components/ui/input"; +import { cn } from "@/lib/utils"; +import { Search, ChevronLeft, ChevronRight } from "lucide-react"; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export type YearGroup = { year: number; count: number }; +export type RecipientGroup = { name: string; count: number }; +export type LocationGroup = { name: string; count: number }; + +// --------------------------------------------------------------------------- +// Diacritic-aware normalize (matches books-grid pattern) +// --------------------------------------------------------------------------- + +function normalize(text: string): string { + return text + .toLowerCase() + .replace(/ṛ/g, "r") + .replace(/ṝ/g, "r") + .replace(/ṣ/g, "s") + .replace(/ś/g, "s") + .replace(/ṭ/g, "t") + .replace(/ḍ/g, "d") + .replace(/ṇ/g, "n") + .replace(/ñ/g, "n") + .replace(/ṅ/g, "n") + .replace(/ḥ/g, "h") + .replace(/ṁ/g, "m") + .replace(/ṃ/g, "m") + .replace(/ḷ/g, "l") + .replace(/ā/g, "a") + .replace(/ī/g, "i") + .replace(/ū/g, "u") + .normalize("NFD") + .replace(/[\u0300-\u036f]/g, ""); +} + +// --------------------------------------------------------------------------- +// Pagination +// --------------------------------------------------------------------------- + +const PAGE_SIZE_RECIPIENTS = 50; +const PAGE_SIZE_LOCATIONS = 60; +const PAGE_SIZE_YEARS = 100; // years are small cards, show all usually + +function Pagination({ + page, + totalPages, + onPageChange, +}: { + page: number; + totalPages: number; + onPageChange: (p: number) => void; +}) { + if (totalPages <= 1) return null; + + return ( +
+ + + + Page {page} of {totalPages} + + + +
+ ); +} + +// --------------------------------------------------------------------------- +// Component +// --------------------------------------------------------------------------- + +export function LettersBrowser({ + currentView, + years, + recipients, + locations, +}: { + currentView: string; + years: YearGroup[]; + recipients: RecipientGroup[]; + locations: LocationGroup[]; +}) { + const [query, setQuery] = useState(""); + const [page, setPage] = useState(1); + + // Reset page when query or view changes + const handleQueryChange = (q: string) => { + setQuery(q); + setPage(1); + }; + + // --- Filtered data --- + + const filteredYears = useMemo(() => { + if (!query.trim()) return years; + const q = normalize(query.trim()); + return years.filter((yg) => String(yg.year).includes(q)); + }, [years, query]); + + const filteredRecipients = useMemo(() => { + if (!query.trim()) return recipients; + const q = normalize(query.trim()); + return recipients.filter((rg) => normalize(rg.name).includes(q)); + }, [recipients, query]); + + const filteredLocations = useMemo(() => { + if (!query.trim()) return locations; + const q = normalize(query.trim()); + return locations.filter((lg) => normalize(lg.name).includes(q)); + }, [locations, query]); + + // --- Paginated data --- + + const activeData = + currentView === "year" + ? filteredYears + : currentView === "recipient" + ? filteredRecipients + : filteredLocations; + + const pageSize = + currentView === "year" + ? PAGE_SIZE_YEARS + : currentView === "recipient" + ? PAGE_SIZE_RECIPIENTS + : PAGE_SIZE_LOCATIONS; + + const totalPages = Math.max(1, Math.ceil(activeData.length / pageSize)); + const safePage = Math.min(page, totalPages); + const startIdx = (safePage - 1) * pageSize; + const endIdx = startIdx + pageSize; + + const placeholder = + currentView === "year" + ? "Search years..." + : currentView === "recipient" + ? "Search recipients..." + : "Search locations..."; + + const resultCount = activeData.length; + const totalCount = + currentView === "year" + ? years.length + : currentView === "recipient" + ? recipients.length + : locations.length; + + return ( +
+ {/* Search */} +
+ + handleQueryChange(e.target.value)} + className="pl-9" + /> +
+ + {/* Result count */} + {query.trim() && ( +

+ {resultCount === 0 + ? "No results found" + : `${resultCount} of ${totalCount} ${currentView === "year" ? "years" : currentView === "recipient" ? "recipients" : "locations"}`} +

+ )} + + {/* Year view */} + {currentView === "year" && ( +
+ {(filteredYears.slice(startIdx, endIdx) as YearGroup[]).map((yg) => ( + + + +

+ {yg.year} +

+

+ {yg.count} {yg.count === 1 ? "letter" : "letters"} +

+ + + + ))} +
+ )} + + {/* Recipient view */} + {currentView === "recipient" && ( +
+ {(filteredRecipients.slice(startIdx, endIdx) as RecipientGroup[]).map( + (rg) => ( + + + +

+ {rg.name} +

+ + {rg.count} + + + + + ), + )} +
+ )} + + {/* Location view */} + {currentView === "location" && ( +
+ {(filteredLocations.slice(startIdx, endIdx) as LocationGroup[]).map( + (lg) => ( + + + +

{lg.name}

+

+ {lg.count} {lg.count === 1 ? "letter" : "letters"} +

+ + + + ), + )} +
+ )} + + {/* Pagination */} + +
+ ); +} diff --git a/app/letters/letters-nav.tsx b/app/letters/letters-nav.tsx new file mode 100644 index 00000000..5e826883 --- /dev/null +++ b/app/letters/letters-nav.tsx @@ -0,0 +1,37 @@ +"use client"; + +import Link from "next/link"; +import { cn } from "@/lib/utils"; +import { Calendar, User, MapPin } from "lucide-react"; + +const views = [ + { key: "year", label: "By Year", icon: Calendar }, + { key: "recipient", label: "By Recipient", icon: User }, + { key: "location", label: "By Location", icon: MapPin }, +]; + +export function LettersNav({ currentView }: { currentView: string }) { + return ( +
+ {views.map((v) => { + const Icon = v.icon; + const isActive = currentView === v.key; + return ( + + + {v.label} + + ); + })} +
+ ); +} diff --git a/app/letters/location/[name]/page.tsx b/app/letters/location/[name]/page.tsx new file mode 100644 index 00000000..ca8baede --- /dev/null +++ b/app/letters/location/[name]/page.tsx @@ -0,0 +1,64 @@ +export const dynamic = "force-dynamic"; + +import Link from "next/link"; +import { notFound } from "next/navigation"; +import { getLettersByLocation } from "@/lib/letters"; +import { PageHeader } from "@/components/page-header"; +import { Card, CardContent } from "@/components/ui/card"; + +export default async function LettersByLocationPage({ + params, +}: { + params: Promise<{ name: string }>; +}) { + const { name: encodedName } = await params; + const name = decodeURIComponent(encodedName); + + const letters = await getLettersByLocation(name); + if (letters.length === 0) notFound(); + + return ( +
+ + + ← Letters by Location + +

+ Letters from {name} +

+

+ {letters.length} {letters.length === 1 ? "letter" : "letters"} +

+ + +
+ + +
    + {letters.map((letter) => ( +
  • + +
    +

    + To: {letter.recipientDisplay} +

    +

    + {letter.date || "Undated"} +

    +
    + +
    • + ))} +
+ + +
+
+ ); +} diff --git a/app/letters/page.tsx b/app/letters/page.tsx new file mode 100644 index 00000000..1de0d238 --- /dev/null +++ b/app/letters/page.tsx @@ -0,0 +1,55 @@ +export const dynamic = "force-dynamic"; + +import Link from "next/link"; +import { PageHeader } from "@/components/page-header"; +import { + getLetterYears, + getLetterRecipients, + getLetterLocations, + getTotalLetterCount, +} from "@/lib/letters"; +import { LettersNav } from "./letters-nav"; +import { LettersBrowser } from "./letters-browser"; + +export default async function LettersPage({ + searchParams, +}: { + searchParams: Promise<{ view?: string }>; +}) { + const { view = "year" } = await searchParams; + + const [years, recipients, locations, totalCount] = await Promise.all([ + getLetterYears(), + getLetterRecipients(), + getLetterLocations(), + getTotalLetterCount(), + ]); + + return ( +
+ + + ← Home + +

Letters

+

+ {totalCount.toLocaleString()} letters by Śrīla Prabhupāda (1947–1977) +

+ + +
+ + + +
+
+ ); +} diff --git a/app/letters/recipient/[name]/page.tsx b/app/letters/recipient/[name]/page.tsx new file mode 100644 index 00000000..a95cc89c --- /dev/null +++ b/app/letters/recipient/[name]/page.tsx @@ -0,0 +1,65 @@ +export const dynamic = "force-dynamic"; + +import Link from "next/link"; +import { notFound } from "next/navigation"; +import { getLettersByRecipient } from "@/lib/letters"; +import { PageHeader } from "@/components/page-header"; +import { Card, CardContent } from "@/components/ui/card"; + +export default async function LettersByRecipientPage({ + params, +}: { + params: Promise<{ name: string }>; +}) { + const { name: encodedName } = await params; + const name = decodeURIComponent(encodedName); + + const letters = await getLettersByRecipient(name); + if (letters.length === 0) notFound(); + + return ( +
+ + + ← Letters by Recipient + +

+ Letters to or including {name} +

+

+ {letters.length} {letters.length === 1 ? "letter" : "letters"} +

+ + +
+ + +
    + {letters.map((letter) => ( +
  • + +
    +

    + To: {letter.recipientDisplay} +

    +

    + {letter.date || "Undated"} + {letter.location ? ` · ${letter.location}` : ""} +

    +
    + +
    • + ))} +
+ + +
+
+ ); +} diff --git a/app/letters/year/[year]/page.tsx b/app/letters/year/[year]/page.tsx new file mode 100644 index 00000000..40af1aaf --- /dev/null +++ b/app/letters/year/[year]/page.tsx @@ -0,0 +1,66 @@ +export const dynamic = "force-dynamic"; + +import Link from "next/link"; +import { notFound } from "next/navigation"; +import { getLettersByYear } from "@/lib/letters"; +import { PageHeader } from "@/components/page-header"; +import { Card, CardContent } from "@/components/ui/card"; + +export default async function LettersByYearPage({ + params, +}: { + params: Promise<{ year: string }>; +}) { + const { year: yearStr } = await params; + const year = parseInt(yearStr, 10); + if (isNaN(year)) notFound(); + + const letters = await getLettersByYear(year); + if (letters.length === 0) notFound(); + + return ( +
+ + + ← Letters by Year + +

+ Letters from {year} +

+

+ {letters.length} {letters.length === 1 ? "letter" : "letters"} +

+ + +
+ + +
    + {letters.map((letter) => ( +
  • + +
    +

    + To: {letter.recipientDisplay} +

    +

    + {letter.date} + {letter.location ? ` · ${letter.location}` : ""} +

    +
    + +
    • + ))} +
+ + +
+
+ ); +} diff --git a/app/magazines/[slug]/[issue]/page.tsx b/app/magazines/[slug]/[issue]/page.tsx new file mode 100644 index 00000000..bbc36385 --- /dev/null +++ b/app/magazines/[slug]/[issue]/page.tsx @@ -0,0 +1,223 @@ +export const dynamic = "force-dynamic"; + +import Link from "next/link"; +import { notFound } from "next/navigation"; +import { + getMagazineContent, + getMagazineMeta, + getMagazineSidebarItems, + slugToBookCode, + type MagazineSidebarItem, +} from "@/lib/magazines"; +import { PageHeader } from "@/components/page-header"; +import { Card, CardContent } from "@/components/ui/card"; +import { buttonVariants } from "@/components/ui/button"; +import { ChevronLeft, ChevronRight } from "lucide-react"; +import { cn } from "@/lib/utils"; +import { MobileNav, type MobileNavGroup } from "@/components/mobile-nav"; + +export default async function MagazineIssuePage({ + params, +}: { + params: Promise<{ slug: string; issue: string }>; +}) { + const { slug, issue: issueSlug } = await params; + const bookCode = slugToBookCode(slug); + + const [meta, content, sidebarItems] = await Promise.all([ + getMagazineMeta(bookCode), + getMagazineContent(bookCode, issueSlug), + getMagazineSidebarItems(bookCode), + ]); + + if (!meta || !content) notFound(); + + const navItems = sidebarItems.filter((item) => item.type !== "heading"); + const currentIdx = navItems.findIndex((item) => item.slug === issueSlug); + const prev = currentIdx > 0 ? navItems[currentIdx - 1] : null; + const next = + currentIdx < navItems.length - 1 ? navItems[currentIdx + 1] : null; + + return ( +
+ + + ← {meta.title} + +

+ {content.title} +

+ + +
+ + +
+ + + + +
+ {content.paragraphs.map((paragraph, i) => ( +
+ ))} +
+ + + + +
+
+
+ ); +} + +function IssueSidebar({ + items, + currentSlug, + magSlug, +}: { + items: MagazineSidebarItem[]; + currentSlug: string; + magSlug: string; +}) { + return ( + + ); +} + +function IssueNav({ + prev, + next, + navItems, + currentSlug, + magSlug, + magTitle, + showMobileNav = false, +}: { + prev: MagazineSidebarItem | null; + next: MagazineSidebarItem | null; + navItems: MagazineSidebarItem[]; + currentSlug: string; + magSlug: string; + magTitle: string; + showMobileNav?: boolean; +}) { + const mobileNavGroups: MobileNavGroup[] = []; + if (showMobileNav && navItems.length > 0) { + mobileNavGroups.push({ + title: "Issues", + items: navItems.map((i) => ({ + slug: i.slug, + label: i.label, + href: `/magazines/${magSlug}/${i.slug}`, + type: i.type, + })), + }); + } + + return ( +
+
+ {showMobileNav && ( + + )} + {prev ? ( + + + + {prev.label} + + Previous + + ) : ( +
+ )} +
+ + {next ? ( + + + {next.label} + + Next + + + ) : ( +
+ )} +
+ ); +} diff --git a/app/magazines/[slug]/page.tsx b/app/magazines/[slug]/page.tsx new file mode 100644 index 00000000..43017a21 --- /dev/null +++ b/app/magazines/[slug]/page.tsx @@ -0,0 +1,96 @@ +export const dynamic = "force-dynamic"; + +import Link from "next/link"; +import { notFound } from "next/navigation"; +import { + getMagazineMeta, + slugToBookCode, + type MagazineChapterEntry, +} from "@/lib/magazines"; +import { PageHeader } from "@/components/page-header"; +import { Card, CardContent } from "@/components/ui/card"; + +export default async function MagazineIndexPage({ + params, +}: { + params: Promise<{ slug: string }>; +}) { + const { slug } = await params; + const bookCode = slugToBookCode(slug); + const meta = await getMagazineMeta(bookCode); + if (!meta) notFound(); + + return ( +
+ + + ← Magazines + +

+ {meta.title} +

+

{meta.author}

+ + +
+ {meta.description && ( +

{meta.description}

+ )} + + {meta.chapters.map((ch: MagazineChapterEntry, idx: number) => { + if ("slug" in ch && !("subChapters" in ch)) { + return ( + + + +

{ch.title}

+ + + + ); + } + + if ("subChapters" in ch) { + const group = ch as { + title: string; + subChapters: { slug: string; title: string }[]; + }; + return ( +
+

+ {group.title} +

+ + +
    + {group.subChapters.map((sc) => ( +
  • + +

    + {sc.title} +

    + +
    • + ))} +
+ + +
+ ); + } + + return null; + })} +
+
+ ); +} diff --git a/app/magazines/page.tsx b/app/magazines/page.tsx new file mode 100644 index 00000000..60d9f85e --- /dev/null +++ b/app/magazines/page.tsx @@ -0,0 +1,110 @@ +export const dynamic = "force-dynamic"; + +import Link from "next/link"; +import { PageHeader } from "@/components/page-header"; +import { + Card, + CardHeader, + CardTitle, + CardDescription, + CardContent, +} from "@/components/ui/card"; +import { Badge } from "@/components/ui/badge"; +import { getAllMagazines, bookCodeToSlug } from "@/lib/magazines"; + +export default async function MagazinesPage() { + const magazines = await getAllMagazines(); + + // Group magazines by flag + const grouped: Record = {}; + for (const mag of magazines) { + const key = mag.flag ?? "Other"; + if (!grouped[key]) grouped[key] = []; + grouped[key].push(mag); + } + + // Define display order for groups + const groupOrder = ["Pre 1978", "Other"]; + const groupLabels: Record = { + "Pre 1978": "Pre 1978 Editions", + Other: "Other Editions", + }; + + const sortedGroups = Object.keys(grouped).sort( + (a, b) => groupOrder.indexOf(a) - groupOrder.indexOf(b), + ); + + // If there's only one group, don't show group headings + const showGroupHeadings = sortedGroups.length > 1; + + return ( +
+ + + ← Home + +

Magazines

+

+ Back to Godhead and other periodicals by Śrīla Prabhupāda +

+ + +
+ {sortedGroups.map((group) => ( +
+ {showGroupHeadings && ( +

+ {groupLabels[group] ?? group} +

+ )} + +
+ {grouped[group].map((mag) => ( + + + + + {mag.title} + + {mag.description && ( + + {mag.description} + + )} + + +
+ {mag.flag && ( + + {mag.flag} + + )} + {mag.year && ( + + Since {mag.year} + + )} + {mag.totalIssues > 0 && ( + + {mag.totalIssues} issues + + )} +
+ + + + ))} +
+
+ ))} +
+
+ ); +} diff --git a/app/page.tsx b/app/page.tsx index b70c7b22..4ef5d7cd 100644 --- a/app/page.tsx +++ b/app/page.tsx @@ -16,6 +16,8 @@ import { PenLine, FileStack, NotebookPen, + Newspaper, + Mail, } from "lucide-react"; const allowRtf = process.env.ALLOW_RTF === "true"; @@ -27,11 +29,11 @@ export default function Home() { -
-
+
+
{/* Structured Books (JSON) */} - - + +
@@ -58,8 +60,8 @@ export default function Home() { {/* Essays */} - - + +
@@ -83,9 +85,64 @@ export default function Home() { + {/* Magazines */} + + + +
+
+ +
+
+ + Magazines + + + Back to Godhead & other periodicals + +
+
+ + +

+ The original Back to Godhead magazine founded by Śrīla + Prabhupāda in 1944 — 29 issues across 3 volumes with 89 + articles on Kṛṣṇa consciousness, Vedic philosophy, and + spiritual life. +

+ + + + + {/* Letters */} + + + +
+
+ +
+
+ Letters + + 6,800+ letters (1947–1977) + +
+
+ + +

+ Śrīla Prabhupāda's personal correspondence — over 6,800 + letters to disciples, Godbrothers, government officials, and + well-wishers. Browse by year, recipient, or location. +

+ + + + {/* Journals */} - - + +
@@ -113,8 +170,8 @@ export default function Home() { {/* Glossary */} - - + +
@@ -141,8 +198,8 @@ export default function Home() { {/* Other Documents */} - - + +
@@ -171,8 +228,8 @@ export default function Home() { {/* RTF Document Viewer — only visible when ALLOW_RTF=true */} {allowRtf && ( - - + +
diff --git a/lib/letters.ts b/lib/letters.ts new file mode 100644 index 00000000..034719f7 --- /dev/null +++ b/lib/letters.ts @@ -0,0 +1,249 @@ +/** + * Data access layer for Letters. + * + * Provides queries for browsing letters by year, recipient, and location. + * Uses the Prisma Letter model (collection: letters). + * + * Recipients are stored as an array (e.g., ["Mukunda", "Janaki"]). + * The recipient list aggregates individual names across all letters, + * and searching for "Mukunda" finds all letters where Mukunda is + * ANY recipient (individual or group). + */ + +import { prisma } from "@/lib/db"; + +// Types +export type LetterSummary = { + slug: string; + recipients: string[]; + recipientDisplay: string; + date: string | null; + year: number | null; + month: number | null; + location: string | null; +}; + +export type LetterFull = { + slug: string; + recipients: string[]; + recipientDisplay: string; + date: string | null; + dateRef: string | null; + year: number | null; + month: number | null; + day: number | null; + location: string | null; + salutation: string | null; + paragraphs: string[]; +}; + +export type YearGroup = { + year: number; + count: number; +}; + +export type RecipientGroup = { + name: string; + count: number; +}; + +export type LocationGroup = { + name: string; + count: number; +}; + +// Sanitize +function sanitizeHtml(html: string): string { + let clean = html.replace(/][\s\S]*?<\/script>/gi, ""); + clean = clean.replace(/][\s\S]*?<\/style>/gi, ""); + clean = clean.replace(/][\s\S]*?<\/iframe>/gi, ""); + clean = clean.replace( + /<\/?(?:script|style|iframe|object|embed|form|input|textarea|select|button)[^>]*>/gi, + "", + ); + clean = clean.replace(/\s+on\w+\s*=\s*(?:"[^"]*"|'[^']*'|[^\s>]+)/gi, ""); + return clean; +} + +// --- Index/overview queries --- + +export async function getLetterYears(): Promise { + const letters = await prisma.letter.findMany({ + where: { year: { not: null } }, + select: { year: true }, + }); + + const counts: Record = {}; + for (const l of letters) { + if (l.year) { + counts[l.year] = (counts[l.year] || 0) + 1; + } + } + + return Object.entries(counts) + .map(([year, count]) => ({ year: Number(year), count })) + .filter((g) => g.year >= 1940 && g.year <= 1980) + .sort((a, b) => a.year - b.year); +} + +/** + * Get all unique individual recipients with letter counts. + * + * Each letter has a `recipients` array (e.g., ["Mukunda", "Janaki"]). + * This function aggregates across all letters so that "Mukunda" appears + * once with count = total letters where Mukunda is ANY recipient. + */ +export async function getLetterRecipients(): Promise { + const letters = await prisma.letter.findMany({ + select: { recipients: true }, + }); + + const counts: Record = {}; + for (const l of letters) { + for (const r of l.recipients) { + counts[r] = (counts[r] || 0) + 1; + } + } + + return Object.entries(counts) + .map(([name, count]) => ({ name, count })) + .sort((a, b) => b.count - a.count); +} + +export async function getLetterLocations(): Promise { + const letters = await prisma.letter.findMany({ + where: { location: { not: null } }, + select: { location: true }, + }); + + const counts: Record = {}; + for (const l of letters) { + if (l.location) { + counts[l.location] = (counts[l.location] || 0) + 1; + } + } + + return Object.entries(counts) + .map(([name, count]) => ({ name, count })) + .sort((a, b) => b.count - a.count); +} + +// --- Letter list queries --- + +export async function getLettersByYear(year: number): Promise { + const letters = await prisma.letter.findMany({ + where: { year }, + orderBy: [{ month: "asc" }, { day: "asc" }], + select: { + slug: true, + recipients: true, + recipientDisplay: true, + date: true, + year: true, + month: true, + location: true, + }, + }); + return letters; +} + +/** + * Get all letters where the given name is ANY recipient. + * + * Uses Prisma's `has` filter on the recipients array. + * Searching for "Mukunda" returns letters addressed to: + * - "Mukunda" (individual) + * - "Mukunda, Janaki" (group containing Mukunda) + * - "Yamuna, Mukunda, Syamasundara" (group containing Mukunda) + */ +export async function getLettersByRecipient( + recipient: string, +): Promise { + const letters = await prisma.letter.findMany({ + where: { recipients: { has: recipient } }, + orderBy: [{ year: "asc" }, { month: "asc" }, { day: "asc" }], + select: { + slug: true, + recipients: true, + recipientDisplay: true, + date: true, + year: true, + month: true, + location: true, + }, + }); + return letters; +} + +export async function getLettersByLocation( + location: string, +): Promise { + const letters = await prisma.letter.findMany({ + where: { location }, + orderBy: [{ year: "asc" }, { month: "asc" }, { day: "asc" }], + select: { + slug: true, + recipients: true, + recipientDisplay: true, + date: true, + year: true, + month: true, + location: true, + }, + }); + return letters; +} + +// --- Single letter --- + +export async function getLetter(slug: string): Promise { + const letter = await prisma.letter.findUnique({ where: { slug } }); + if (!letter) return null; + + return { + slug: letter.slug, + recipients: letter.recipients, + recipientDisplay: letter.recipientDisplay, + date: letter.date, + dateRef: letter.dateRef, + year: letter.year, + month: letter.month, + day: letter.day, + location: letter.location, + salutation: letter.salutation, + paragraphs: letter.paragraphs.map(sanitizeHtml), + }; +} + +// --- Navigation helpers --- + +export async function getAdjacentLetters( + slug: string, + year: number | null, +): Promise<{ prev: LetterSummary | null; next: LetterSummary | null }> { + if (!year) return { prev: null, next: null }; + + const yearLetters = await prisma.letter.findMany({ + where: { year }, + orderBy: [{ month: "asc" }, { day: "asc" }, { slug: "asc" }], + select: { + slug: true, + recipients: true, + recipientDisplay: true, + date: true, + year: true, + month: true, + location: true, + }, + }); + + const idx = yearLetters.findIndex((l) => l.slug === slug); + return { + prev: idx > 0 ? yearLetters[idx - 1] : null, + next: idx < yearLetters.length - 1 ? yearLetters[idx + 1] : null, + }; +} + +export async function getTotalLetterCount(): Promise { + return prisma.letter.count(); +} diff --git a/lib/magazines.ts b/lib/magazines.ts new file mode 100644 index 00000000..37268b78 --- /dev/null +++ b/lib/magazines.ts @@ -0,0 +1,282 @@ +/** + * Data access layer for Magazines (Back to Godhead, etc.). + * + * Uses dedicated Prisma models: + * - MagazineMeta (collection: magazine_meta) + * - MagazineIssue (collection: magazine_issues) + */ + +import { prisma } from "@/lib/db"; +import type { JsonValue } from "@prisma/client/runtime/library"; +import fs from "fs"; +import path from "path"; + +const LOCAL_JSON_DIR = path.join(process.cwd(), "lib", "jsons", "magazines"); + +export type MagazineSubChapter = { slug: string; title: string }; +export type MagazineVolumeGroup = { + title: string; + subChapters: MagazineSubChapter[]; +}; +export type MagazineChapterEntry = MagazineSubChapter | MagazineVolumeGroup; + +export type MagazineMetaData = { + book: string; + title: string; + author: string; + year: number | null; + description: string | null; + totalIssues: number; + totalArticles: number; + flag: string | null; + chapters: MagazineChapterEntry[]; +}; + +export type MagazineIssueData = { + book: string; + volume: number; + part: number; + title: string; + slug: string; + paragraphs: string[]; +}; + +export type MagazineSummary = { + book: string; + title: string; + author: string; + year: number | null; + description: string | null; + totalIssues: number; + flag: string | null; +}; + +export type MagazineSidebarItem = { + slug: string; + label: string; + type: string; +}; + +function sanitizeHtml(html: string): string { + let clean = html.replace(/][\s\S]*?<\/script>/gi, ""); + clean = clean.replace(/][\s\S]*?<\/style>/gi, ""); + clean = clean.replace(/][\s\S]*?<\/iframe>/gi, ""); + clean = clean.replace( + /<\/?(?:script|style|iframe|object|embed|form|input|textarea|select|button)[^>]*>/gi, + "", + ); + clean = clean.replace(/\s+on\w+\s*=\s*(?:"[^"]*"|'[^']*'|[^\s>]+)/gi, ""); + clean = clean.replace(/href\s*=\s*"javascript:[^"]*"/gi, 'href=""'); + clean = clean.replace(/href\s*=\s*'javascript:[^']*'/gi, "href=''"); + return clean; +} + +function normalizeParagraphs(paragraphs: string[] | JsonValue): string[] { + let result: string[]; + if (typeof paragraphs === "string") { + try { + const parsed = JSON.parse(paragraphs); + result = Array.isArray(parsed) ? parsed : [paragraphs]; + } catch { + result = [paragraphs]; + } + } else if (Array.isArray(paragraphs)) { + result = paragraphs as string[]; + } else { + result = []; + } + return result.map(sanitizeHtml); +} + +function findLocalDirForBook(bookCode: string): string | null { + if (!fs.existsSync(LOCAL_JSON_DIR)) return null; + const entries = fs.readdirSync(LOCAL_JSON_DIR); + for (const entry of entries) { + const introPath = path.join(LOCAL_JSON_DIR, entry, "intro.json"); + if (fs.existsSync(introPath)) { + try { + const intro = JSON.parse(fs.readFileSync(introPath, "utf-8")); + if (intro.book === bookCode) return entry; + } catch { + /* skip */ + } + } + } + return null; +} + +function readLocalJson(filePath: string): T | null { + try { + if (!fs.existsSync(filePath)) return null; + return JSON.parse(fs.readFileSync(filePath, "utf-8")) as T; + } catch { + return null; + } +} + +function getAllLocalMetas(): MagazineMetaData[] { + if (!fs.existsSync(LOCAL_JSON_DIR)) return []; + const entries = fs.readdirSync(LOCAL_JSON_DIR).sort(); + const metas: MagazineMetaData[] = []; + for (const entry of entries) { + const intro = readLocalJson( + path.join(LOCAL_JSON_DIR, entry, "intro.json"), + ); + if (intro) metas.push(intro); + } + return metas; +} + +function getLocalMeta(bookCode: string): MagazineMetaData | null { + const dir = findLocalDirForBook(bookCode); + if (!dir) return null; + return readLocalJson( + path.join(LOCAL_JSON_DIR, dir, "intro.json"), + ); +} + +function getLocalIssue( + bookCode: string, + slug: string, +): MagazineIssueData | null { + const dir = findLocalDirForBook(bookCode); + if (!dir) return null; + const data = readLocalJson( + path.join(LOCAL_JSON_DIR, dir, "issues", `${slug}.json`), + ); + if (!data) return null; + return { ...data, paragraphs: data.paragraphs.map(sanitizeHtml) }; +} + +// Public API + +export async function getAllMagazines(): Promise { + try { + const results = await prisma.magazineMeta.findMany({ + orderBy: { book: "asc" }, + }); + if (results.length > 0) { + return results.map((row) => { + const data = row.data as unknown as MagazineMetaData; + return { + book: row.book, + title: data.title, + author: data.author, + year: data.year ?? null, + description: data.description ?? null, + totalIssues: data.totalIssues, + flag: data.flag ?? null, + }; + }); + } + } catch { + /* fall through */ + } + const localMetas = getAllLocalMetas(); + return localMetas.map((meta) => ({ + book: meta.book, + title: meta.title, + author: meta.author, + year: meta.year ?? null, + description: meta.description ?? null, + totalIssues: meta.totalIssues, + flag: meta.flag ?? null, + })); +} + +export async function getMagazineMeta( + bookCode: string, +): Promise { + try { + const row = await prisma.magazineMeta.findUnique({ + where: { book: bookCode }, + }); + if (row) return row.data as unknown as MagazineMetaData; + } catch { + /* fall through */ + } + return getLocalMeta(bookCode); +} + +export async function getMagazineIssue( + bookCode: string, + slug: string, +): Promise { + if (!slug) return null; + try { + const doc = await prisma.magazineIssue.findUnique({ + where: { book_slug: { book: bookCode, slug } }, + }); + if (doc) { + // Support both paragraphsJson (string) and paragraphs (array) formats. + // The seed script stores paragraphs as a JSON string to avoid + // MongoDB BSON wire protocol issues with large string arrays. + let paras: string[]; + if (doc.paragraphsJson) { + try { + paras = JSON.parse(doc.paragraphsJson); + } catch { + paras = []; + } + } else { + paras = normalizeParagraphs(doc.paragraphs); + } + return { + book: doc.book, + volume: doc.volume, + part: doc.part, + title: doc.title, + slug: doc.slug, + paragraphs: paras, + }; + } + } catch { + /* fall through */ + } + return getLocalIssue(bookCode, slug); +} + +export async function getMagazineContent( + bookCode: string, + slug: string, +): Promise { + return getMagazineIssue(bookCode, slug); +} + +function isSubChapter(ch: MagazineChapterEntry): ch is MagazineSubChapter { + return "slug" in ch && !("subChapters" in ch); +} + +function hasSubChapters(ch: MagazineChapterEntry): ch is MagazineVolumeGroup { + return "subChapters" in ch && Array.isArray(ch.subChapters); +} + +export async function getMagazineSidebarItems( + bookCode: string, +): Promise { + const meta = await getMagazineMeta(bookCode); + if (!meta) return []; + + const items: MagazineSidebarItem[] = []; + function addEntry(entry: MagazineChapterEntry) { + if (isSubChapter(entry)) { + items.push({ slug: entry.slug, label: entry.title, type: "chapter" }); + } else if (hasSubChapters(entry)) { + items.push({ slug: "", label: entry.title, type: "heading" }); + for (const sc of entry.subChapters) { + addEntry(sc); + } + } + } + for (const ch of meta.chapters) { + addEntry(ch); + } + return items; +} + +export function bookCodeToSlug(bookCode: string): string { + return bookCode; +} +export function slugToBookCode(slug: string): string { + return slug; +} diff --git a/prisma/schema.prisma b/prisma/schema.prisma index ad0ba8f7..50719592 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -198,9 +198,87 @@ model JournalChapter { @@map("journal_chapters") } +// --------------------------------------------------------------------------- +// MagazineMeta — metadata for magazines (Back to Godhead, etc.) +// +// book: slug identifier (e.g. "btg") +// data: JSON blob containing title, author, description, volumes/issues list +// --------------------------------------------------------------------------- +model MagazineMeta { + id String @id @default(auto()) @map("_id") @db.ObjectId + book String @unique + data Json + + @@map("magazine_meta") +} + +// --------------------------------------------------------------------------- +// MagazineIssue — individual issues within a magazine +// +// book: slug identifier matching MagazineMeta.book +// volume: volume number +// part: part/issue number within the volume +// title: display title (e.g. "Volume 3, Part 5, May 5, 1956") +// slug: URL-friendly identifier (e.g. "v3-p5") +// order: sort order +// paragraphs: array of HTML paragraph strings (all articles concatenated) +// --------------------------------------------------------------------------- +model MagazineIssue { + id String @id @default(auto()) @map("_id") @db.ObjectId + book String + volume Int + part Int + title String + slug String + order Int + paragraphs String[] + paragraphsJson String? + + @@unique([book, slug]) + @@index([book]) + @@index([book, volume]) + @@map("magazine_issues") +} + // --------------------------------------------------------------------------- // GlossaryEntry — individual glossary term + definition // --------------------------------------------------------------------------- +// --------------------------------------------------------------------------- +// Letter — individual letters written by Śrīla Prabhupāda +// +// slug: unique identifier (e.g. "47-07-12-mahatma-gandhi", "68-01-15-hayagriva") +// recipients: array of individual recipient names (split from comma-separated) +// recipientDisplay: original recipient string for display (e.g. "Mukunda, Janaki") +// date: date string as written (e.g. "12 July, 1947") +// dateRef: reference code (e.g. "47-07-12") +// year: extracted year number for filtering/grouping +// month: extracted month number (1-12) for filtering +// day: extracted day number for filtering +// location: city/place where the letter was written +// salutation: the "Dear..." line +// paragraphs: array of HTML paragraph strings (letter body) +// --------------------------------------------------------------------------- +model Letter { + id String @id @default(auto()) @map("_id") @db.ObjectId + slug String @unique + recipients String[] + recipientDisplay String + date String? + dateRef String? + year Int? + month Int? + day Int? + location String? + salutation String? + paragraphs String[] + + @@index([year]) + @@index([recipients]) + @@index([location]) + @@index([year, month]) + @@map("letters") +} + model GlossaryEntry { id String @id @default(auto()) @map("_id") @db.ObjectId term String diff --git a/scripts/btg_1978_extract.py b/scripts/btg_1978_extract.py new file mode 100644 index 00000000..a57cec91 --- /dev/null +++ b/scripts/btg_1978_extract.py @@ -0,0 +1,717 @@ +#!/usr/bin/env python3 +"""Extract Back to Godhead magazine (1966-1973) from btg-1978.html. + +Each magazine issue becomes its own JSON file. +Issues are grouped by volume/era in the intro.json metadata. +Advertisement sections are excluded. + +Structure: + - p2 = Issue bookmark (bold, e.g., "Back to Godhead Magazine. Vol. 1, No. 01, October 23, 1966") + - p8 = Article bookmark within an issue + - p9 = Article display title (bold) + - p7, p5, p12, p4, p3, etc. = Body text + - Articles titled "Advertisement" are skipped entirely + +Output: + lib/jsons/magazines/btg-1978/intro.json + lib/jsons/magazines/btg-1978/issues/no-01.json + lib/jsons/magazines/btg-1978/issues/no-02.json + ... + lib/jsons/magazines/btg-1978/issues/no-54.json + +Usage: + python3 scripts/btg_1978_extract.py +""" + +import json +import os +import re +import sys + +from bs4 import BeautifulSoup, Tag + +# --------------------------------------------------------------------------- +# ScaGoudy → IAST Unicode mapping +# --------------------------------------------------------------------------- +SCAGOUDY_TO_IAST = { + "ä": "ā", + "Ä": "Ā", + "å": "ṛ", + "Å": "Ṛ", + "ñ": "ṣ", + "Ñ": "Ṣ", + "ö": "ṭ", + "Ö": "Ṭ", + "ë": "ṇ", + "Ë": "Ṇ", + "ò": "ḍ", + "Ò": "Ḍ", + "ç": "ś", + "Ç": "Ś", + "ì": "ṅ", + "Ì": "Ṅ", + "ï": "ñ", + "Ï": "Ñ", + "ü": "ū", + "Ü": "Ū", + "é": "ī", + "É": "Ī", + "à": "ṁ", + "ù": "ḥ", + "è": "ṝ", + "ÿ": "ḷ", +} + + +def decode_scagoudy(text: str) -> str: + """Replace ScaGoudy-encoded characters with proper IAST Unicode.""" + for src, dst in SCAGOUDY_TO_IAST.items(): + text = text.replace(src, dst) + return text + + +def get_css_class(tag: Tag) -> str: + classes = tag.get("class", []) + return classes[0] if classes else "" + + +def inner_html(tag: Tag) -> str: + """Get inner HTML of a tag, decode ScaGoudy, clean up spans.""" + html = "".join(str(child) for child in tag.children) + html = decode_scagoudy(html) + + # text → text (italic marker in btg-1978) + html = re.sub( + r']*class="s2"[^>]*>(.*?)', r"\1", html, flags=re.DOTALL + ) + + # Remove all other span tags but keep inner content + html = re.sub(r"]*>(.*?)", r"\1", html, flags=re.DOTALL) + + # Remove Apple-tab-span and tab characters + html = html.replace("\t", " ") + + # Collapse multiple spaces (but not around
) + html = re.sub(r"[ ]+", " ", html) + html = html.strip() + + # Remove leading
tags + html = re.sub(r"^([\s]*)+", "", html) + # Remove trailing
tags + html = re.sub(r"([\s]*)+$", "", html) + + return html + + +def write_json(data: dict, path: str): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + f.write("\n") + + +# --------------------------------------------------------------------------- +# Parse issue info from p2 text +# --------------------------------------------------------------------------- + + +def parse_issue_info(text: str): + """Parse issue info from the p2 bookmark text. + + Handles various formats: + "Back to Godhead Magazine. Vol. 1, No. 01, October 23, 1966" + "Back to Godhead Magazine. Vol. 1, No. 06, January 20, 1967" + "Back to Godhead Magazine. [VOL II No 1.] No. 16, 1968" + "Back to Godhead Magazine VOL II. No 2. [#17], 1968" + "Back to Godhead Magazine no. 30, 1969" + "Back to Godhead Magazine No. 36, 1970" + + Returns (issue_number, volume, display_title, date_str, slug) or None. + """ + text = text.strip() + # Remove "Back to Godhead Magazine" prefix variations + text_clean = re.sub( + r"^Back to Godhead Magazine\.?\s*", "", text, flags=re.IGNORECASE + ).strip() + + issue_num = None + volume = None + date_str = "" + + # Try to extract issue number from various patterns + + # Pattern 1: "Vol. X, No. YY, date" or "Vol. X No. YY, date" + m = re.search( + r"Vol\.?\s*(\d+)\.?\s*,?\s*No\.?\s*(\d+)\s*,?\s*(.*)", text_clean, re.IGNORECASE + ) + if m: + volume = int(m.group(1)) + issue_num = int(m.group(2)) + date_str = m.group(3).strip().rstrip(",. ") + + # Pattern 2: "[VOL II No 1.] No. 16, 1968" — use the absolute number + if issue_num is None: + m = re.search(r"No\.?\s*(\d+)\s*,?\s*(\d{4})", text_clean, re.IGNORECASE) + if m: + issue_num = int(m.group(1)) + date_str = m.group(2).strip() + + # Pattern 3: "VOL II. No 2. [#17], 1968" + if issue_num is None: + m = re.search(r"#(\d+)", text_clean) + if m: + issue_num = int(m.group(1)) + + # Pattern 4: "no. 30, 1969" or "no 31, 1969" + if issue_num is None: + m = re.search(r"no\.?\s*(\d+)\s*,?\s*(.*)", text_clean, re.IGNORECASE) + if m: + issue_num = int(m.group(1)) + date_str = m.group(2).strip().rstrip(",. ") + + # Try to extract volume from text if not found yet + if volume is None: + m = re.search(r"VOL\.?\s*(?:II|2)", text_clean, re.IGNORECASE) + if m: + volume = 2 + else: + m = re.search(r"VOL\.?\s*(?:I|1)", text_clean, re.IGNORECASE) + if m: + volume = 1 + + if issue_num is None: + print(f" ⚠ Could not parse issue number from: {text}") + return None + + # Extract year from date_str if present + year_match = re.search(r"(\d{4})", date_str) + year = int(year_match.group(1)) if year_match else None + + # Build display title + if volume and issue_num <= 17: + display_title = f"Vol. {volume}, No. {issue_num:02d}" + if date_str: + display_title += f", {date_str}" + else: + display_title = f"No. {issue_num}" + if date_str: + display_title += f", {date_str}" + + slug = f"no-{issue_num:02d}" + + return issue_num, volume, display_title, date_str, year, slug + + +# --------------------------------------------------------------------------- +# Main extraction +# --------------------------------------------------------------------------- + + +def main(): + base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + html_path = os.path.join(base_dir, "public", "html", "btg-1978.html") + output_dir = os.path.join(base_dir, "lib", "jsons", "magazines", "btg-1978") + issues_dir = os.path.join(output_dir, "issues") + + if not os.path.exists(html_path): + print(f"ERROR: {html_path} not found!") + sys.exit(1) + + print(f"Reading {html_path}...") + with open(html_path, "r", encoding="utf-8") as f: + content = f.read() + + soup = BeautifulSoup(content, "html.parser") + all_tags = soup.find_all("p") + print(f" Parsed {len(all_tags)}

tags total") + + # --------------------------------------------------------------------------- + # Phase 1: Find all issue boundaries + # --------------------------------------------------------------------------- + issue_boundaries = [] # List of (tag_index, issue_num, volume, display_title, date_str, year, slug) + + for i, tag in enumerate(all_tags): + cls = get_css_class(tag) + if cls == "p2": + text = tag.get_text().strip() + if "Back to Godhead" in text: + parsed = parse_issue_info(text) + if parsed: + issue_num, volume, display_title, date_str, year, slug = parsed + issue_boundaries.append( + (i, issue_num, volume, display_title, date_str, year, slug) + ) + + print(f"\n Found {len(issue_boundaries)} issues") + for ib in issue_boundaries: + tag_idx, inum, vol, title, ds, yr, slug = ib + print(f" tag[{tag_idx}]: #{inum} (Vol {vol}) {title} → {slug}") + + # --------------------------------------------------------------------------- + # Phase 2: Extract articles and paragraphs for each issue + # --------------------------------------------------------------------------- + + # Known skip classes (bookmarks, empty lines) + SKIP_CLASSES = {"p2", "p11", "p28", "p24"} + + issues_data = [] + skipped_ads = 0 + + # Maximum paragraphs per issue — prevents mega-issues from absorbing + # unmarked content that belongs to missing intermediate issues. + MAX_PARAGRAPHS_PER_ISSUE = 500 + + for idx, ( + start_tag_idx, + issue_num, + vol, + display_title, + date_str, + year, + slug, + ) in enumerate(issue_boundaries): + # Determine end boundary + if idx + 1 < len(issue_boundaries): + end_tag_idx = issue_boundaries[idx + 1][0] + else: + end_tag_idx = len(all_tags) + + issue_tags = all_tags[start_tag_idx:end_tag_idx] + + paragraphs = [] + articles = [] + in_advertisement = False + + for tag in issue_tags: + cls = get_css_class(tag) + text = tag.get_text().strip() + + # Skip bookmark tags and empty lines + if cls in SKIP_CLASSES: + continue + if not text: + continue + + # p8 = Article bookmark + if cls == "p8": + article_title = decode_scagoudy(text).strip() + + # Check if this is an advertisement + if re.match(r"^Advertisements?", article_title, re.IGNORECASE): + in_advertisement = True + skipped_ads += 1 + continue + else: + in_advertisement = False + articles.append({"title": article_title}) + continue + + # p122 and other classes can also be advertisement markers + if re.match(r"^Advertisements?\s*$", text, re.IGNORECASE): + in_advertisement = True + skipped_ads += 1 + continue + + # p9 = Article display title + if cls == "p9": + if in_advertisement: + continue + # Also check the p9 display title itself for advertisement + plain_title = decode_scagoudy(text).strip() + if re.match(r"^Advertisements?\s*$", plain_title, re.IGNORECASE): + in_advertisement = True + skipped_ads += 1 + continue + html = inner_html(tag) + if html: + # Make it a bold article heading if not already + if not html.startswith(""): + html = f"{html}" + paragraphs.append(html) + continue + + # Skip content within advertisement sections + if in_advertisement: + # Check if next p8 will reset it — for now just skip + continue + + # p15 = Verse block — center align + if cls == "p15": + html = inner_html(tag) + if html: + paragraphs.append(f'

{html}
') + continue + + # All other classes — extract as body text + html = inner_html(tag) + if html and html != "
": + # Check if this looks like a verse (entirely italic with
breaks) + stripped = html.strip() + if ( + stripped.startswith("") + and "' not in stripped + ): + plain = re.sub(r"", "", stripped) + lines = [ + l.strip() for l in re.split(r"", plain) if l.strip() + ] + if len(lines) >= 2: + html = f'
{stripped}
' + + paragraphs.append(html) + + # Post-process: remove any ad-like paragraphs that slipped through. + # Two-pass approach: + # 1. Detect ad section start markers and skip until next real article + # 2. Filter individual ad-like paragraphs by content patterns + + # --- Pass 1: Section-level ad detection --- + ad_section_patterns = [ + r"^Advertisements?\s*$", + r"^CIRCULATION AND SUBSCRIPTIONS", + r"^Order Form", + r"^SPECIAL OFFER", + r"^Subscriptions?:", + r"^WORLDWIDE IMPORTS", + r"^THE\s+ORIENT\s+SHOP", + r"^PROTECT YOUR CHECKS", + ] + ad_section_re = re.compile("|".join(ad_section_patterns), re.IGNORECASE) + + cleaned_pass1 = [] + skip_until_next_bold = False + for p in paragraphs: + plain = re.sub(r"<[^>]+>", "", p).strip() + # Detect ad section start + if ad_section_re.search(plain): + skip_until_next_bold = True + continue + if skip_until_next_bold: + # A bold article heading that's NOT an ad resets the skip + if p.startswith("") and len(plain) < 200: + if ad_section_re.search(plain): + continue + else: + skip_until_next_bold = False + cleaned_pass1.append(p) + continue + # If it looks like real content (long paragraph), stop skipping + if len(plain) > 150: + skip_until_next_bold = False + cleaned_pass1.append(p) + continue + # Short paragraphs in ad zone — skip + continue + cleaned_pass1.append(p) + + # --- Pass 2: Individual paragraph ad detection --- + # These patterns catch standalone ad fragments that weren't part of + # a marked advertisement section. + ad_para_keywords = [ + "subscription", + "per copy", + "enclosed is", + "please send me", + "order form", + "postage paid", + "money order", + "check or money order", + "zip code", + "state_____", + "address_____", + "city_____", + "available from iskcon", + "waterpipes", + "djellabas", + "panchos", + "orient shop", + "worldwide imports", + "sullivan st", + "bleecker", + "check protector", + "safeguards your checks", + "satisfaction guaranteed", + "import-export", + "davis & manswell", + "davis & manswell", + "bedford ave", + "brooklyn, n.y.", + "immediate delivery", + "free gift guide", + "write for free", + "free catalog", + "free catalogue", + "protect your checks", + "name of purchaser", + "such dresses at $", + "and gowns yet $", + "price per copy", + "one-year subscription", + "year subscription costs", + "back issues available", + "a subscription to", + "send check or money", + "your name & address", + "your name & address", + "674-9511", + ] + # Also detect short paragraphs that look like addresses or phone numbers + address_re = re.compile( + r"^\d+\s+\w+\s+(st|street|ave|avenue|blvd|road|rd|lane|ln|dr|drive)\b", + re.IGNORECASE, + ) + phone_re = re.compile(r"^\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}\s*$") + zipcode_re = re.compile( + r"(N\.?Y\.?\s+\d{5}|CA\s+\d{5}|Brooklyn|Manhattan)", + re.IGNORECASE, + ) + + cleaned = [] + for p in cleaned_pass1: + plain = re.sub(r"<[^>]+>", "", p).strip() + plain_lower = plain.lower() + + # Skip if matches any ad keyword + if any(kw in plain_lower for kw in ad_para_keywords): + continue + + # Skip short paragraphs that look like addresses + if len(plain) < 80 and address_re.search(plain): + continue + + # Skip standalone phone numbers + if phone_re.match(plain): + continue + + # Skip short paragraphs with zip codes (likely address fragments) + if len(plain) < 60 and zipcode_re.search(plain): + continue + + # Skip price-only paragraphs like "$1 plus 25c" + if re.match( + r"^\$?\d+[\.\d]*\s*(plus|postage|each|per)", plain, re.IGNORECASE + ): + continue + + cleaned.append(p) + paragraphs = cleaned + + # Cap paragraphs to prevent mega-issues from absorbing unmarked + # intermediate issues (e.g., issues 18-29 have no p2 markers and + # would otherwise be absorbed into issue 17). + if len(paragraphs) > MAX_PARAGRAPHS_PER_ISSUE: + print( + f" ⚠ {slug}: Capped from {len(paragraphs)} to {MAX_PARAGRAPHS_PER_ISSUE} paragraphs " + f"(likely absorbed unmarked intermediate issues)" + ) + paragraphs = paragraphs[:MAX_PARAGRAPHS_PER_ISSUE] + + issue_data = { + "book": "btg-1978", + "volume": vol, + "part": issue_num, + "title": display_title, + "slug": slug, + "year": year, + "articles": articles, + "paragraphs": paragraphs, + } + issues_data.append(issue_data) + + print( + f" {slug}: {display_title} — " + f"{len(articles)} articles, {len(paragraphs)} paragraphs" + ) + + print(f"\n Skipped {skipped_ads} advertisement sections") + + # --------------------------------------------------------------------------- + # Phase 3: Build intro.json with hierarchical groups + # --------------------------------------------------------------------------- + + # Group issues by era/volume + # Early issues (1-17) have Vol 1/Vol 2 designations + # Later issues (30+) just have issue numbers + group_early = [] # Issues 1-17 (1966-1968) + group_later = [] # Issues 30+ (1969-1973) + + for issue in issues_data: + if issue["part"] <= 17: + group_early.append(issue) + else: + group_later.append(issue) + + chapters = [] + + if group_early: + sub = [] + for issue in group_early: + sub.append({"slug": issue["slug"], "title": issue["title"]}) + chapters.append( + { + "title": "Issues #1–17 (1966–1968)", + "subChapters": sub, + } + ) + + if group_later: + sub = [] + for issue in group_later: + sub.append({"slug": issue["slug"], "title": issue["title"]}) + chapters.append( + { + "title": "Issues #30–54 (1969–1973)", + "subChapters": sub, + } + ) + + total_issues = len(issues_data) + total_articles = sum(len(i["articles"]) for i in issues_data) + + # Determine year range + years = [i["year"] for i in issues_data if i["year"]] + min_year = min(years) if years else None + max_year = max(years) if years else None + + intro_json = { + "book": "btg-1978", + "title": "Back to Godhead (1966–1973)", + "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda", + "year": min_year, + "description": ( + f"Back to Godhead magazine issues from the Western era ({min_year}–{max_year}). " + f"{total_issues} issues containing {total_articles} articles — lectures, essays, " + f"poems, and philosophical discussions on Kṛṣṇa consciousness." + ), + "totalIssues": total_issues, + "totalArticles": total_articles, + "flag": "Pre 1978", + "chapters": chapters, + } + + # --------------------------------------------------------------------------- + # Phase 4: Write JSON files + # --------------------------------------------------------------------------- + + # Clean output directory + if os.path.exists(issues_dir): + for f in os.listdir(issues_dir): + os.remove(os.path.join(issues_dir, f)) + + # Write intro.json + write_json(intro_json, os.path.join(output_dir, "intro.json")) + print(f"\n Written: intro.json ({total_issues} issues, {total_articles} articles)") + + # Write issue files + order = 100 + for issue in issues_data: + issue_json = { + "book": "btg-1978", + "volume": issue["volume"], + "part": issue["part"], + "title": issue["title"], + "slug": issue["slug"], + "order": order, + "articles": issue["articles"], + "paragraphs": issue["paragraphs"], + } + filepath = os.path.join(issues_dir, f"{issue['slug']}.json") + write_json(issue_json, filepath) + order += 1 + + print(f" Written: {len(issues_data)} issue files") + + # --------------------------------------------------------------------------- + # Phase 5: Quality checks + # --------------------------------------------------------------------------- + print(f"\n{'=' * 60}") + print(f" QUALITY CHECKS") + print(f"{'=' * 60}") + + # Check for ScaGoudy remnants + sca_chars = set(SCAGOUDY_TO_IAST.keys()) + sca_remnants = 0 + for issue in issues_data: + for p in issue["paragraphs"]: + if any(c in p for c in ("å", "ë", "ö")): + sca_remnants += 1 + break + if sca_remnants == 0: + print(f" ✅ No ScaGoudy remnants detected") + else: + print(f" ⚠ {sca_remnants} issues may have ScaGoudy remnants") + + # Check for empty issues + empty_issues = [i for i in issues_data if len(i["paragraphs"]) == 0] + if empty_issues: + print( + f" ⚠ {len(empty_issues)} empty issues: {[i['slug'] for i in empty_issues]}" + ) + else: + print(f" ✅ No empty issues") + + # Check centered verses + verse_count = 0 + for issue in issues_data: + for p in issue["paragraphs"]: + if '
' in p: + verse_count += 1 + print(f" ℹ️ {verse_count} center-aligned verse paragraphs") + + # Check for remaining advertisement content + ad_check = 0 + ad_keywords = ["advertisement", "subscribe", "subscription form"] + for issue in issues_data: + for p in issue["paragraphs"]: + lower_p = p.lower() + if any(kw in lower_p for kw in ad_keywords): + # Only flag if the paragraph is primarily an ad + if "advertisement" in lower_p and len(p) < 200: + ad_check += 1 + if ad_check == 0: + print(f" ✅ No advertisement content detected in output") + else: + print(f" ⚠ {ad_check} paragraphs may contain advertisement content") + + # --------------------------------------------------------------------------- + # Summary + # --------------------------------------------------------------------------- + print(f"\n{'=' * 60}") + print(f" EXTRACTION SUMMARY") + print(f"{'=' * 60}") + + if group_early: + early_arts = sum(len(i["articles"]) for i in group_early) + early_paras = sum(len(i["paragraphs"]) for i in group_early) + print( + f" Issues #1–17 (1966–1968): {len(group_early)} issues, " + f"{early_arts} articles, {early_paras} paragraphs" + ) + for issue in group_early: + print( + f" {issue['slug']}: {issue['title']} " + f"({len(issue['articles'])} articles, {len(issue['paragraphs'])} paras)" + ) + + if group_later: + later_arts = sum(len(i["articles"]) for i in group_later) + later_paras = sum(len(i["paragraphs"]) for i in group_later) + print( + f" Issues #30–54 (1969–1973): {len(group_later)} issues, " + f"{later_arts} articles, {later_paras} paragraphs" + ) + for issue in group_later: + print( + f" {issue['slug']}: {issue['title']} " + f"({len(issue['articles'])} articles, {len(issue['paragraphs'])} paras)" + ) + + print(f"\n Total: {total_issues} issues, {total_articles} articles") + print(f" Skipped: {skipped_ads} advertisement sections") + print(f"{'=' * 60}") + print(f"\nDone! ✨") + + +if __name__ == "__main__": + main() diff --git a/scripts/btg_extract.py b/scripts/btg_extract.py new file mode 100644 index 00000000..2b3a706c --- /dev/null +++ b/scripts/btg_extract.py @@ -0,0 +1,546 @@ +#!/usr/bin/env python3 +"""Extract Back to Godhead magazine from btg-1960.html. + +Each magazine issue becomes its own JSON file. +Issues are grouped by volume in the intro.json metadata. + +Structure: + - p2 = Volume/Part bookmark (e.g., "BTGPY1: Volume 1, Part 1, 1944") + - p3 = Volume/Part display title + - p8 = Article bookmark within an issue (e.g., "BTGPY1a: Message of His Divine Grace") + - p9 = Article display title (bold) + - p12, p35 = Body paragraphs + - p14, p17, p88 = Continuation notes / misc text + - p15 = Verse blocks (italic, with ScaGoudy encoding) + - p33, p40, p47, p54 = Magazine mastheads/headers + +Output: + lib/jsons/magazines/btg/intro.json + lib/jsons/magazines/btg/issues/v1-p1.json + lib/jsons/magazines/btg/issues/v1-p2.json + ... + lib/jsons/magazines/btg/issues/v4-p1.json + +Usage: + python3 scripts/btg_extract.py +""" + +import json +import os +import re +import sys + +from bs4 import BeautifulSoup, Tag + +# --------------------------------------------------------------------------- +# ScaGoudy → IAST Unicode mapping +# --------------------------------------------------------------------------- +SCAGOUDY_TO_IAST = { + "ä": "ā", + "Ä": "Ā", + "å": "ṛ", + "Å": "Ṛ", + "ñ": "ṣ", + "Ñ": "Ṣ", + "ö": "ṭ", + "Ö": "Ṭ", + "ë": "ṇ", + "Ë": "Ṇ", + "ò": "ḍ", + "Ò": "Ḍ", + "ç": "ś", + "Ç": "Ś", + "ì": "ṅ", + "Ì": "Ṅ", + "ï": "ñ", + "Ï": "Ñ", + "ü": "ū", + "Ü": "Ū", + "é": "ī", + "É": "Ī", + "à": "ṁ", + "ù": "ḥ", + "è": "ṝ", + "ÿ": "ḷ", +} + + +def decode_scagoudy(text: str) -> str: + """Replace ScaGoudy-encoded characters with proper IAST Unicode.""" + for src, dst in SCAGOUDY_TO_IAST.items(): + text = text.replace(src, dst) + return text + + +def get_css_class(tag: Tag) -> str: + classes = tag.get("class", []) + return classes[0] if classes else "" + + +def inner_html(tag: Tag) -> str: + """Get inner HTML of a tag, decode ScaGoudy, clean up spans.""" + html = "".join(str(child) for child in tag.children) + html = decode_scagoudy(html) + # text → keep inner text (s1 is a ref style) + html = re.sub( + r']*class="s1"[^>]*>(.*?)', r"\1", html, flags=re.DOTALL + ) + # text → keep inner text (superscript digit) + html = re.sub( + r']*class="s3"[^>]*>(.*?)', r"\1", html, flags=re.DOTALL + ) + # text → keep inner text + html = re.sub( + r']*class="s4"[^>]*>(.*?)', r"\1", html, flags=re.DOTALL + ) + # text → keep inner text + html = re.sub( + r']*class="s5"[^>]*>(.*?)', r"\1", html, flags=re.DOTALL + ) + # text → keep inner text + html = re.sub( + r']*class="s7"[^>]*>(.*?)', r"\1", html, flags=re.DOTALL + ) + # text → keep inner text + html = re.sub( + r']*class="s8"[^>]*>(.*?)', r"\1", html, flags=re.DOTALL + ) + # Remove any remaining spans but keep content + html = re.sub(r"]*>(.*?)", r"\1", html, flags=re.DOTALL) + # Remove Apple-tab-span + html = html.replace("\t", " ") + # Collapse multiple spaces (but not around
) + html = re.sub(r"[ ]+", " ", html) + html = html.strip() + # Remove leading
tags + html = re.sub(r"^([\s]*)+", "", html) + # Remove trailing
tags + html = re.sub(r"([\s]*)+$", "", html) + return html + + +def is_verse_paragraph(tag: Tag) -> bool: + """Check if a paragraph is a verse block (italic with line breaks).""" + cls = get_css_class(tag) + # p15 is commonly used for verse blocks + if cls == "p15": + return True + # Also check for paragraphs that are entirely italic with
breaks + # (some verses use p14 or other classes but are entirely in tags) + return False + + +def format_verse(tag: Tag) -> str: + """Format a verse paragraph as a center-aligned div.""" + html = inner_html(tag) + if not html: + return "" + # Wrap in center-aligned div + return f'
{html}
' + + +def write_json(data: dict, path: str): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + f.write("\n") + + +# --------------------------------------------------------------------------- +# Parse volume/part info from p2 bookmark text +# --------------------------------------------------------------------------- + + +def parse_volume_part(text: str): + """Parse 'BTGPY1: Volume 1, Part 1, 1944' → (volume, part, date_str, slug). + Returns (volume_num, part_num, date_string, slug) or None. + """ + # Remove the BTGPY prefix + text = text.strip() + # Match "Volume X, Part Y, ..." + m = re.search(r"Volume\s+(\d+),?\s*Part\s+(\d+),?\s*(.*)", text, re.IGNORECASE) + if m: + vol = int(m.group(1)) + part = int(m.group(2)) + date_str = m.group(3).strip().rstrip(" ") + slug = f"v{vol}-p{part}" + return vol, part, date_str, slug + return None + + +# --------------------------------------------------------------------------- +# Main extraction +# --------------------------------------------------------------------------- + + +def main(): + base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + html_path = os.path.join(base_dir, "public", "html", "btg-1960.html") + output_dir = os.path.join(base_dir, "lib", "jsons", "magazines", "btg") + issues_dir = os.path.join(output_dir, "issues") + + if not os.path.exists(html_path): + print(f"ERROR: {html_path} not found!") + sys.exit(1) + + print(f"Reading {html_path}...") + with open(html_path, "r", encoding="utf-8") as f: + content = f.read() + + soup = BeautifulSoup(content, "html.parser") + all_tags = soup.find_all("p") + print(f" Parsed {len(all_tags)}

tags total") + + # --------------------------------------------------------------------------- + # Phase 1: Find all issue boundaries + # --------------------------------------------------------------------------- + # Each issue starts with a p2 bookmark tag + issue_boundaries = [] # List of (tag_index, volume, part, date_str, slug) + + for i, tag in enumerate(all_tags): + cls = get_css_class(tag) + if cls == "p2": + text = tag.get_text().strip() + parsed = parse_volume_part(text) + if parsed: + vol, part, date_str, slug = parsed + issue_boundaries.append((i, vol, part, date_str, slug)) + + print(f"\n Found {len(issue_boundaries)} issues") + for ib in issue_boundaries: + print(f" tag[{ib[0]}]: Vol {ib[1]}, Part {ib[2]}, {ib[3]} → {ib[4]}") + + # --------------------------------------------------------------------------- + # Phase 2: Extract articles and paragraphs for each issue + # --------------------------------------------------------------------------- + + # Classes to skip entirely + SKIP_CLASSES = {"p2", "p11", "p24", "p28"} + + # Classes that are header/masthead material (include as paragraphs) + MASTHEAD_CLASSES = {"p3", "p4", "p5", "p6", "p7", "p32", "p33", "p34", "p40", "p47"} + + # Classes that are body text + BODY_CLASSES = { + "p10", + "p12", + "p13", + "p14", + "p15", + "p16", + "p17", + "p35", + "p36", + "p37", + "p38", + "p39", + "p41", + "p42", + "p43", + "p44", + "p45", + "p46", + "p48", + "p49", + "p50", + "p51", + "p52", + "p53", + "p54", + "p55", + "p56", + "p57", + "p58", + "p59", + "p60", + "p61", + "p62", + "p63", + "p64", + "p65", + "p66", + "p67", + "p68", + "p69", + "p70", + "p71", + "p72", + "p73", + "p74", + "p75", + "p76", + "p77", + "p78", + "p79", + "p80", + "p81", + "p82", + "p83", + "p84", + "p85", + "p86", + "p87", + "p88", + "p89", + "p90", + "p91", + "p92", + "p93", + "p94", + "p95", + "p96", + "p97", + "p98", + "p99", + } + + issues_data = [] + + for idx, (start_tag_idx, vol, part, date_str, slug) in enumerate(issue_boundaries): + # Determine end boundary + if idx + 1 < len(issue_boundaries): + end_tag_idx = issue_boundaries[idx + 1][0] + else: + end_tag_idx = len(all_tags) + + issue_tags = all_tags[start_tag_idx:end_tag_idx] + + paragraphs = [] + articles = [] # List of { title, slug } + + for tag in issue_tags: + cls = get_css_class(tag) + text = tag.get_text().strip() + + # Skip bookmark tags and empty lines + if cls in SKIP_CLASSES: + continue + if not text and cls not in ("p15",): + continue + + # p8 = Article bookmark — extract title for metadata + if cls == "p8": + # Extract article title from bookmark text (remove BTGPYxx: prefix) + article_text = text + m = re.match(r"BTGPY\d+[a-z]?:\s*(.*)", article_text) + if m: + article_title = m.group(1).strip() + else: + article_title = article_text + article_title = decode_scagoudy(article_title) + articles.append({"title": article_title}) + continue + + # p9 = Article display title — render as bold heading + if cls == "p9": + html = inner_html(tag) + if html: + # Make it a bold article heading + if not html.startswith(""): + html = f"{html}" + paragraphs.append(html) + continue + + # p15 = Verse block — center align + if cls == "p15": + html = format_verse(tag) + if html: + paragraphs.append(html) + continue + + # p3 = Volume title — skip (it's the display title of the issue header) + if cls == "p3": + continue + + # Masthead classes — include but as regular paragraphs + if cls in MASTHEAD_CLASSES: + html = inner_html(tag) + if html and html != "
": + paragraphs.append(html) + continue + + # Body text classes + html = inner_html(tag) + if html and html != "
": + # Check if this looks like a verse (entirely italic with
breaks) + stripped = html.strip() + if ( + stripped.startswith("") + and "' in stripped + ): + # Check if it's predominantly a verse (italic block with line breaks) + # by seeing if removing the tags leaves mostly text with
+ plain = re.sub(r"", "", stripped) + lines = [ + l.strip() for l in re.split(r"", plain) if l.strip() + ] + if len(lines) >= 2: + html = f'

{stripped}
' + + paragraphs.append(html) + + # Build display title + if date_str: + display_title = f"Volume {vol}, Part {part}, {date_str}" + else: + display_title = f"Volume {vol}, Part {part}" + + issue_data = { + "book": "btg", + "volume": vol, + "part": part, + "title": display_title, + "slug": slug, + "articles": articles, + "paragraphs": paragraphs, + } + issues_data.append(issue_data) + + print( + f" {slug}: {display_title} — " + f"{len(articles)} articles, {len(paragraphs)} paragraphs" + ) + + # --------------------------------------------------------------------------- + # Phase 3: Build intro.json with hierarchical volume groups + # --------------------------------------------------------------------------- + + # Group issues by volume + volume_groups = {} + for issue in issues_data: + vol = issue["volume"] + if vol not in volume_groups: + volume_groups[vol] = [] + volume_groups[vol].append(issue) + + # Build chapters array for intro.json + chapters = [] + for vol_num in sorted(volume_groups.keys()): + vol_issues = volume_groups[vol_num] + sub_chapters = [] + for issue in vol_issues: + sub_chapters.append( + { + "slug": issue["slug"], + "title": issue["title"], + } + ) + chapters.append( + { + "title": f"Volume {vol_num}", + "subChapters": sub_chapters, + } + ) + + total_issues = len(issues_data) + total_articles = sum(len(i["articles"]) for i in issues_data) + + intro_json = { + "book": "btg", + "title": "Back to Godhead", + "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda", + "year": 1944, + "description": f"The original Back to Godhead magazine founded in 1944. {total_issues} issues across {len(volume_groups)} volumes, containing {total_articles} articles on Kṛṣṇa consciousness, Vedic philosophy, and spiritual life.", + "totalIssues": total_issues, + "totalArticles": total_articles, + "chapters": chapters, + } + + # --------------------------------------------------------------------------- + # Phase 4: Write JSON files + # --------------------------------------------------------------------------- + + # Clean output directory + if os.path.exists(issues_dir): + for f in os.listdir(issues_dir): + os.remove(os.path.join(issues_dir, f)) + + # Write intro.json + write_json(intro_json, os.path.join(output_dir, "intro.json")) + print(f"\n Written: intro.json ({total_issues} issues, {total_articles} articles)") + + # Write issue files + order = 100 + for issue in issues_data: + issue_json = { + "book": "btg", + "volume": issue["volume"], + "part": issue["part"], + "title": issue["title"], + "slug": issue["slug"], + "order": order, + "articles": issue["articles"], + "paragraphs": issue["paragraphs"], + } + filepath = os.path.join(issues_dir, f"{issue['slug']}.json") + write_json(issue_json, filepath) + order += 1 + + print(f" Written: {len(issues_data)} issue files") + + # --------------------------------------------------------------------------- + # Phase 5: Quality checks + # --------------------------------------------------------------------------- + print(f"\n{'=' * 60}") + print(f" QUALITY CHECKS") + print(f"{'=' * 60}") + + # Check for ScaGoudy remnants + sca_chars = set(SCAGOUDY_TO_IAST.keys()) + sca_remnants = 0 + for issue in issues_data: + for p in issue["paragraphs"]: + if any(c in p for c in sca_chars if c not in ("é", "ü", "à")): + # Skip common French/German chars that might be legitimate + # Focus on clearly ScaGoudy-only chars + if "å" in p or "ë" in p or "ö" in p and "Ö" not in p: + sca_remnants += 1 + break + if sca_remnants == 0: + print(f" ✅ No ScaGoudy remnants detected") + else: + print(f" ⚠ {sca_remnants} issues may have ScaGoudy remnants") + + # Check for empty issues + empty_issues = [i for i in issues_data if len(i["paragraphs"]) == 0] + if empty_issues: + print( + f" ⚠ {len(empty_issues)} empty issues: {[i['slug'] for i in empty_issues]}" + ) + else: + print(f" ✅ No empty issues") + + # Check centered verses + verse_count = 0 + for issue in issues_data: + for p in issue["paragraphs"]: + if '
' in p: + verse_count += 1 + print(f" ℹ️ {verse_count} center-aligned verse paragraphs") + + # --------------------------------------------------------------------------- + # Summary + # --------------------------------------------------------------------------- + print(f"\n{'=' * 60}") + print(f" EXTRACTION SUMMARY") + print(f"{'=' * 60}") + for vol_num in sorted(volume_groups.keys()): + vol_issues = volume_groups[vol_num] + vol_articles = sum(len(i["articles"]) for i in vol_issues) + vol_paras = sum(len(i["paragraphs"]) for i in vol_issues) + print( + f" Volume {vol_num}: {len(vol_issues)} issues, {vol_articles} articles, {vol_paras} paragraphs" + ) + for issue in vol_issues: + print( + f" {issue['slug']}: {issue['title']} " + f"({len(issue['articles'])} articles, {len(issue['paragraphs'])} paras)" + ) + print(f"\n Total: {total_issues} issues, {total_articles} articles") + print(f"{'=' * 60}") + print(f"\nDone! ✨") + + +if __name__ == "__main__": + main() diff --git a/scripts/letters_extract.py b/scripts/letters_extract.py new file mode 100644 index 00000000..6190366e --- /dev/null +++ b/scripts/letters_extract.py @@ -0,0 +1,689 @@ +#!/usr/bin/env python3 +"""Extract letters from letters.html. + +Each letter starts with a p7 tag containing "Letter to: ". +The header contains recipient, location, and date information. + +Structure: + - p7 = Letter header (recipient, dash, location, date) + - p8 = Reference code (YY-MM-DD) and also postscript/closing lines + - p9 = Recipient address block + - p10 = Salutation ("Dear...") + - p11 = Body paragraphs + - p12 = Centered text (verses, centered quotes) + - p13 = Closing/signature block (color: #408080) + - p14 = Indented/numbered items + - p16-p22, p28-p37 = Various body text styles + +Output: + lib/jsons/letters/letters.json (array of all letters with metadata) + +Usage: + python3 scripts/letters_extract.py +""" + +import json +import os +import re +import sys +from collections import Counter + +from bs4 import BeautifulSoup, Tag + +# --------------------------------------------------------------------------- +# ScaGoudy → IAST Unicode mapping +# --------------------------------------------------------------------------- +SCAGOUDY_TO_IAST = { + "ä": "ā", + "Ä": "Ā", + "å": "ṛ", + "Å": "Ṛ", + "ñ": "ṣ", + "Ñ": "Ṣ", + "ö": "ṭ", + "Ö": "Ṭ", + "ë": "ṇ", + "Ë": "Ṇ", + "ò": "ḍ", + "Ò": "Ḍ", + "ç": "ś", + "Ç": "Ś", + "ì": "ṅ", + "Ì": "Ṅ", + "ï": "ñ", + "Ï": "Ñ", + "ü": "ū", + "Ü": "Ū", + "é": "ī", + "É": "Ī", + "à": "ṁ", + "ù": "ḥ", + "è": "ṝ", + "ÿ": "ḷ", +} + + +def decode_scagoudy(text: str) -> str: + """Replace ScaGoudy-encoded characters with proper IAST Unicode.""" + for src, dst in SCAGOUDY_TO_IAST.items(): + text = text.replace(src, dst) + return text + + +def get_css_class(tag: Tag) -> str: + classes = tag.get("class", []) + return classes[0] if classes else "" + + +def inner_html(tag: Tag) -> str: + """Get inner HTML of a tag, decode ScaGoudy, clean up spans.""" + html = "".join(str(child) for child in tag.children) + html = decode_scagoudy(html) + + # text → keep inner text (reference style) + html = re.sub( + r']*class="s\d+"[^>]*>(.*?)', r"\1", html, flags=re.DOTALL + ) + # Remove any remaining spans but keep content + html = re.sub(r"]*>(.*?)", r"\1", html, flags=re.DOTALL) + # Remove tab characters + html = html.replace("\t", " ") + # Collapse multiple spaces + html = re.sub(r"[ ]+", " ", html) + html = html.strip() + # Remove leading
tags + html = re.sub(r"^([\s]*)+", "", html) + # Remove trailing
tags + html = re.sub(r"([\s]*)+$", "", html) + return html + + +def write_json(data, path: str): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + f.write("\n") + + +# --------------------------------------------------------------------------- +# Month name → number mapping +# --------------------------------------------------------------------------- +MONTH_MAP = { + "january": 1, + "february": 2, + "march": 3, + "april": 4, + "may": 5, + "june": 6, + "july": 7, + "august": 8, + "september": 9, + "october": 10, + "november": 11, + "december": 12, + "jan": 1, + "feb": 2, + "mar": 3, + "apr": 4, + "jun": 6, + "jul": 7, + "aug": 8, + "sep": 9, + "sept": 9, + "oct": 10, + "nov": 11, + "dec": 12, +} + + +def parse_date(date_str: str): + """Parse a date string like '12 July, 1947' or 'May 1949'. + + Returns (year, month, day) where month and day can be None. + """ + if not date_str: + return None, None, None + + date_str = date_str.strip().rstrip(".") + + # Try "DD Month, YYYY" or "DD Month YYYY" + m = re.match(r"(\d{1,2})\s+([A-Za-z]+),?\s*(\d{4})", date_str) + if m: + day = int(m.group(1)) + month_name = m.group(2).lower() + year = int(m.group(3)) + month = MONTH_MAP.get(month_name) + if year >= 1900 and year <= 2000: + return year, month, day + + # Try "Month DD, YYYY" or "Month DD YYYY" + m = re.match(r"([A-Za-z]+)\s+(\d{1,2}),?\s*(\d{4})", date_str) + if m: + month_name = m.group(1).lower() + day = int(m.group(2)) + year = int(m.group(3)) + month = MONTH_MAP.get(month_name) + if year >= 1900 and year <= 2000: + return year, month, day + + # Try "Month, YYYY" or "Month YYYY" + m = re.match(r"([A-Za-z]+),?\s*(\d{4})", date_str) + if m: + month_name = m.group(1).lower() + year = int(m.group(2)) + month = MONTH_MAP.get(month_name) + if month and year >= 1900 and year <= 2000: + return year, month, None + + # Try just "YYYY" + m = re.match(r"(\d{4})$", date_str.strip()) + if m: + year = int(m.group(1)) + if year >= 1900 and year <= 2000: + return year, None, None + + return None, None, None + + +def parse_header(tag: Tag): + """Parse a p7 letter header tag. + + Expected format: + Letter to: + — + + + + Or sometimes: + Letter to: + + + Returns (recipient, location, date_str). + """ + text = tag.get_text().strip() + lines = [l.strip() for l in text.split("\n") if l.strip()] + + recipient = None + location = None + date_str = None + + if not lines: + return recipient, location, date_str + + # First line: "Letter to: " + first = lines[0] + m = re.match(r"Letter to:\s*(.*)", first, re.IGNORECASE) + if m: + recipient = m.group(1).strip().rstrip(",.") + recipient = decode_scagoudy(recipient) + + # Remove "—" lines + content_lines = [l for l in lines[1:] if l != "—" and l != "-"] + + if len(content_lines) >= 2: + # Last line is usually the date + potential_date = content_lines[-1] + year, month, day = parse_date(potential_date) + if year: + date_str = potential_date + # Everything before the date is location + loc_lines = content_lines[:-1] + if loc_lines: + location = loc_lines[0].strip().rstrip(",.") + location = decode_scagoudy(location) + else: + # Maybe the second-to-last is location and last is something else + # Try to find a date anywhere + for cl in reversed(content_lines): + yr, mo, dy = parse_date(cl) + if yr: + date_str = cl + idx = content_lines.index(cl) + if idx > 0: + location = content_lines[0].strip().rstrip(",.") + location = decode_scagoudy(location) + break + elif len(content_lines) == 1: + # Could be just a date, or just a location + potential_date = content_lines[0] + year, month, day = parse_date(potential_date) + if year: + date_str = potential_date + else: + location = potential_date.strip().rstrip(",.") + location = decode_scagoudy(location) + + return recipient, location, date_str + + +def make_slug(recipient: str, date_ref: str, year, month, day, index: int) -> str: + """Create a unique slug for a letter.""" + parts = [] + + # Date part + if date_ref and re.match(r"\d{2}-\d{2}", date_ref): + parts.append(date_ref) + elif year: + yr = str(year)[2:] # Last 2 digits + if month: + if day: + parts.append(f"{yr}-{month:02d}-{day:02d}") + else: + parts.append(f"{yr}-{month:02d}-00") + else: + parts.append(f"{yr}-00-00") + else: + parts.append(f"unknown-{index}") + + # Recipient part + if recipient: + # Slugify recipient name + name = recipient.lower() + name = re.sub(r"[^a-z0-9]+", "-", name) + name = name.strip("-")[:40] + parts.append(name) + + slug = "-".join(parts) + return slug + + +# --------------------------------------------------------------------------- +# Main extraction +# --------------------------------------------------------------------------- + + +def main(): + base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + html_path = os.path.join(base_dir, "public", "html", "letters.html") + output_dir = os.path.join(base_dir, "lib", "jsons", "letters") + + if not os.path.exists(html_path): + print(f"ERROR: {html_path} not found!") + sys.exit(1) + + print(f"Reading {html_path}...") + with open(html_path, "r", encoding="utf-8") as f: + content = f.read() + + soup = BeautifulSoup(content, "html.parser") + all_tags = soup.find_all("p") + print(f" Parsed {len(all_tags)}

tags total") + + # --------------------------------------------------------------------------- + # Phase 1: Find all letter boundaries + # --------------------------------------------------------------------------- + letter_starts = [] + for i, tag in enumerate(all_tags): + cls = get_css_class(tag) + text = tag.get_text().strip() + if cls == "p7" and text.startswith("Letter to:"): + letter_starts.append(i) + + print(f"\n Found {len(letter_starts)} letters") + + # --------------------------------------------------------------------------- + # Phase 2: Extract each letter + # --------------------------------------------------------------------------- + + # Classes that are just empty spacers + SKIP_CLASSES = {"p2", "p15", "p23", "p24", "p26"} + + # p12 = centered (verse-like) + CENTERED_CLASSES = {"p12", "p18", "p19", "p20", "p21", "p27"} + + # p13, p34, p38 = signature/closing blocks (colored) + SIGNATURE_CLASSES = {"p13", "p34", "p38"} + + letters = [] + slug_counts = Counter() + + for li, start_idx in enumerate(letter_starts): + # Determine end boundary + end_idx = ( + letter_starts[li + 1] if li + 1 < len(letter_starts) else len(all_tags) + ) + + # Parse header + header_tag = all_tags[start_idx] + recipient, location, date_str = parse_header(header_tag) + + # Parse date into components + year, month, day = parse_date(date_str) if date_str else (None, None, None) + + # Find date reference code (first p8 after header that matches YY-MM-DD) + date_ref = None + salutation = None + paragraphs = [] + first_p8_seen = False + + for i in range(start_idx + 1, end_idx): + tag = all_tags[i] + cls = get_css_class(tag) + text = tag.get_text().strip() + + if cls in SKIP_CLASSES: + continue + if not text: + continue + + # p8 = reference code or body text + if cls == "p8": + if not first_p8_seen: + # First p8 after header — check if it's a date reference + if re.match(r"\d{2}-\d{2}", text): + date_ref = text.strip() + first_p8_seen = True + + # Try to extract year/month/day from ref if not already parsed + if not year and date_ref: + ref_parts = date_ref.split("-") + if len(ref_parts) >= 3: + try: + yr = int(ref_parts[0]) + mo = int(ref_parts[1]) + dy = int(ref_parts[2]) + year = 1900 + yr if yr < 100 else yr + month = mo if 1 <= mo <= 12 else None + day = dy if 1 <= dy <= 31 else None + except ValueError: + pass + continue + else: + first_p8_seen = True + # Not a date ref — treat as body text + html = inner_html(tag) + if html: + paragraphs.append(html) + continue + else: + # Subsequent p8 tags — body/closing text + html = inner_html(tag) + if html: + paragraphs.append(html) + continue + + # p9 = recipient address block + if cls == "p9": + html = inner_html(tag) + if html: + paragraphs.append(html) + continue + + # p10 = salutation + if cls == "p10": + salutation = decode_scagoudy(text) + html = inner_html(tag) + if html: + paragraphs.append(html) + continue + + # Centered classes (verse blocks) + if cls in CENTERED_CLASSES: + html = inner_html(tag) + if html: + # Check if it has line breaks (verse-like) + if "{html}

' + paragraphs.append(html) + continue + + # Signature/closing blocks + if cls in SIGNATURE_CLASSES: + html = inner_html(tag) + if html: + paragraphs.append(html) + continue + + # All other body text classes (p11, p14, p16, p22, p28, p29, etc.) + html = inner_html(tag) + if html and html != "
": + # Check for verse-like content + stripped = html.strip() + if ( + stripped.startswith("") + and "' not in stripped + ): + plain = re.sub(r"", "", stripped) + lines = [ + l.strip() for l in re.split(r"", plain) if l.strip() + ] + if len(lines) >= 2: + html = f'
{stripped}
' + + paragraphs.append(html) + + # Clean recipient name — strip trailing punctuation + if recipient: + recipient = recipient.rstrip(":,.; ").strip() + # Also clean " :" (space before colon) + recipient = re.sub(r"\s+:$", "", recipient).strip() + + # Split multi-recipient strings into individual names + # Patterns: "Mukunda, Janaki" or "Mukunda and Janaki" + recipient_display = recipient or "Unknown" + if recipient: + # Split by comma, semicolon, " and ", " & " + parts = re.split(r"\s*[,;]\s*|\s+and\s+|\s+&\s+", recipient) + recipients_list = [] + for p in parts: + name = p.strip().rstrip(":,.; ").strip() + # Clean " :" pattern + name = re.sub(r"\s+:$", "", name).strip() + if name: + recipients_list.append(name) + if not recipients_list: + recipients_list = ["Unknown"] + else: + recipients_list = ["Unknown"] + + # Build slug + slug = make_slug(recipient_display, date_ref, year, month, day, li) + + # Ensure unique slug + if slug in slug_counts: + slug_counts[slug] += 1 + slug = f"{slug}-{slug_counts[slug]}" + else: + slug_counts[slug] = 1 + + # Normalize location + if location: + # Clean up common variants + location = location.rstrip(",.") + # Normalize "Vrndavana" variants + if location.lower() in ("vrndavana", "vrndävana"): + location = "Vrindaban" + + letter = { + "slug": slug, + "recipients": recipients_list, + "recipientDisplay": recipient_display, + "date": date_str, + "dateRef": date_ref, + "year": year, + "month": month, + "day": day, + "location": location, + "salutation": salutation, + "paragraphs": paragraphs, + } + letters.append(letter) + + print(f"\n Extracted {len(letters)} letters") + + # --------------------------------------------------------------------------- + # Phase 3: Statistics + # --------------------------------------------------------------------------- + + # Year distribution + year_counts = Counter() + for letter in letters: + if letter["year"]: + year_counts[letter["year"]] += 1 + print(f"\n Year distribution:") + for yr in sorted(year_counts.keys()): + if 1940 <= yr <= 1980: + print(f" {yr}: {year_counts[yr]} letters") + + # Location distribution (top 20) + loc_counts = Counter() + for letter in letters: + if letter["location"]: + loc_counts[letter["location"]] += 1 + print(f"\n Top 20 locations:") + for loc, count in loc_counts.most_common(20): + print(f" {loc}: {count}") + + # Recipient distribution (by individual name, not group) + recip_counts = Counter() + for letter in letters: + for r in letter["recipients"]: + recip_counts[r] += 1 + print(f"\n Top 20 recipients (individual):") + for recip, count in recip_counts.most_common(20): + print(f" {recip}: {count}") + + # Letters with missing data + no_year = sum(1 for l in letters if not l["year"]) + no_location = sum(1 for l in letters if not l["location"]) + no_date = sum(1 for l in letters if not l["date"]) + no_recipient = sum(1 for l in letters if l["recipients"] == ["Unknown"]) + print(f"\n Missing data:") + print(f" No year: {no_year}") + print(f" No location: {no_location}") + print(f" No date: {no_date}") + print(f" No recipient (Unknown): {no_recipient}") + + # Unique counts + all_individual_recipients = set() + for l in letters: + for r in l["recipients"]: + all_individual_recipients.add(r) + unique_recipients = len(all_individual_recipients) + unique_locations = len(set(l["location"] for l in letters if l["location"])) + unique_years = len(set(l["year"] for l in letters if l["year"])) + print(f"\n Unique counts:") + print(f" Recipients: {unique_recipients}") + print(f" Locations: {unique_locations}") + print(f" Years: {unique_years}") + + # --------------------------------------------------------------------------- + # Phase 4: Write output + # --------------------------------------------------------------------------- + + # Write individual letter files grouped by year for manageable sizes + years_with_letters = sorted( + set(l["year"] for l in letters if l["year"] and 1940 <= l["year"] <= 1980) + ) + + # Write a summary/index file + index_data = { + "totalLetters": len(letters), + "uniqueRecipients": unique_recipients, + "uniqueLocations": unique_locations, + "years": [], + "topRecipients": [ + {"name": r, "count": c} for r, c in recip_counts.most_common(50) + ], + "topLocations": [ + {"name": l, "count": c} for l, c in loc_counts.most_common(50) + ], + } + + for yr in years_with_letters: + yr_letters = [l for l in letters if l["year"] == yr] + index_data["years"].append( + { + "year": yr, + "count": len(yr_letters), + } + ) + + # Also count letters without a valid year + undated = [ + l for l in letters if not l["year"] or l["year"] < 1940 or l["year"] > 1980 + ] + if undated: + index_data["years"].append( + { + "year": 0, + "count": len(undated), + } + ) + + write_json(index_data, os.path.join(output_dir, "index.json")) + print(f"\n Written: index.json") + + # Write letters grouped by year (one JSON file per year) + for yr in years_with_letters: + yr_letters = [l for l in letters if l["year"] == yr] + yr_letters.sort(key=lambda l: (l["month"] or 0, l["day"] or 0)) + write_json(yr_letters, os.path.join(output_dir, f"{yr}.json")) + + # Write undated letters + if undated: + write_json(undated, os.path.join(output_dir, "undated.json")) + + print(f" Written: {len(years_with_letters)} year files + undated.json") + + # --------------------------------------------------------------------------- + # Phase 5: Quality checks + # --------------------------------------------------------------------------- + print(f"\n{'=' * 60}") + print(f" QUALITY CHECKS") + print(f"{'=' * 60}") + + # Check for ScaGoudy remnants + sca_chars = {"å", "ë", "ö"} + sca_remnants = 0 + for letter in letters[:100]: # Check first 100 + for p in letter["paragraphs"]: + if any(c in p for c in sca_chars): + sca_remnants += 1 + break + if sca_remnants == 0: + print(f" ✅ No ScaGoudy remnants detected (checked first 100)") + else: + print(f" ⚠ {sca_remnants} letters may have ScaGoudy remnants") + + # Check empty letters + empty = [l for l in letters if len(l["paragraphs"]) == 0] + if empty: + print(f" ⚠ {len(empty)} empty letters (no paragraphs)") + else: + print(f" ✅ No empty letters") + + # Check duplicate slugs + slug_list = [l["slug"] for l in letters] + dupes = len(slug_list) - len(set(slug_list)) + if dupes: + print(f" ⚠ {dupes} duplicate slugs") + else: + print(f" ✅ No duplicate slugs") + + # Verse count + verse_count = 0 + for letter in letters: + for p in letter["paragraphs"]: + if '
' in p: + verse_count += 1 + print(f" ℹ️ {verse_count} center-aligned verse paragraphs") + + # --------------------------------------------------------------------------- + # Summary + # --------------------------------------------------------------------------- + print(f"\n{'=' * 60}") + print(f" EXTRACTION SUMMARY") + print(f"{'=' * 60}") + print(f" Total letters: {len(letters)}") + print(f" Unique recipients: {unique_recipients}") + print(f" Unique locations: {unique_locations}") + print(f" Year range: {min(years_with_letters)}–{max(years_with_letters)}") + print(f" Total paragraphs: {sum(len(l['paragraphs']) for l in letters)}") + print(f"{'=' * 60}") + print(f"\nDone! ✨") + + +if __name__ == "__main__": + main() diff --git a/scripts/seed-letters.ts b/scripts/seed-letters.ts new file mode 100644 index 00000000..90d1b604 --- /dev/null +++ b/scripts/seed-letters.ts @@ -0,0 +1,158 @@ +/** + * seed-letters.ts — Push letters into MongoDB. + * + * Collection written: + * - letters → 6,866 individual letter documents + * + * JSON sources: + * - lib/jsons/letters/1947.json through 1977.json + * - lib/jsons/letters/undated.json + * + * Usage: + * npx tsx scripts/seed-letters.ts + */ + +import dotenv from "dotenv"; +import fs from "fs"; +import path from "path"; +import { MongoClient } from "mongodb"; + +dotenv.config({ path: ".env.local" }); +dotenv.config({ path: ".env" }); + +const DATABASE_URL = process.env.DATABASE_URL; +if (!DATABASE_URL) { + console.error("❌ DATABASE_URL is not set"); + process.exit(1); +} + +const BASE_DIR = path.resolve(__dirname, ".."); +const LETTERS_DIR = path.join(BASE_DIR, "lib", "jsons", "letters"); + +function readJson(filePath: string): T { + return JSON.parse(fs.readFileSync(filePath, "utf-8")); +} + +function sanitizeString(s: string): string { + s = s.normalize("NFC"); + s = s.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, ""); + s = s.replace(/[\x7f-\x9f]/g, ""); + s = s.replace(/[\uD800-\uDFFF]/g, ""); + s = s.replace(/[\u200B-\u200F\u2028-\u202F\uFEFF]/g, ""); + s = s.replace(/[\uFDD0-\uFDEF\uFFFE\uFFFF]/g, ""); + s = Buffer.from(s, "utf-8").toString("utf-8"); + s = s.split("\0").join(""); + return s; +} + +function sanitizeDoc(obj: unknown): unknown { + if (typeof obj === "string") return sanitizeString(obj); + if (Array.isArray(obj)) return obj.map(sanitizeDoc); + if (obj !== null && typeof obj === "object") { + const result: Record = {}; + for (const [key, value] of Object.entries(obj as Record)) + result[key] = sanitizeDoc(value); + return result; + } + return obj; +} + +type LetterJson = { + slug: string; + recipients: string[]; + recipientDisplay: string; + date: string | null; + dateRef: string | null; + year: number | null; + month: number | null; + day: number | null; + location: string | null; + salutation: string | null; + paragraphs: string[]; +}; + +async function main() { + console.log("🌱 Seeding Letters...\n"); + + const client = new MongoClient(DATABASE_URL!); + await client.connect(); + console.log("📡 Connected to MongoDB\n"); + + const db = client.db(); + const col = db.collection("letters"); + const start = Date.now(); + + // Delete all existing letters + const deleted = await col.deleteMany({}); + console.log(` 🗑️ Deleted ${deleted.deletedCount} existing letters`); + + // Ensure indexes + await col.createIndex({ slug: 1 }, { unique: true }); + await col.createIndex({ year: 1 }); + await col.createIndex({ recipient: 1 }); + await col.createIndex({ location: 1 }); + await col.createIndex({ year: 1, month: 1 }); + + // Read all year files + const files = fs + .readdirSync(LETTERS_DIR) + .filter((f) => f.endsWith(".json") && f !== "index.json") + .sort(); + + console.log(`\n 📂 Found ${files.length} letter files`); + + let inserted = 0; + let errors = 0; + + for (const file of files) { + const letters = readJson(path.join(LETTERS_DIR, file)); + console.log(`\n 📄 ${file}: ${letters.length} letters`); + + for (const letter of letters) { + const doc = sanitizeDoc(letter) as Record; + + try { + await col.insertOne(doc); + inserted++; + + if (inserted % 500 === 0) { + console.log(` ... ${inserted} letters inserted`); + } + } catch (e: unknown) { + const msg = e instanceof Error ? e.message : String(e); + if (msg.includes("duplicate key")) { + // Skip duplicates silently + continue; + } + console.error( + ` ❌ Error inserting ${letter.slug}: ${msg.substring(0, 100)}`, + ); + errors++; + if (errors > 10) { + console.error(" Too many errors, stopping."); + break; + } + } + } + + if (errors > 10) break; + } + + // Verify + const totalDocs = await col.countDocuments(); + + const elapsed = ((Date.now() - start) / 1000).toFixed(1); + console.log("\n" + "─".repeat(50)); + console.log(`📊 Summary:`); + console.log(` Inserted: ${inserted} letters`); + console.log(` Errors: ${errors}`); + console.log(` Total in collection: ${totalDocs}`); + console.log(`\n🎉 Done in ${elapsed}s`); + + await client.close(); +} + +main().catch((err) => { + console.error("❌ Seed failed:", err); + process.exit(1); +}); diff --git a/scripts/seed-magazines.ts b/scripts/seed-magazines.ts new file mode 100644 index 00000000..87a432b2 --- /dev/null +++ b/scripts/seed-magazines.ts @@ -0,0 +1,189 @@ +/** + * seed-magazines.ts — Push Magazines into MongoDB. + * + * Collections written: + * - magazine_meta → one doc per magazine + * - magazine_issues → N issue docs per magazine + * + * Usage: + * npx tsx scripts/seed-magazines.ts + */ + +import dotenv from "dotenv"; +import fs from "fs"; +import path from "path"; +import { MongoClient } from "mongodb"; + +dotenv.config({ path: ".env.local" }); +dotenv.config({ path: ".env" }); + +const DATABASE_URL = process.env.DATABASE_URL; +if (!DATABASE_URL) { + console.error("❌ DATABASE_URL is not set"); + process.exit(1); +} + +const BASE_DIR = path.resolve(__dirname, ".."); +const MAGAZINES_DIR = path.join(BASE_DIR, "lib", "jsons", "magazines"); + +function readJson(filePath: string): T { + return JSON.parse(fs.readFileSync(filePath, "utf-8")); +} + +function sanitizeString(s: string): string { + // Normalize to NFC + s = s.normalize("NFC"); + // Remove null bytes and C0 control chars (keep \t \n \r) + s = s.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, ""); + // Remove C1 control chars + s = s.replace(/[\x7f-\x9f]/g, ""); + // Remove lone surrogates (U+D800–U+DFFF) that are invalid in BSON + s = s.replace(/[\uD800-\uDFFF]/g, ""); + // Remove zero-width characters that can confuse BSON length calculations + s = s.replace(/[\u200B-\u200F\u2028-\u202F\uFEFF]/g, ""); + // Remove non-characters (U+FDD0–U+FDEF, U+FFFE, U+FFFF) + s = s.replace(/[\uFDD0-\uFDEF\uFFFE\uFFFF]/g, ""); + // Remove any remaining non-BMP characters that could cause issues + // (keep common supplementary planes like emoji, but remove PUA etc.) + s = s.replace(/[\uDB80-\uDBFF][\uDC00-\uDFFF]/g, ""); // Supplementary PUA + // Round-trip through Buffer to guarantee valid UTF-8 byte sequence + s = Buffer.from(s, "utf-8").toString("utf-8"); + // Final safety: ensure the string is null-terminated compatible + // by removing any embedded nulls that survived + s = s.split("\0").join(""); + return s; +} + +function sanitizeDoc(obj: unknown): unknown { + if (typeof obj === "string") { + return sanitizeString(obj); + } + if (Array.isArray(obj)) return obj.map(sanitizeDoc); + if (obj !== null && typeof obj === "object") { + const result: Record = {}; + for (const [key, value] of Object.entries(obj as Record)) + result[key] = sanitizeDoc(value); + return result; + } + return obj; +} + +async function seedOneMagazine( + db: ReturnType, + magDir: string, +): Promise<{ bookCode: string; issues: number }> { + const introPath = path.join(magDir, "intro.json"); + if (!fs.existsSync(introPath)) throw new Error(`No intro.json in ${magDir}`); + const intro = sanitizeDoc( + readJson>(introPath), + ) as Record; + const bookCode = intro.book as string; + + const metaCol = db.collection("magazine_meta"); + await metaCol.replaceOne( + { book: bookCode }, + { book: bookCode, data: intro }, + { upsert: true }, + ); + console.log(` 📖 Upserted magazine_meta for "${intro.title}" (${bookCode})`); + + const issuesCol = db.collection("magazine_issues"); + // Delete old issues for this magazine + await issuesCol.deleteMany({ book: bookCode }); + + const issuesDir = path.join(magDir, "issues"); + let inserted = 0; + if (fs.existsSync(issuesDir)) { + const files = fs + .readdirSync(issuesDir) + .filter((f) => f.endsWith(".json")) + .sort(); + for (const file of files) { + const issue = sanitizeDoc( + readJson>(path.join(issuesDir, file)), + ) as Record; + + // Deep-sanitize each paragraph string + const rawParas = (issue.paragraphs as string[]) || []; + const cleanParas = rawParas.map((p: string) => sanitizeString(String(p))); + + // Store paragraphs as a single JSON string to avoid MongoDB BSON + // wire protocol issues with large string arrays. + // The data layer will JSON.parse() this back into an array on read. + const paragraphsJson = JSON.stringify(cleanParas); + + const doc = { + book: bookCode, + volume: issue.volume, + part: issue.part, + title: issue.title, + slug: issue.slug, + order: issue.order, + paragraphsJson, + paragraphs: [] as string[], // Keep empty array for schema compat + }; + + await issuesCol.replaceOne({ book: bookCode, slug: issue.slug }, doc, { + upsert: true, + }); + inserted++; + } + console.log(` 📄 Issues: ${inserted}`); + } + + const total = await issuesCol.countDocuments({ book: bookCode }); + console.log(` ✅ ${bookCode}: ${total} issue entries`); + return { bookCode, issues: inserted }; +} + +async function main() { + console.log("🌱 Seeding Magazines...\n"); + const client = new MongoClient(DATABASE_URL!); + await client.connect(); + console.log("📡 Connected to MongoDB\n"); + const db = client.db(); + const start = Date.now(); + + if (!fs.existsSync(MAGAZINES_DIR)) { + console.error(`❌ ${MAGAZINES_DIR} not found`); + process.exit(1); + } + + const subdirs = fs + .readdirSync(MAGAZINES_DIR) + .filter((e) => { + const p = path.join(MAGAZINES_DIR, e); + return ( + fs.statSync(p).isDirectory() && + fs.existsSync(path.join(p, "intro.json")) + ); + }) + .sort(); + + if (subdirs.length === 0) { + console.error("❌ No magazine directories found"); + process.exit(1); + } + console.log(`📂 Found ${subdirs.length} magazine(s)\n`); + + let totalIssues = 0; + const bookCodes: string[] = []; + for (const subdir of subdirs) { + console.log(`\n📝 Processing: ${subdir}`); + const result = await seedOneMagazine(db, path.join(MAGAZINES_DIR, subdir)); + bookCodes.push(result.bookCode); + totalIssues += result.issues; + } + + const elapsed = ((Date.now() - start) / 1000).toFixed(1); + console.log("\n" + "─".repeat(50)); + console.log(`📊 Summary: ${subdirs.length} magazines, ${totalIssues} issues`); + console.log(` Book codes: ${bookCodes.join(", ")}`); + console.log(`\n🎉 Done in ${elapsed}s`); + await client.close(); +} + +main().catch((err) => { + console.error("❌ Seed failed:", err); + process.exit(1); +});