+ {content.paragraphs.map((paragraph, i) => (
+
+ ))}
+
+ {meta.author}
+{meta.description}
+ )} + + {/* Chapters */} ++ {ch.title} +
+{meta.author}
+{meta.description}
+ )} + + {/* Chapters */} ++ {ch.title} +
+{meta.author}
+{meta.description}
+ )} + + {/* Front Matter */} + {introSegments && introSegments.length > 0 && ( ++ {seg.title} +
++ {ch.title} +
+{meta.author}
+{meta.description}
+ )} + + {/* Chapters (Prayers) */} ++ {ch.title} +
+{meta.author}
+{meta.description}
+ )} + ++ {ch.title} +
++ {filtered.length === 0 + ? "No essays found" + : `${filtered.length} essay${filtered.length === 1 ? "" : "s"} found`} +
+ )} + + {/* Grid */} +- {essay.title} -
-- {essay.author} - {essay.year ? ` · ${essay.year}` : ""} -
-{meta.author}
+{meta.description}
+ )} + + {meta.chapters.map((ch: JournalChapterEntry, idx: number) => { + // Top-level chapter with a slug (Preface, Introduction, etc.) + if ("slug" in ch && !("subChapters" in ch)) { + return ( +{ch.title}
+ ++ {day.title} +
+ ++ {sc.title} +
+ ++ Personal diaries and journals of Śrīla Prabhupāda +
+{meta.author}
+{meta.description}
+ )} + + {/* Chapters */} + {meta.chapters.length > 0 && ( ++ {ch.title} +
++ {filtered.length === 0 + ? "No documents found" + : `${filtered.length} document${filtered.length === 1 ? "" : "s"} found`} +
+ )} + + {/* Grid */} ++ Miscellaneous documents, analyses, and writings by Śrīla Prabhupāda +
++ Śrīla Prabhupāda's personal diaries — including the + historic Jaladuta Diary from his 1965 voyage to America and + The Beginning, his 1966 New York journal documenting the + founding of the Hare Kṛṣṇa movement. +
++ Miscellaneous documents, analyses, and writings by Śrīla + Prabhupāda that don't fit neatly into the books or essays + collections — including structured analyses with verse + references. +
+containing "Bhagaväner kathä" + End:
containing "Indian Bhagavatams" (stop before) + +Chapter boundaries:
tags containing "Bk:Chapter"
+
+IMPORTANT: Unlike MOG/CBHC, this book's body text uses ScaGoudy font encoding.
+Characters like ä, å, ñ, ö, ë, ò, ç, ì, ï, ü, é, à, ù must be decoded to
+proper IAST Unicode (ā, ṛ, ṣ, ṭ, ṇ, ḍ, ś, ṅ, ñ, ū, ī, ṁ, ḥ).
+
+CSS classes:
+ p56 — Book title header (skip)
+ p17 — Chapter bookmark headers ("Bk:Chapter N") — boundary detection, skip
+ p5 — Chapter title headings (skip — we derive titles ourselves)
+ p6 — Body paragraphs (MAIN content) — HAS ScaGoudy encoding
+ p12 — Body paragraph variant (longer text) — HAS ScaGoudy encoding
+ p38 — Body paragraph variant — HAS ScaGoudy encoding
+ p9 — Sanskrit verse lines (italic, with ) — centered italic
+ p10 — Reference markers like [RTW 1.1] — include as paragraphs
+ p61 — Reference marker variant [RTW 1.2] — include
+ p57 — Footnote header ("Footnote") — include
+ p58 — Footnote text — include
+ p19 — Sanskrit verse block (multi-line) — centered italic
+ p59 — Sanskrit verse block variant — centered italic
+ p23 — Sanskrit verse line variant — centered italic
+ p36 — Sanskrit verse line variant — centered italic
+ p60 — Sanskrit verse line variant — centered italic
+
+Output:
+ lib/jsons/books/bk/intro.json — Book metadata
+ lib/jsons/books/bk/chapters/01.json — Chapter 1: This World of Misery
+ lib/jsons/books/bk/chapters/02.json — Chapter 2: The Cause of Misery
+
+Usage:
+ python3 scripts/bk_extract.py
+"""
+
+import json
+import os
+import re
+import sys
+
+from bs4 import BeautifulSoup, Tag
+
+# ---------------------------------------------------------------------------
+# ScaGoudy → IAST Unicode mapping
+# ---------------------------------------------------------------------------
+
+SCAGOUDY_TO_IAST = {
+ "ä": "ā",
+ "Ä": "Ā",
+ "å": "ṛ",
+ "Å": "Ṛ",
+ "ñ": "ṣ",
+ "Ñ": "Ṣ",
+ "ö": "ṭ",
+ "Ö": "Ṭ",
+ "ë": "ṇ",
+ "Ë": "Ṇ",
+ "ò": "ḍ",
+ "Ò": "Ḍ",
+ "ç": "ś",
+ "Ç": "Ś",
+ "ì": "ṅ",
+ "Ì": "Ṅ",
+ "ï": "ñ",
+ "Ï": "Ñ",
+ "ü": "ū",
+ "Ü": "Ū",
+ "é": "ī",
+ "É": "Ī",
+ "à": "ṁ",
+ "À": "Ṁ",
+ "ù": "ḥ",
+ "Ù": "Ḥ",
+ "è": "ṝ",
+ "û": "ṁl",
+ "ÿ": "ḷ",
+}
+
+# Build a translation table for fast character-by-character replacement
+_SCAGOUDY_TRANS = str.maketrans(
+ {k: v for k, v in SCAGOUDY_TO_IAST.items() if len(k) == 1}
+)
+
+
+def scagoudy_decode(text: str) -> str:
+ """Decode ScaGoudy-encoded text to proper IAST Unicode."""
+ return text.translate(_SCAGOUDY_TRANS)
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def decode_html_content(tag: Tag, apply_scagoudy: bool = True) -> str:
+ """Extract inner HTML from a tag, preserve formatting tags, strip spans,
+ and optionally apply ScaGoudy decoding.
+
+ Steps:
+ 1. Get inner HTML
+ 2. Strip wrappers (keep content)
+ 3. Apply ScaGoudy decode to the entire result (if requested)
+ 4. Normalize whitespace
+ """
+ inner = "".join(str(child) for child in tag.children)
+ # Remove span tags with CSS classes (RTF conversion artifacts) — keep content
+ inner = re.sub(
+ r' containing "Bhagaväner kathä"
+ End: containing "Indian Bhagavatams" (stop before)
+ """
+ start_idx = None
+ end_idx = len(all_paragraphs)
+
+ for i, p in enumerate(all_paragraphs):
+ cls = get_css_class(p)
+ text = decode_text(p)
+
+ # Find start: p56 containing "Bhagaväner" or p2 TOC entry
+ if start_idx is None:
+ if cls == BOOK_TITLE_CLASS and "Bhag" in text and "kath" in text:
+ start_idx = i
+ continue
+ continue
+
+ # Find end: p35 containing "Indian Bhagavatams"
+ if cls == "p35" and "Indian" in text and "Bhagavatam" in text:
+ end_idx = i
+ break
+
+ if start_idx is None:
+ print("ERROR: Could not find 'Bhagaväner kathä' in others-before-1977.html!")
+ sys.exit(1)
+
+ return start_idx, end_idx
+
+
+def find_chapter_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
+ """Find all BK chapter bookmark tags (p17) in the BK range.
+
+ Returns list of (paragraph_index, raw_title_text).
+ """
+ bookmarks = []
+ for i in range(start_idx, end_idx):
+ p = all_paragraphs[i]
+ cls = get_css_class(p)
+ if cls == CHAPTER_BOOKMARK_CLASS:
+ text = decode_text(p).strip()
+ if text and "Bk:" in text:
+ bookmarks.append((i, text))
+ return bookmarks
+
+
+# ---------------------------------------------------------------------------
+# Paragraph extraction
+# ---------------------------------------------------------------------------
+
+
+def extract_paragraphs(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
+ """Extract body paragraphs for a chapter section.
+
+ Collects:
+ - p6, p12, p38 (body paragraphs — ScaGoudy decoded)
+ - p9, p19, p59, p23, p36, p60 (verse lines — centered italic, ScaGoudy decoded)
+ - p10, p61 (reference markers — ScaGoudy decoded)
+ - p57, p58 (footnotes — ScaGoudy decoded)
+ Skips:
+ - p56 (book title), p17 (bookmarks), p5 (chapter titles), p2 (TOC), empties
+ """
+ result = []
+
+ for i in range(start_idx, end_idx):
+ p = all_paragraphs[i]
+ cls = get_css_class(p)
+
+ # Skip empty paragraphs
+ if is_empty(p):
+ continue
+
+ # Skip known skip classes
+ if cls in SKIP_CLASSES:
+ continue
+
+ # Skip book title header
+ if cls == BOOK_TITLE_CLASS:
+ continue
+
+ # Skip chapter bookmark headers
+ if cls == CHAPTER_BOOKMARK_CLASS:
+ continue
+
+ # Skip chapter title headings (p5)
+ if cls == CHAPTER_TITLE_CLASS:
+ continue
+
+ # Skip TOC entries (p2)
+ if cls == TOC_CLASS:
+ continue
+
+ # Body paragraphs (ScaGoudy decoded)
+ if cls in BODY_CLASSES:
+ html = decode_html_content(p, apply_scagoudy=True)
+ if html:
+ result.append(html)
+ continue
+
+ # Sanskrit verse lines (centered italic, ScaGoudy decoded)
+ if cls in VERSE_CLASSES:
+ html = decode_html_content_as_verse(p)
+ if html:
+ result.append(html)
+ continue
+
+ # Reference markers (ScaGoudy decoded)
+ if cls in REFERENCE_CLASSES:
+ html = decode_html_content(p, apply_scagoudy=True)
+ if html:
+ result.append(html)
+ continue
+
+ # Footnote classes (ScaGoudy decoded)
+ if cls in FOOTNOTE_CLASSES:
+ html = decode_html_content(p, apply_scagoudy=True)
+ if html:
+ result.append(html)
+ continue
+
+ # Fallback: try to extract content from unknown classes
+ html = decode_html_content(p, apply_scagoudy=True)
+ if html:
+ print(f" ⚠ Unknown class '{cls}' at paragraph {i}: {html[:80]}...")
+ result.append(html)
+
+ return result
+
+
+# ---------------------------------------------------------------------------
+# Main extraction logic
+# ---------------------------------------------------------------------------
+
+
+def extract_bk(soup: BeautifulSoup) -> dict:
+ """Extract the entire Bhagavāner Kathā from the parsed HTML.
+
+ Returns a dict with:
+ "chapters": list of chapter dicts
+ "chapter_listing": list of {number, title} for intro.json
+ """
+ all_paragraphs = list(soup.find_all("p"))
+
+ # Step 1: Find BK boundaries
+ bk_start, bk_end = find_bk_boundaries(all_paragraphs)
+ print(f" BK content: paragraphs {bk_start} to {bk_end}")
+ print(f" Total paragraphs in range: {bk_end - bk_start}")
+
+ # Step 2: Find all chapter bookmarks (p17 with "Bk:")
+ bookmarks = find_chapter_bookmarks(all_paragraphs, bk_start, bk_end)
+ print(f" Found {len(bookmarks)} BK chapter bookmarks:")
+ for pos, title in bookmarks:
+ print(f" Paragraph {pos}: {title}")
+
+ # Step 3: Map bookmarks to chapters with proper titles
+ # Expected: 2 chapters
+ chapter_titles_map = {
+ 1: "This World of Misery",
+ 2: "The Cause of Misery",
+ }
+
+ chapter_bookmarks = [] # (para_idx, chapter_num, normalized_title)
+ for idx, (pos, _raw_title) in enumerate(bookmarks):
+ ch_num = idx + 1
+ ch_title = chapter_titles_map.get(ch_num, f"Chapter {ch_num}")
+ chapter_bookmarks.append((pos, ch_num, ch_title))
+
+ print(f"\n Mapped {len(chapter_bookmarks)} chapters:")
+ for pos, num, title in chapter_bookmarks:
+ print(f" Chapter {num}: {title} (paragraph {pos})")
+
+ # Step 4: Extract chapters
+ chapters = []
+ for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_bookmarks):
+ # Content starts after the bookmark line (skip p17 and p5 heading)
+ content_start = ch_pos + 1
+
+ # Content ends at next chapter bookmark or bk_end
+ if idx + 1 < len(chapter_bookmarks):
+ content_end = chapter_bookmarks[idx + 1][0]
+ else:
+ content_end = bk_end
+
+ paragraphs = extract_paragraphs(all_paragraphs, content_start, content_end)
+
+ ch = {
+ "book": "bk",
+ "chapter": ch_num,
+ "title": ch_title,
+ "type": "chapter",
+ "paragraphs": paragraphs,
+ }
+ chapters.append(ch)
+
+ # Step 5: Build chapter listing for intro.json
+ chapter_listing = [
+ {"number": ch["chapter"], "title": ch["title"]} for ch in chapters
+ ]
+
+ return {
+ "chapters": chapters,
+ "chapter_listing": chapter_listing,
+ }
+
+
+# ---------------------------------------------------------------------------
+# Output
+# ---------------------------------------------------------------------------
+
+
+def write_json(data, filepath):
+ """Write data as formatted JSON to filepath, creating directories as needed."""
+ os.makedirs(os.path.dirname(filepath), exist_ok=True)
+ with open(filepath, "w", encoding="utf-8") as f:
+ json.dump(data, f, ensure_ascii=False, indent=2)
+
+
+def main():
+ base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+ html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html")
+ output_base = os.path.join(base_dir, "lib", "jsons", "books", "bk")
+
+ if not os.path.exists(html_path):
+ print(f"ERROR: {html_path} not found!")
+ sys.exit(1)
+
+ print(f"Reading {html_path}...")
+ with open(html_path, "r", encoding="utf-8") as f:
+ soup = BeautifulSoup(f, "html.parser")
+
+ print("Extracting Bhagavāner Kathā...\n")
+ result = extract_bk(soup)
+
+ chapters = result["chapters"]
+ chapter_listing = result["chapter_listing"]
+
+ # --- Write intro.json ---
+ intro_meta = {
+ "book": "bk",
+ "title": "Bhagavāner Kathā",
+ "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
+ "year": None,
+ "description": (
+ "Early Bengali writings on the topics of Godhead "
+ "— part of 'Renunciation through Wisdom' (Vairāgya-vidyā)."
+ ),
+ "totalChapters": 2,
+ "chapters": chapter_listing,
+ }
+ intro_path = os.path.join(output_base, "intro.json")
+ write_json(intro_meta, intro_path)
+ print(f"\n Written: {intro_path}")
+
+ # --- Write chapters ---
+ print(f"\n--- Chapters ---")
+ for ch in chapters:
+ ch_num_str = str(ch["chapter"]).zfill(2)
+ filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
+ write_json(ch, filepath)
+ print(
+ f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
+ f"{ch['title']}"
+ )
+
+ # --- Summary ---
+ total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
+
+ print(f"\n{'=' * 60}")
+ print(f" EXTRACTION SUMMARY")
+ print(f"{'=' * 60}")
+ print(f" Chapters: {len(chapters)}")
+ for ch in chapters:
+ print(
+ f" Ch {ch['chapter']}: {len(ch['paragraphs'])} paragraphs — {ch['title']}"
+ )
+ print(f" Total paragraphs: {total_chapter_paras}")
+ if chapters:
+ print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
+ print(f" Output directory: {output_base}/")
+
+ # --- Samples ---
+ for ch in chapters:
+ ch_num = ch["chapter"]
+ print(f"\n--- Sample: Chapter {ch_num} ({ch['title']}) ---")
+ for i, p in enumerate(ch["paragraphs"][:3]):
+ preview = re.sub(r"<[^>]+>", "", p)[:120]
+ print(f" [{i}] {preview}...")
+ if len(ch["paragraphs"]) > 3:
+ print(f" ... ({len(ch['paragraphs']) - 3} more paragraphs)")
+
+ # --- Quality Checks ---
+ print(f"\n--- Quality Checks ---")
+
+ # Check all 2 chapters present
+ expected_chapters = set(range(1, 3))
+ actual_chapters = set(ch["chapter"] for ch in chapters)
+ missing = expected_chapters - actual_chapters
+ extra = actual_chapters - expected_chapters
+ if missing:
+ print(f" ⚠ Missing chapters: {sorted(missing)}")
+ if extra:
+ print(f" ⚠ Extra chapters: {sorted(extra)}")
+ if not missing and not extra:
+ print(f" ✅ All 2 chapters present")
+
+ # Check empty chapters
+ empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
+ if empty_chapters:
+ print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
+ else:
+ print(f" ✅ No empty chapters")
+
+ # Check ScaGoudy decoding worked — look for common ScaGoudy characters
+ scagoudy_chars = set("äåñöëòçìïüéàù")
+ all_text = ""
+ for ch in chapters:
+ all_text += " ".join(ch["paragraphs"])
+ remaining_scagoudy = [c for c in all_text if c in scagoudy_chars]
+ if remaining_scagoudy:
+ # Count occurrences
+ from collections import Counter
+
+ counts = Counter(remaining_scagoudy)
+ print(
+ f" ⚠ Remaining ScaGoudy characters found (may be false positives): {dict(counts)}"
+ )
+ else:
+ print(f" ✅ No ScaGoudy characters remaining — decoding looks clean")
+
+ # Check for IAST characters (positive signal)
+ iast_chars = set("āṛṣṭṇḍśṅñūīṁḥ")
+ iast_found = [c for c in all_text if c in iast_chars]
+ if iast_found:
+ from collections import Counter
+
+ counts = Counter(iast_found)
+ print(f" ✅ IAST Unicode characters present: {dict(counts)}")
+ else:
+ print(f" ⚠ No IAST Unicode characters found — decoding may have failed")
+
+ # Check chapter 1 has [RTW 1.1] reference
+ if chapters:
+ ch1_text = " ".join(chapters[0]["paragraphs"])
+ if "RTW" in ch1_text and "1.1" in ch1_text:
+ print(f" ✅ '[RTW 1.1]' reference found in Chapter 1")
+ else:
+ print(f" ⚠ '[RTW 1.1]' reference NOT found in Chapter 1")
+
+ # Check chapter 2 has [RTW 1.2] reference
+ if len(chapters) >= 2:
+ ch2_text = " ".join(chapters[1]["paragraphs"])
+ if "RTW" in ch2_text and "1.2" in ch2_text:
+ print(f" ✅ '[RTW 1.2]' reference found in Chapter 2")
+ else:
+ print(f" ⚠ '[RTW 1.2]' reference NOT found in Chapter 2")
+
+ # Check for content leak from next work (Indian Bhagavatams)
+ if "Indian Bhagavatam" in all_text:
+ print(f" ⚠ Content from next work (Indian Bhagavatams) may have leaked!")
+ else:
+ print(f" ✅ No content leak from next work detected")
+
+ # Check Kṛṣṇa appears (decoded from Kåñëa)
+ if "Kṛṣṇa" in all_text:
+ print(f" ✅ 'Kṛṣṇa' properly decoded (from 'Kåñëa')")
+ else:
+ print(f" ⚠ 'Kṛṣṇa' not found — ScaGoudy decoding may need review")
+
+ # Check last chapter ends cleanly
+ if chapters:
+ last_ch = chapters[-1]
+ last_para = last_ch["paragraphs"][-1] if last_ch["paragraphs"] else ""
+ last_preview = re.sub(r"<[^>]+>", "", last_para)[:100]
+ print(f' ℹ Last paragraph of Ch {last_ch["chapter"]}: "{last_preview}..."')
+
+ print(f"\nDone! ✨")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/cbhc_extract.py b/scripts/cbhc_extract.py
new file mode 100644
index 00000000..a137c907
--- /dev/null
+++ b/scripts/cbhc_extract.py
@@ -0,0 +1,444 @@
+#!/usr/bin/env python3
+"""Extract Śrī Caitanya-bhāgavata Commentary from others-before-1977.html.
+
+The book has 4 chapters (no intro segments):
+ 1. Chapter 1
+ 2. Lord Caitanya's Appearance
+ 3. Chapter 3
+ 4. Chapter 4
+
+Source: public/html/others-before-1977.html, lines ~4139–4180.
+ Start: Çré Caitanya-bhägavata-Commentary Bhagaväner kathä [Original Documents] tags containing "Caitanya-bhägavata"
+
+CSS classes:
+ p3 — Book title header (skip)
+ p17 — Chapter bookmark headers (boundary detection, skip from output)
+ p5 — Chapter title headings (skip — we derive titles ourselves)
+ p6 — Body paragraphs (MAIN content)
+ p38 — Body paragraph variant
+ p55 — Body paragraph variant (last paragraph of chapter 4)
+ p9 — End-of-chapter markers like "(End Chapter One)" — include
+
+Output:
+ lib/jsons/books/cbhc/intro.json — Book metadata
+ lib/jsons/books/cbhc/chapters/01.json — Chapter 1
+ lib/jsons/books/cbhc/chapters/02.json — Chapter 2
+ lib/jsons/books/cbhc/chapters/03.json — Chapter 3
+ lib/jsons/books/cbhc/chapters/04.json — Chapter 4
+
+Usage:
+ python3 scripts/cbhc_extract.py
+"""
+
+import json
+import os
+import re
+import sys
+
+from bs4 import BeautifulSoup, Tag
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def decode_html_content(tag: Tag) -> str:
+ """Extract inner HTML from a tag, preserve formatting tags, strip spans."""
+ inner = "".join(str(child) for child in tag.children)
+ # Remove span tags with CSS classes (RTF conversion artifacts)
+ inner = re.sub(
+ r' containing "Caitanya-bhägavata-Commentary"
+ End: containing "Bhagaväner kathä" (next work)
+ """
+ start_idx = None
+ end_idx = len(all_paragraphs)
+
+ for i, p in enumerate(all_paragraphs):
+ cls = get_css_class(p)
+ text = decode_text(p)
+
+ # Find start: p3 containing "Caitanya-bhägavata"
+ if start_idx is None:
+ if cls == BOOK_TITLE_CLASS and "Caitanya-bh" in text:
+ start_idx = i
+ continue
+ continue
+
+ # Find end: p2 containing "Bhagaväner" (next work boundary)
+ if cls == "p2" and "Bhag" in text and "kath" in text:
+ end_idx = i
+ break
+
+ if start_idx is None:
+ print(
+ "ERROR: Could not find 'Çré Caitanya-bhägavata-Commentary' "
+ "in others-before-1977.html!"
+ )
+ sys.exit(1)
+
+ return start_idx, end_idx
+
+
+def find_chapter_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
+ """Find all CBHC chapter bookmark tags (p17) in the CBHC range.
+
+ Returns list of (paragraph_index, raw_title_text).
+ """
+ bookmarks = []
+ for i in range(start_idx, end_idx):
+ p = all_paragraphs[i]
+ cls = get_css_class(p)
+ if cls == CHAPTER_BOOKMARK_CLASS:
+ text = decode_text(p).strip()
+ if text and "Caitanya-bh" in text:
+ bookmarks.append((i, text))
+ return bookmarks
+
+
+# ---------------------------------------------------------------------------
+# Paragraph extraction
+# ---------------------------------------------------------------------------
+
+
+def extract_paragraphs(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
+ """Extract body paragraphs for a chapter section.
+
+ Collects p6, p38, p55, and p9 (end markers).
+ Skips p3 (book header), p17 (bookmarks), p5 (chapter titles), p2, empties.
+ """
+ result = []
+
+ for i in range(start_idx, end_idx):
+ p = all_paragraphs[i]
+ cls = get_css_class(p)
+
+ # Skip empty paragraphs
+ if is_empty(p):
+ continue
+
+ # Skip known skip classes
+ if cls in SKIP_CLASSES:
+ continue
+
+ # Skip book title header
+ if cls == BOOK_TITLE_CLASS:
+ continue
+
+ # Skip chapter bookmark headers
+ if cls == CHAPTER_BOOKMARK_CLASS:
+ continue
+
+ # Skip chapter title headings (p5)
+ if cls == CHAPTER_TITLE_CLASS:
+ continue
+
+ # Skip p2 (TOC entries)
+ if cls == "p2":
+ continue
+
+ # Body paragraphs
+ if cls in BODY_CLASSES:
+ html = decode_html_content(p)
+ if html:
+ result.append(html)
+ continue
+
+ # End-of-chapter markers (p9) — include as a paragraph
+ if cls == END_MARKER_CLASS:
+ html = decode_html_content(p)
+ if html:
+ result.append(html)
+ continue
+
+ # Fallback: try to extract content from unknown classes
+ html = decode_html_content(p)
+ if html:
+ print(f" ⚠ Unknown class '{cls}' at paragraph {i}: {html[:80]}...")
+ result.append(html)
+
+ return result
+
+
+# ---------------------------------------------------------------------------
+# Main extraction logic
+# ---------------------------------------------------------------------------
+
+
+def extract_cbhc(soup: BeautifulSoup) -> dict:
+ """Extract the entire Śrī Caitanya-bhāgavata Commentary from the parsed HTML.
+
+ Returns a dict with:
+ "chapters": list of chapter dicts
+ "chapter_listing": list of {number, title} for intro.json
+ """
+ all_paragraphs = list(soup.find_all("p"))
+
+ # Step 1: Find CBHC boundaries
+ cbhc_start, cbhc_end = find_cbhc_boundaries(all_paragraphs)
+ print(f" CBHC content: paragraphs {cbhc_start} to {cbhc_end}")
+ print(f" Total paragraphs in range: {cbhc_end - cbhc_start}")
+
+ # Step 2: Find all chapter bookmarks (p17)
+ bookmarks = find_chapter_bookmarks(all_paragraphs, cbhc_start, cbhc_end)
+ print(f" Found {len(bookmarks)} CBHC chapter bookmarks:")
+ for pos, title in bookmarks:
+ print(f" Paragraph {pos}: {title}")
+
+ # Step 3: Map bookmarks to chapters with proper titles
+ # Expected: 4 chapters
+ # We derive proper titles from the p5 heading that follows each bookmark
+ chapter_titles_map = {
+ 1: "Chapter 1",
+ 2: "Lord Caitanya's Appearance",
+ 3: "Chapter 3",
+ 4: "Chapter 4",
+ }
+
+ chapter_bookmarks = [] # (para_idx, chapter_num, normalized_title)
+ for idx, (pos, _raw_title) in enumerate(bookmarks):
+ ch_num = idx + 1
+ ch_title = chapter_titles_map.get(ch_num, f"Chapter {ch_num}")
+ chapter_bookmarks.append((pos, ch_num, ch_title))
+
+ print(f"\n Mapped {len(chapter_bookmarks)} chapters:")
+ for pos, num, title in chapter_bookmarks:
+ print(f" Chapter {num}: {title} (paragraph {pos})")
+
+ # Step 4: Extract chapters
+ chapters = []
+ for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_bookmarks):
+ # Content starts after the bookmark line (skip p17 and p5 heading)
+ content_start = ch_pos + 1
+
+ # Content ends at next chapter bookmark or cbhc_end
+ if idx + 1 < len(chapter_bookmarks):
+ content_end = chapter_bookmarks[idx + 1][0]
+ else:
+ content_end = cbhc_end
+
+ paragraphs = extract_paragraphs(all_paragraphs, content_start, content_end)
+
+ ch = {
+ "book": "cbhc",
+ "chapter": ch_num,
+ "title": ch_title,
+ "type": "chapter",
+ "paragraphs": paragraphs,
+ }
+ chapters.append(ch)
+
+ # Step 5: Build chapter listing for intro.json
+ chapter_listing = [
+ {"number": ch["chapter"], "title": ch["title"]} for ch in chapters
+ ]
+
+ return {
+ "chapters": chapters,
+ "chapter_listing": chapter_listing,
+ }
+
+
+# ---------------------------------------------------------------------------
+# Output
+# ---------------------------------------------------------------------------
+
+
+def write_json(data, filepath):
+ """Write data as formatted JSON to filepath, creating directories as needed."""
+ os.makedirs(os.path.dirname(filepath), exist_ok=True)
+ with open(filepath, "w", encoding="utf-8") as f:
+ json.dump(data, f, ensure_ascii=False, indent=2)
+
+
+def main():
+ base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+ html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html")
+ output_base = os.path.join(base_dir, "lib", "jsons", "books", "cbhc")
+
+ if not os.path.exists(html_path):
+ print(f"ERROR: {html_path} not found!")
+ sys.exit(1)
+
+ print(f"Reading {html_path}...")
+ with open(html_path, "r", encoding="utf-8") as f:
+ soup = BeautifulSoup(f, "html.parser")
+
+ print("Extracting Śrī Caitanya-bhāgavata Commentary...\n")
+ result = extract_cbhc(soup)
+
+ chapters = result["chapters"]
+ chapter_listing = result["chapter_listing"]
+
+ # --- Write intro.json ---
+ intro_meta = {
+ "book": "cbhc",
+ "title": "Śrī Caitanya-bhāgavata Commentary",
+ "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
+ "year": None,
+ "description": (
+ "Commentary on Śrī Caitanya-bhāgavata by Vṛndāvana dāsa Ṭhākura "
+ "— covering the first four chapters of Lord Caitanya's pastimes."
+ ),
+ "totalChapters": 4,
+ "chapters": chapter_listing,
+ }
+ intro_path = os.path.join(output_base, "intro.json")
+ write_json(intro_meta, intro_path)
+ print(f"\n Written: {intro_path}")
+
+ # --- Write chapters ---
+ print(f"\n--- Chapters ---")
+ for ch in chapters:
+ ch_num_str = str(ch["chapter"]).zfill(2)
+ filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
+ write_json(ch, filepath)
+ print(
+ f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
+ f"{ch['title']}"
+ )
+
+ # --- Summary ---
+ total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
+
+ print(f"\n{'=' * 60}")
+ print(f" EXTRACTION SUMMARY")
+ print(f"{'=' * 60}")
+ print(f" Chapters: {len(chapters)}")
+ for ch in chapters:
+ print(
+ f" Ch {ch['chapter']}: {len(ch['paragraphs'])} paragraphs — {ch['title']}"
+ )
+ print(f" Total paragraphs: {total_chapter_paras}")
+ if chapters:
+ print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
+ print(f" Output directory: {output_base}/")
+
+ # --- Samples ---
+ for ch in chapters:
+ ch_num = ch["chapter"]
+ print(f"\n--- Sample: Chapter {ch_num} ({ch['title']}) ---")
+ for i, p in enumerate(ch["paragraphs"][:3]):
+ preview = re.sub(r"<[^>]+>", "", p)[:120]
+ print(f" [{i}] {preview}...")
+ if len(ch["paragraphs"]) > 3:
+ print(f" ... ({len(ch['paragraphs']) - 3} more paragraphs)")
+
+ # --- Quality Checks ---
+ print(f"\n--- Quality Checks ---")
+
+ # Check all 4 chapters present
+ expected_chapters = set(range(1, 5))
+ actual_chapters = set(ch["chapter"] for ch in chapters)
+ missing = expected_chapters - actual_chapters
+ extra = actual_chapters - expected_chapters
+ if missing:
+ print(f" ⚠ Missing chapters: {sorted(missing)}")
+ if extra:
+ print(f" ⚠ Extra chapters: {sorted(extra)}")
+ if not missing and not extra:
+ print(f" ✅ All 4 chapters present")
+
+ # Check empty chapters
+ empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
+ if empty_chapters:
+ print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
+ else:
+ print(f" ✅ No empty chapters")
+
+ # Check chapter 1 has "(End Chapter One)"
+ if chapters:
+ ch1_text = " ".join(chapters[0]["paragraphs"])
+ if "End Chapter One" in ch1_text:
+ print(f" ✅ '(End Chapter One)' end marker found in Chapter 1")
+ else:
+ print(f" ⚠ '(End Chapter One)' end marker NOT found in Chapter 1")
+
+ # Check chapter 2 mentions Lord Caitanya's appearance
+ if len(chapters) >= 2:
+ ch2_text = " ".join(chapters[1]["paragraphs"])
+ if "appearance" in ch2_text.lower() or "born" in ch2_text.lower():
+ print(f" ✅ Chapter 2 contains appearance/birth content")
+ else:
+ print(f" ⚠ Chapter 2 may not contain appearance content")
+
+ # Check for content leak from next work (Bhagaväner kathä)
+ all_text = ""
+ for ch in chapters:
+ all_text += " ".join(ch["paragraphs"])
+ if "Original Documents" in all_text:
+ print(f" ⚠ Content from next work (Bhagaväner kathä) may have leaked!")
+ else:
+ print(f" ✅ No content leak from next work detected")
+
+ # Check last chapter ends cleanly
+ if chapters:
+ last_ch = chapters[-1]
+ last_para = last_ch["paragraphs"][-1] if last_ch["paragraphs"] else ""
+ last_preview = re.sub(r"<[^>]+>", "", last_para)[:100]
+ print(f' ℹ Last paragraph of Ch {last_ch["chapter"]}: "{last_preview}..."')
+
+ print(f"\nDone! ✨")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/cc_essays_extract.py b/scripts/cc_essays_extract.py
new file mode 100644
index 00000000..3c91a046
--- /dev/null
+++ b/scripts/cc_essays_extract.py
@@ -0,0 +1,675 @@
+#!/usr/bin/env python3
+"""Extract five "Essays on Śrī Caitanya-caritāmṛta" from others-before-1977.html.
+
+Essay 1: "The Spiritual Masters" (cc-essays, order 1, 1960)
+Essay 2: "The Absolute Truth" (cc-essays, order 2, 1957)
+Essay 3: "The Causes of Lord Caitanya's Incarnation" (cc-essays, order 3, 1957)
+Essay 4: "The Confidential Reasons for Lord Caitanya's Appearance" (cc-essays, order 4, 1957)
+Essay 5: "The Glories of Lord Nityānanda" (cc-essays, order 5, 1957)
+
+All five are plain English (NO ScaGoudy decoding needed).
+
+Output:
+ lib/jsons/essays/the-spiritual-masters.json
+ lib/jsons/essays/the-absolute-truth.json
+ lib/jsons/essays/causes-of-lord-caitanyas-incarnation.json
+ lib/jsons/essays/confidential-reasons-for-lord-caitanyas-appearance.json
+ lib/jsons/essays/glories-of-lord-nityananda.json
+
+Usage:
+ python3 scripts/cc_essays_extract.py
+"""
+
+import json
+import os
+import re
+import sys
+
+from bs4 import BeautifulSoup, Tag
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def get_css_class(tag: Tag) -> str:
+ """Get the first CSS class of a tag, or empty string."""
+ classes = tag.get("class", [])
+ return classes[0] if classes else ""
+
+
+def is_empty(tag: Tag) -> bool:
+ """Check if a tag is empty or whitespace-only."""
+ text = tag.get_text().strip()
+ return not text or text == "\xa0"
+
+
+def clean_html_content(tag: Tag) -> str:
+ """Extract inner HTML from a tag, strip span class artifacts, preserve , , tags."""
+ with open(html_path, "r", encoding="utf-8") as f:
+ content = f.read()
+ soup = BeautifulSoup(content, "html.parser")
+ return list(soup.find_all("p"))
+
+
+def find_tag_index(tags: list[Tag], start_from: int, predicate) -> int | None:
+ """Find index of first tag matching predicate, starting from start_from."""
+ for i in range(start_from, len(tags)):
+ if predicate(tags[i]):
+ return i
+ return None
+
+
+def extract_body_paragraphs(
+ tags: list[Tag], start_idx: int, end_idx: int, include_p9_content: bool = True
+) -> list[str]:
+ """Extract body paragraphs from tags[start_idx] to tags[end_idx] (exclusive).
+
+ Includes p6 paragraphs (main body) and optionally p9 paragraphs that
+ contain actual content (Sanskrit verses, section markers) but skips
+ p9 paragraphs that are dividers or page headers.
+ """
+ paragraphs = []
+ body_classes = {"p6"}
+ if include_p9_content:
+ body_classes.add("p9")
+
+ for i in range(start_idx, end_idx):
+ tag = tags[i]
+ css = get_css_class(tag)
+
+ if css not in body_classes:
+ continue
+
+ if is_empty(tag):
+ continue
+
+ # For p9 tags, skip dividers and page headers
+ if css == "p9":
+ if is_divider(tag) or is_page_header(tag):
+ continue
+
+ html = clean_html_content(tag)
+ if html and html != " ":
+ paragraphs.append(html)
+
+ return paragraphs
+
+
+def extract_body_paragraphs_extended(
+ tags: list[Tag],
+ start_idx: int,
+ end_idx: int,
+ body_classes: set[str] | None = None,
+ skip_editor_notes: bool = False,
+) -> list[str]:
+ """Extract body paragraphs from tags[start_idx] to tags[end_idx] (exclusive).
+
+ Like extract_body_paragraphs but with configurable body classes and
+ optional editor-note skipping.
+ """
+ if body_classes is None:
+ body_classes = {"p6"}
+
+ paragraphs = []
+
+ for i in range(start_idx, end_idx):
+ tag = tags[i]
+ css = get_css_class(tag)
+
+ if css not in body_classes:
+ continue
+
+ if is_empty(tag):
+ continue
+
+ # For p9 tags, skip dividers and page headers
+ if css == "p9":
+ if is_divider(tag) or is_page_header(tag):
+ continue
+
+ # Skip editor notes if requested
+ if skip_editor_notes and is_editor_note(tag):
+ continue
+
+ html = clean_html_content(tag)
+ if html and html != " ":
+ paragraphs.append(html)
+
+ return paragraphs
+
+
+def extract_essay_1(tags: list[Tag]) -> dict:
+ """Extract 'The Spiritual Masters' essay.
+
+ Start: (The Spiritual Masters). (contains [BTG 1960])
+ """
+ # Find the title tag
+ start_idx = find_tag_index(
+ tags,
+ 0,
+ lambda t: get_css_class(t) == "p8" and "Spiritual Masters" in t.get_text(),
+ )
+ if start_idx is None:
+ raise ValueError("Could not find Essay 1 start marker (p8 'Spiritual Masters')")
+
+ # Find the end marker: next p10
+ end_idx = find_tag_index(tags, start_idx + 1, lambda t: get_css_class(t) == "p10")
+ if end_idx is None:
+ raise ValueError("Could not find Essay 1 end marker (p10)")
+
+ print(f" Essay 1: tags[{start_idx}] → tags[{end_idx}] (exclusive)")
+
+ # Body starts after the p8 title tag
+ paragraphs = extract_body_paragraphs(tags, start_idx + 1, end_idx)
+
+ essay = dict(ESSAYS[0])
+ essay["paragraphs"] = paragraphs
+ return essay
+
+
+def extract_essay_2(tags: list[Tag]) -> dict:
+ """Extract 'The Absolute Truth' essay.
+
+ Start: (Second Part) after start
+ """
+ # Find the start marker
+ start_idx = find_tag_index(
+ tags,
+ 0,
+ lambda t: get_css_class(t) == "p9" and "(Second Part)" in t.get_text(),
+ )
+ if start_idx is None:
+ raise ValueError("Could not find Essay 2 start marker (p9 '(Second Part)')")
+
+ # Find the end marker: next p11
+ end_idx = find_tag_index(tags, start_idx + 1, lambda t: get_css_class(t) == "p11")
+ if end_idx is None:
+ raise ValueError("Could not find Essay 2 end marker (p11)")
+
+ print(f" Essay 2: tags[{start_idx}] → tags[{end_idx}] (exclusive)")
+
+ # Body starts after the "(Second Part)" header
+ paragraphs = extract_body_paragraphs(tags, start_idx + 1, end_idx)
+
+ essay = dict(ESSAYS[1])
+ essay["paragraphs"] = paragraphs
+ return essay
+
+
+def extract_essay_3(tags: list[Tag]) -> dict:
+ """Extract 'The Causes of Lord Caitanya's Incarnation' essay.
+
+ Start: containing "Essays on Shri Chaitanya Charitamrita. III"
+ Body: p6 paragraphs after start
+ End: Next after start.
+ The ____ Thus ends the III part... containing "4th Chapter"
+ Body: p6 paragraphs after start (skip editor note, skip p4/p5/p29/p30 headers)
+ End: Next after start (next chapter header "Ädi-lélä, Chapter 5")
+ """
+ # Find the start marker: p30 containing "4th Chapter"
+ start_idx = find_tag_index(
+ tags,
+ 0,
+ lambda t: get_css_class(t) == "p30" and "4th Chapter" in t.get_text(),
+ )
+ if start_idx is None:
+ raise ValueError("Could not find Essay 4 start marker (p30 '4th Chapter')")
+
+ # Find the end marker: next p4 after the start (next chapter header)
+ end_idx = find_tag_index(tags, start_idx + 1, lambda t: get_css_class(t) == "p4")
+ if end_idx is None:
+ raise ValueError("Could not find Essay 4 end marker (p4)")
+
+ print(f" Essay 4: tags[{start_idx}] → tags[{end_idx}] (exclusive)")
+
+ # Collect p6 paragraphs, skipping editor notes
+ paragraphs = extract_body_paragraphs_extended(
+ tags,
+ start_idx + 1,
+ end_idx,
+ body_classes={"p6"},
+ skip_editor_notes=True,
+ )
+
+ essay = dict(ESSAYS[3])
+ essay["paragraphs"] = paragraphs
+ return essay
+
+
+def extract_essay_5(tags: list[Tag]) -> dict:
+ """Extract 'The Glories of Lord Nityānanda' essay.
+
+ Start: containing "1.V" (i.e. "Shi Chaitanya Charitamrita 1.V")
+ Body: p6 and p34 paragraphs after start (skip editor note, skip p4/p5/p31 headers,
+ skip p9 divider underscores)
+ End: Next after start ("Easy Journey To Other Planets")
+ """
+ # Find the start marker: p31 containing "1.V"
+ start_idx = find_tag_index(
+ tags,
+ 0,
+ lambda t: get_css_class(t) == "p31" and "1.V" in t.get_text(),
+ )
+ if start_idx is None:
+ raise ValueError("Could not find Essay 5 start marker (p31 containing '1.V')")
+
+ # Find the end marker: next p35 after the start
+ end_idx = find_tag_index(tags, start_idx + 1, lambda t: get_css_class(t) == "p35")
+ if end_idx is None:
+ raise ValueError("Could not find Essay 5 end marker (p35)")
+
+ print(f" Essay 5: tags[{start_idx}] → tags[{end_idx}] (exclusive)")
+
+ # Collect p6 and p34 paragraphs, skipping editor notes and p9 dividers
+ paragraphs = extract_body_paragraphs_extended(
+ tags,
+ start_idx + 1,
+ end_idx,
+ body_classes={"p6", "p34"},
+ skip_editor_notes=True,
+ )
+
+ essay = dict(ESSAYS[4])
+ essay["paragraphs"] = paragraphs
+ return essay
+
+
+# ---------------------------------------------------------------------------
+# Quality checks
+# ---------------------------------------------------------------------------
+
+
+def quality_check(essay: dict, essay_num: int) -> None:
+ """Run quality checks on an extracted essay."""
+ paragraphs = essay["paragraphs"]
+ title = essay["title"]
+
+ print(f"\n--- Quality Checks: Essay {essay_num} ({title}) ---")
+
+ # Check paragraph count is reasonable
+ if len(paragraphs) < 5:
+ print(f" ⚠ Only {len(paragraphs)} paragraphs (suspiciously low)")
+ elif len(paragraphs) > 150:
+ print(f" ⚠ {len(paragraphs)} paragraphs (suspiciously high)")
+ else:
+ print(f" ✅ {len(paragraphs)} paragraphs (reasonable)")
+
+ # Check for residual span class artifacts
+ span_count = sum(1 for p in paragraphs if 'class="' in p and " tags total\n")
+
+ # Extract all five essays
+ print("Locating essays...")
+ essay_1 = extract_essay_1(tags)
+ essay_2 = extract_essay_2(tags)
+ essay_3 = extract_essay_3(tags)
+ essay_4 = extract_essay_4(tags)
+ essay_5 = extract_essay_5(tags)
+
+ all_essays = [essay_1, essay_2, essay_3, essay_4, essay_5]
+
+ # Write JSON files
+ print(f"\nWriting JSON files...")
+ for essay in all_essays:
+ output_path = os.path.join(output_dir, f"{essay['slug']}.json")
+ write_json(essay, output_path)
+ print(f" Written: {output_path}")
+
+ # Quality checks
+ for i, essay in enumerate(all_essays, 1):
+ quality_check(essay, i)
+
+ # Summary
+ print(f"\n{'=' * 60}")
+ print(f" EXTRACTION SUMMARY")
+ print(f"{'=' * 60}")
+ for i, essay in enumerate(all_essays, 1):
+ print(
+ f" Essay {i}: {essay['title']}"
+ f" ({len(essay['paragraphs'])} paragraphs, {essay['year']})"
+ )
+ print(
+ f" slug={essay['slug']}, group={essay['group']}, order={essay['order']}"
+ )
+ print(f"{'=' * 60}")
+ print(f" Total essays: {len(all_essays)}")
+ print(f" Total paragraphs: {sum(len(e['paragraphs']) for e in all_essays)}")
+ print(f"\nDone! ✨")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/jaladuta_diary_extract.py b/scripts/jaladuta_diary_extract.py
new file mode 100644
index 00000000..21b26ab4
--- /dev/null
+++ b/scripts/jaladuta_diary_extract.py
@@ -0,0 +1,688 @@
+#!/usr/bin/env python3
+"""Extract The Jaladuta Diary (1965) from others-before-1977.html.
+
+Each diary day entry becomes its own chapter file.
+Chapters are grouped by month in the intro.json metadata.
+
+Structure:
+ - Preface and Introduction are standalone chapters (slugs "preface", "introduction")
+ - Day entries use slugs like "apr-3", "may-2", "aug-13", "sep-1", etc.
+ - The "Memoranda" entry at the end uses slug "memoranda"
+ - Months are parent groups in intro.json (non-clickable headings)
+
+Output:
+ lib/jsons/journals/jaladuta-diary/intro.json
+ lib/jsons/journals/jaladuta-diary/chapters/preface.json
+ lib/jsons/journals/jaladuta-diary/chapters/introduction.json
+ lib/jsons/journals/jaladuta-diary/chapters/apr-3.json
+ lib/jsons/journals/jaladuta-diary/chapters/apr-5.json
+ ...
+ lib/jsons/journals/jaladuta-diary/chapters/nov-28.json
+ lib/jsons/journals/jaladuta-diary/chapters/memoranda.json
+
+Usage:
+ python3 scripts/jaladuta_diary_extract.py
+"""
+
+import json
+import os
+import re
+import sys
+
+from bs4 import BeautifulSoup, Tag
+
+# ---------------------------------------------------------------------------
+# ScaGoudy → IAST Unicode mapping
+# ---------------------------------------------------------------------------
+SCAGOUDY_TO_IAST = {
+ "ä": "ā",
+ "Ä": "Ā",
+ "å": "ṛ",
+ "Å": "Ṛ",
+ "ñ": "ṣ",
+ "Ñ": "Ṣ",
+ "ö": "ṭ",
+ "Ö": "Ṭ",
+ "ë": "ṇ",
+ "Ë": "Ṇ",
+ "ò": "ḍ",
+ "Ò": "Ḍ",
+ "ç": "ś",
+ "Ç": "Ś",
+ "ì": "ṅ",
+ "Ì": "Ṅ",
+ "ï": "ñ",
+ "Ï": "Ñ",
+ "ü": "ū",
+ "Ü": "Ū",
+ "é": "ī",
+ "É": "Ī",
+ "à": "ṁ",
+ "ù": "ḥ",
+ "è": "ṝ",
+ "ÿ": "ḷ",
+}
+
+
+def decode_scagoudy(text: str) -> str:
+ """Replace ScaGoudy-encoded characters with proper IAST Unicode."""
+ for src, dst in SCAGOUDY_TO_IAST.items():
+ text = text.replace(src, dst)
+ return text
+
+
+def get_css_class(tag: Tag) -> str:
+ classes = tag.get("class", [])
+ return classes[0] if classes else ""
+
+
+def inner_html(tag: Tag) -> str:
+ """Get inner HTML of a tag, decode ScaGoudy, clean up spans."""
+ html = "".join(str(child) for child in tag.children)
+ html = decode_scagoudy(html)
+ # text → text
+ html = re.sub(
+ r' tags total")
+
+ # Find the Jaladuta Diary section boundaries
+ start_idx = None
+ end_idx = None
+ for i, tag in enumerate(all_tags):
+ text = tag.get_text().strip()
+ cls = get_css_class(tag)
+ if cls == "p35" and "Jaladuta Diary" in text:
+ start_idx = i
+ if start_idx is not None and i > start_idx:
+ # End at "Prayers Written by Śrīla Prabhupāda aboard the Jaladuta" or the next p4 bookmark after JD
+ if cls == "p4" and "JD 8" in text:
+ end_idx = i
+ break
+
+ if start_idx is None:
+ print("ERROR: Could not find Jaladuta Diary start")
+ sys.exit(1)
+ if end_idx is None:
+ # Fallback: use the memoranda end
+ for i, tag in enumerate(all_tags):
+ if (
+ i > start_idx
+ and "MEMORANDA" in tag.get_text()
+ and get_css_class(tag) == "p110"
+ ):
+ end_idx = i + 1
+ break
+
+ print(f" Section: tags[{start_idx}] → tags[{end_idx}]")
+
+ section_tags = all_tags[start_idx:end_idx]
+
+ # ---------------------------------------------------------------------------
+ # Phase 1: Extract Preface and Introduction
+ # ---------------------------------------------------------------------------
+ preface_paragraphs = []
+ intro_paragraphs = []
+ current_section = None # "preface" | "introduction" | None
+
+ # Also track where months/diary entries begin
+ diary_start_idx = None
+
+ for i, tag in enumerate(section_tags):
+ cls = get_css_class(tag)
+ text = tag.get_text().strip()
+
+ # Detect Preface header
+ if cls == "p3" and text == "Preface":
+ current_section = "preface"
+ continue
+ if cls == "p153": # bookmark
+ continue
+
+ # Detect Introduction header
+ if cls in ("p11", "p157") and "Introduction" in text:
+ if cls == "p157":
+ current_section = "introduction"
+ continue
+ elif cls == "p11":
+ current_section = "introduction"
+ continue
+
+ # Detect month start (diary entries begin)
+ if cls == "p11" and re.match(r"JD \d+:", text):
+ diary_start_idx = i
+ current_section = None
+ break
+
+ if cls == "p29":
+ # Bold month header like "APRIL 1965" — this is diary territory
+ diary_start_idx = i
+ current_section = None
+ break
+
+ # Collect preface/intro paragraphs
+ if current_section == "preface" and cls in ("p154", "p155", "p156"):
+ html = inner_html(tag)
+ if html:
+ preface_paragraphs.append(html)
+ elif current_section == "introduction" and cls in ("p154", "p155", "p156"):
+ html = inner_html(tag)
+ if html:
+ intro_paragraphs.append(html)
+
+ print(f"\n Preface: {len(preface_paragraphs)} paragraphs")
+ print(f" Introduction: {len(intro_paragraphs)} paragraphs")
+
+ # ---------------------------------------------------------------------------
+ # Phase 2: Extract diary entries grouped by month
+ # ---------------------------------------------------------------------------
+ # Walk through remaining tags, tracking current month
+
+ current_month = None # "apr", "may", etc.
+ current_month_full = None # "April 1965", etc.
+
+ # Section headers that appear mid-month
+ section_headers = {} # Maps tag index to header text
+
+ # All day entries: list of { month, day, date, slug, title, paragraphs }
+ day_entries = []
+ current_entry = None
+
+ MONTH_MAP = {
+ "APRIL": ("apr", "April 1965"),
+ "MAY": ("may", "May 1965"),
+ "JUNE": ("jun", "June 1965"),
+ "JULY": ("jul", "July 1965"),
+ "AUGUST": ("aug", "August 1965"),
+ "SEPTEMBER": ("sep", "September 1965"),
+ "OCTOBER": ("oct", "October 1965"),
+ "NOVEMBER": ("nov", "November 1965"),
+ }
+
+ def finalize_entry():
+ nonlocal current_entry
+ if current_entry and current_entry["paragraphs"]:
+ day_entries.append(current_entry)
+ current_entry = None
+
+ diary_tags = section_tags[diary_start_idx:] if diary_start_idx else []
+
+ for i, tag in enumerate(diary_tags):
+ cls = get_css_class(tag)
+ text = tag.get_text().strip()
+
+ # Skip bookmarks
+ if cls in ("p11", "p4", "p39", "p153"):
+ continue
+
+ # Detect month headers (p29 bold)
+ if cls == "p29":
+ html = inner_html(tag)
+ # Check if this is a month header like "APRIL 1965" or "MAY 1965"
+ for month_key, (abbr, full) in MONTH_MAP.items():
+ if month_key in text.upper():
+ finalize_entry()
+ current_month = abbr
+ current_month_full = full
+ break
+ else:
+ # Could be a section header like "Verse 1." — skip if not month
+ # Or it could be a section header within a month
+ pass
+ continue
+
+ # Detect section headers (p68 class — "The Journey", "America")
+ if cls == "p68":
+ finalize_entry()
+ html = inner_html(tag)
+ # Create a section-header entry
+ if html and current_month:
+ section_entry = {
+ "month": current_month,
+ "month_full": current_month_full,
+ "day": None,
+ "date": None,
+ "slug": None,
+ "title": None,
+ "paragraphs": [f"{re.sub('<[^>]+>', ' ', html).strip()}"],
+ "is_section_header": True,
+ }
+ # Extract title text
+ plain = re.sub("<[^>]+>", " ", html)
+ plain = re.sub(r"\s+", " ", plain).strip()
+ section_entry["title"] = plain
+ day_entries.append(section_entry)
+ continue
+
+ # Detect diary entries (p110 class)
+ if cls == "p110":
+ finalize_entry()
+
+ html = inner_html(tag)
+ if not html:
+ continue
+
+ # Parse the bold header
+ b_tag = tag.find("b")
+ if b_tag:
+ bold_text = b_tag.get_text().strip()
+ day_name, date_num = parse_day_header(bold_text)
+ else:
+ day_name, date_num = (None, None)
+
+ if day_name == "MEMORANDA":
+ current_entry = {
+ "month": current_month,
+ "month_full": current_month_full,
+ "day": "MEMORANDA",
+ "date": None,
+ "slug": "memoranda",
+ "title": "Memoranda",
+ "paragraphs": [html],
+ "is_section_header": False,
+ }
+ elif day_name and date_num and current_month:
+ slug = f"{current_month}-{date_num}"
+ title = f"{day_name.capitalize()} {date_num}"
+ current_entry = {
+ "month": current_month,
+ "month_full": current_month_full,
+ "day": day_name,
+ "date": date_num,
+ "slug": slug,
+ "title": title,
+ "paragraphs": [html],
+ "is_section_header": False,
+ }
+ elif day_name and current_month:
+ # Day without date number (rare — like "SATURDAY" alone on Aug 14)
+ # Try to infer from context or just skip
+ slug = f"{current_month}-{day_name.lower()}"
+ title = day_name.capitalize()
+ current_entry = {
+ "month": current_month,
+ "month_full": current_month_full,
+ "day": day_name,
+ "date": None,
+ "slug": slug,
+ "title": title,
+ "paragraphs": [html],
+ "is_section_header": False,
+ }
+ else:
+ # Can't parse — treat as continuation or standalone
+ current_entry = {
+ "month": current_month,
+ "month_full": current_month_full,
+ "day": None,
+ "date": None,
+ "slug": None,
+ "title": text[:50],
+ "paragraphs": [html],
+ "is_section_header": False,
+ }
+ continue
+
+ # Detect p12 paragraphs (continuation text in some entries like Sep 16)
+ if cls == "p12" and current_entry:
+ html = inner_html(tag)
+ if html:
+ current_entry["paragraphs"].append(html)
+ continue
+
+ # p155 (source citations within diary section)
+ if cls == "p155" and current_entry:
+ html = inner_html(tag)
+ if html:
+ current_entry["paragraphs"].append(html)
+ continue
+
+ finalize_entry()
+
+ print(f"\n Total day entries extracted: {len(day_entries)}")
+
+ # Count by month
+ month_counts = {}
+ for entry in day_entries:
+ m = entry["month"]
+ month_counts[m] = month_counts.get(m, 0) + 1
+ for m in MONTH_ORDER:
+ if m in month_counts:
+ print(f" {m}: {month_counts[m]} entries")
+
+ # ---------------------------------------------------------------------------
+ # Phase 3: Fix up entries — handle dateless entries, dedup slugs
+ # ---------------------------------------------------------------------------
+
+ # Some entries have no date — try to fix known cases
+ # Aug "SATURDAY" without number is Aug 14 (follows Aug 13 FRIDAY)
+ for i, entry in enumerate(day_entries):
+ if entry.get("is_section_header"):
+ continue
+ if entry["date"] is None and entry["day"] and entry["day"] != "MEMORANDA":
+ # Try to infer from previous entry
+ if i > 0:
+ prev = day_entries[i - 1]
+ if prev["date"] and prev["month"] == entry["month"]:
+ inferred = prev["date"] + 1
+ entry["date"] = inferred
+ entry["slug"] = f"{entry['month']}-{inferred}"
+ entry["title"] = f"{entry['day'].capitalize()} {inferred}"
+ print(
+ f" Inferred date: {entry['slug']} (from previous {prev['slug']})"
+ )
+
+ # Ensure unique slugs
+ seen_slugs = set()
+ for entry in day_entries:
+ if entry.get("is_section_header"):
+ continue
+ slug = entry["slug"]
+ if slug and slug in seen_slugs:
+ # Append suffix
+ suffix = 2
+ while f"{slug}-{suffix}" in seen_slugs:
+ suffix += 1
+ entry["slug"] = f"{slug}-{suffix}"
+ print(f" Deduped slug: {slug} → {entry['slug']}")
+ if slug:
+ seen_slugs.add(entry["slug"])
+
+ # Merge section headers into the next entry's paragraphs (prepend)
+ merged_entries = []
+ pending_header = None
+ for entry in day_entries:
+ if entry.get("is_section_header"):
+ pending_header = entry["paragraphs"][0]
+ continue
+ if pending_header:
+ entry["paragraphs"] = [pending_header] + entry["paragraphs"]
+ pending_header = None
+ merged_entries.append(entry)
+ day_entries = merged_entries
+
+ # ---------------------------------------------------------------------------
+ # Phase 4: Build intro.json with hierarchical month groups
+ # ---------------------------------------------------------------------------
+
+ # Assign chapter numbers: preface=1, introduction=2, then day entries sequentially
+ chapter_num = 1
+
+ all_chapters = []
+
+ # Preface
+ preface_chapter = {
+ "slug": "preface",
+ "title": "Preface",
+ "number": chapter_num,
+ }
+ all_chapters.append(preface_chapter)
+ chapter_num += 1
+
+ # Introduction
+ intro_chapter = {
+ "slug": "introduction",
+ "title": "Introduction",
+ "number": chapter_num,
+ }
+ all_chapters.append(intro_chapter)
+ chapter_num += 1
+
+ # Group day entries by month
+ months_data = [] # List of { month, title, subChapters: [...] }
+ current_month_group = None
+
+ for entry in day_entries:
+ month = entry["month"]
+ month_full = entry["month_full"]
+
+ if current_month_group is None or current_month_group["month"] != month:
+ current_month_group = {
+ "month": month,
+ "title": month_full,
+ "subChapters": [],
+ }
+ months_data.append(current_month_group)
+
+ entry["number"] = chapter_num
+ current_month_group["subChapters"].append(
+ {
+ "slug": entry["slug"],
+ "title": entry["title"],
+ }
+ )
+ chapter_num += 1
+
+ # Build the intro.json chapters array
+ # Front matter chapters (preface, introduction) are top-level
+ # Month groups use the hierarchical sub-chapter pattern
+ intro_chapters = []
+
+ # Add preface and introduction as simple chapters
+ intro_chapters.append({"slug": "preface", "title": "Preface"})
+ intro_chapters.append({"slug": "introduction", "title": "Introduction"})
+
+ # Add month groups
+ for mg in months_data:
+ intro_chapters.append(
+ {
+ "title": mg["title"],
+ "subChapters": mg["subChapters"],
+ }
+ )
+
+ intro_json = {
+ "book": "jaladuta-diary",
+ "title": "The Jaladuta Diary, 1965 — Journey to the U.S.A.",
+ "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
+ "year": 1965,
+ "description": "Śrīla Prabhupāda's personal diary kept during his historic voyage from Calcutta to New York aboard the cargo ship Jaladuta in 1965.",
+ "totalChapters": chapter_num - 1,
+ "chapters": intro_chapters,
+ }
+
+ # ---------------------------------------------------------------------------
+ # Phase 5: Write JSON files
+ # ---------------------------------------------------------------------------
+
+ # Clean output directory
+ if os.path.exists(chapters_dir):
+ for f in os.listdir(chapters_dir):
+ os.remove(os.path.join(chapters_dir, f))
+
+ # Write intro.json
+ write_json(intro_json, os.path.join(output_dir, "intro.json"))
+ print(f"\n Written: intro.json ({len(intro_chapters)} top-level entries)")
+
+ # Write preface
+ write_json(
+ {
+ "book": "jaladuta-diary",
+ "chapter": 1,
+ "title": "Preface",
+ "type": "preface",
+ "slug": "preface",
+ "paragraphs": preface_paragraphs,
+ },
+ os.path.join(chapters_dir, "preface.json"),
+ )
+ print(f" Written: preface.json ({len(preface_paragraphs)} paragraphs)")
+
+ # Write introduction
+ write_json(
+ {
+ "book": "jaladuta-diary",
+ "chapter": 2,
+ "title": "Introduction",
+ "type": "introduction",
+ "slug": "introduction",
+ "paragraphs": intro_paragraphs,
+ },
+ os.path.join(chapters_dir, "introduction.json"),
+ )
+ print(f" Written: introduction.json ({len(intro_paragraphs)} paragraphs)")
+
+ # Write day entries
+ ch_num = 3
+ for entry in day_entries:
+ slug = entry["slug"]
+ if not slug:
+ print(f" ⚠ Skipping entry with no slug: {entry.get('title', '?')}")
+ continue
+
+ chapter_data = {
+ "book": "jaladuta-diary",
+ "chapter": ch_num,
+ "title": entry["title"],
+ "type": "chapter",
+ "slug": slug,
+ "paragraphs": entry["paragraphs"],
+ }
+ write_json(chapter_data, os.path.join(chapters_dir, f"{slug}.json"))
+ ch_num += 1
+
+ total_files = 2 + len([e for e in day_entries if e.get("slug")])
+ print(f"\n Total chapter files written: {total_files}")
+
+ # ---------------------------------------------------------------------------
+ # Summary
+ # ---------------------------------------------------------------------------
+ print(f"\n{'=' * 60}")
+ print(f" EXTRACTION SUMMARY")
+ print(f"{'=' * 60}")
+ print(f" Preface: {len(preface_paragraphs)} paragraphs")
+ print(f" Introduction: {len(intro_paragraphs)} paragraphs")
+ print(f" Day entries: {len(day_entries)}")
+ for mg in months_data:
+ print(f" {mg['title']}: {len(mg['subChapters'])} entries")
+ for sc in mg["subChapters"]:
+ print(f" - {sc['slug']}: {sc['title']}")
+ print(f" Total chapters: {ch_num - 1}")
+ print(f"{'=' * 60}")
+ print(f"\nDone! ✨")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/mog_extract.py b/scripts/mog_extract.py
new file mode 100644
index 00000000..67bd8af0
--- /dev/null
+++ b/scripts/mog_extract.py
@@ -0,0 +1,539 @@
+#!/usr/bin/env python3
+"""Extract Message of Godhead (1955) from others-before-1977.html.
+
+The book has 1 intro segment (front-pages) and 3 chapters:
+ 1. Message of Godhead (Introductory)
+ 2. Knowledge Transcendental
+ 3. Work with Transcendental Effect (Karma Yoga)
+
+Output:
+ lib/jsons/books/mog/intro.json — Book metadata
+ lib/jsons/books/mog/intro/*.json — Intro segments (front-pages)
+ lib/jsons/books/mog/chapters/*.json — Chapters (01–03)
+
+Usage:
+ python3 scripts/mog_extract.py
+"""
+
+import json
+import os
+import re
+import sys
+
+from bs4 import BeautifulSoup, Tag
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def decode_html_content(tag: Tag) -> str:
+ """Extract inner HTML from a tag, preserve formatting tags, strip spans."""
+ inner = "".join(str(child) for child in tag.children)
+ # Remove span tags with CSS classes (RTF conversion artifacts)
+ inner = re.sub(
+ r' containing "Message of Godhead"
+ End: next or that starts a different book,
+ or containing "Bhakti-rasāmṛta-sindhu" (the next work)
+ """
+ start_idx = None
+ end_idx = len(all_paragraphs)
+
+ for i, p in enumerate(all_paragraphs):
+ cls = get_css_class(p)
+ text = decode_text(p)
+
+ # Find start: p3 containing "Message of Godhead"
+ if start_idx is None:
+ if cls == "p2" and "Message of Godhead" in text:
+ # The p2 is the TOC entry; the actual start is the next p3
+ start_idx = i
+ continue
+ continue
+
+ # Find end: look for the next work boundary
+ # The p6 containing "Bhakti-rasāmṛta-sindhu" starts a different work
+ if cls == "p6" and "Bhakti-ras" in text and "sindhu" in text:
+ end_idx = i
+ break
+
+ # Also check for p2 that starts a different book section
+ if cls == "p2" and start_idx is not None and i > start_idx + 5:
+ text_lower = text.lower()
+ if "message of godhead" not in text_lower:
+ end_idx = i
+ break
+
+ if start_idx is None:
+ print("ERROR: Could not find 'Message of Godhead' in others-before-1977.html!")
+ sys.exit(1)
+
+ return start_idx, end_idx
+
+
+def find_chapter_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
+ """Find all MOG chapter bookmark tags (p11) in the MOG range.
+
+ Returns list of (paragraph_index, chapter_title).
+ """
+ bookmarks = []
+ for i in range(start_idx, end_idx):
+ p = all_paragraphs[i]
+ cls = get_css_class(p)
+ if cls == CHAPTER_BOOKMARK_CLASS:
+ text = decode_text(p).strip()
+ if text:
+ bookmarks.append((i, text))
+ return bookmarks
+
+
+# ---------------------------------------------------------------------------
+# Paragraph extraction
+# ---------------------------------------------------------------------------
+
+
+def extract_paragraphs(
+ all_paragraphs: list,
+ start_idx: int,
+ end_idx: int,
+ is_front_pages: bool = False,
+) -> list:
+ """Extract paragraphs for a section (front pages or chapter body).
+
+ For front pages: collect p3, p19, p15 lines as paragraphs.
+ For chapters: collect body paragraphs (p6, p38, p44, p45, p46, p47).
+ Always skip p43, p24 (empty/break), p11 (bookmark), p5 (duplicate heading).
+ """
+ result = []
+
+ for i in range(start_idx, end_idx):
+ p = all_paragraphs[i]
+ cls = get_css_class(p)
+
+ # Skip empty paragraphs
+ if is_empty(p):
+ continue
+
+ # Skip known empty/break classes
+ if cls in SKIP_CLASSES:
+ continue
+
+ # Skip chapter bookmark headers (p11) — used for boundary detection only
+ if cls == CHAPTER_BOOKMARK_CLASS:
+ continue
+
+ # Skip chapter title duplicate (p5)
+ if cls == CHAPTER_TITLE_CLASS:
+ continue
+
+ # Skip p2 (TOC entry at start)
+ if cls == "p2":
+ continue
+
+ if is_front_pages:
+ # For front pages, collect title, author/address, and date lines
+ if cls in (FRONT_TITLE_CLASS, FRONT_INFO_CLASS, FRONT_DATE_CLASS):
+ html = decode_html_content(p)
+ if html:
+ result.append(html)
+ continue
+ # Also include any other non-skip class if we encounter it
+ # (shouldn't happen much for front pages, but be permissive)
+ if cls in BODY_CLASSES:
+ # Body paragraph before first chapter = still front pages
+ html = decode_html_content(p)
+ if html:
+ result.append(html)
+ continue
+ else:
+ # For chapter body, collect all body paragraph classes
+ if cls in BODY_CLASSES:
+ html = decode_html_content(p)
+ if html:
+ result.append(html)
+ continue
+
+ # p19 can appear inside chapters (e.g., "(Devanāgarī)" in chapter 2)
+ if cls == FRONT_INFO_CLASS:
+ html = decode_html_content(p)
+ if html:
+ result.append(html)
+ continue
+
+ # Title page class inside chapter body (shouldn't happen, but be safe)
+ if cls == FRONT_TITLE_CLASS:
+ html = decode_html_content(p)
+ if html:
+ result.append(html)
+ continue
+
+ # Fallback: try to extract content from unknown classes
+ html = decode_html_content(p)
+ if html:
+ result.append(html)
+
+ return result
+
+
+# ---------------------------------------------------------------------------
+# Main extraction logic
+# ---------------------------------------------------------------------------
+
+
+def extract_mog(soup: BeautifulSoup) -> dict:
+ """Extract the entire Message of Godhead from the parsed HTML.
+
+ Returns a dict with:
+ "intro_segments": list of intro segment dicts
+ "chapters": list of chapter dicts
+ "chapter_listing": list of {number, title} for intro.json
+ """
+ all_paragraphs = list(soup.find_all("p"))
+
+ # Step 1: Find MOG boundaries
+ mog_start, mog_end = find_mog_boundaries(all_paragraphs)
+ print(f" MOG content: paragraphs {mog_start} to {mog_end}")
+ print(f" Total paragraphs in range: {mog_end - mog_start}")
+
+ # Step 2: Find all chapter bookmarks (p11)
+ bookmarks = find_chapter_bookmarks(all_paragraphs, mog_start, mog_end)
+ print(f" Found {len(bookmarks)} MOG chapter bookmarks:")
+ for pos, title in bookmarks:
+ print(f" Paragraph {pos}: {title}")
+
+ # Step 3: Map bookmarks to chapters with proper titles
+ # Expected: 3 chapters
+ # Map raw bookmark text (case-insensitive) to normalized titles
+ chapter_titles = {
+ "message of godhead": "Message of Godhead (Introductory)",
+ "knowledge transcendental": "Knowledge Transcendental",
+ "work with transcendental effect": "Work with Transcendental Effect (Karma Yoga)",
+ }
+
+ chapter_bookmarks = [] # (para_idx, chapter_num, normalized_title)
+ ch_num = 1
+ for pos, raw_title in bookmarks:
+ # Try to match known chapter titles (case-insensitive)
+ matched_title = None
+ raw_lower = raw_title.lower().strip()
+ for key, nice_title in chapter_titles.items():
+ if key in raw_lower:
+ matched_title = nice_title
+ break
+
+ if matched_title is None:
+ # Fallback: use the raw title with title case
+ matched_title = raw_title.strip()
+
+ chapter_bookmarks.append((pos, ch_num, matched_title))
+ ch_num += 1
+
+ print(f"\n Mapped {len(chapter_bookmarks)} chapters:")
+ for pos, num, title in chapter_bookmarks:
+ print(f" Chapter {num}: {title} (paragraph {pos})")
+
+ # Step 4: Extract front pages
+ # From MOG start to first chapter bookmark
+ intro_segments = []
+
+ if chapter_bookmarks:
+ front_pages_end = chapter_bookmarks[0][0]
+ else:
+ front_pages_end = mog_end
+
+ front_pages_paragraphs = extract_paragraphs(
+ all_paragraphs, mog_start, front_pages_end, is_front_pages=True
+ )
+ intro_segments.append(
+ {
+ "slug": "front-pages",
+ "title": "Front Pages",
+ "type": "intro-segment",
+ "book": "mog",
+ "order": 1,
+ "paragraphs": front_pages_paragraphs,
+ }
+ )
+
+ # Step 5: Extract chapters
+ chapters = []
+ for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_bookmarks):
+ # Content starts after the bookmark line
+ # But we also need to skip the p5 duplicate heading that follows p11
+ content_start = ch_pos + 1
+
+ # Content ends at next chapter bookmark or mog_end
+ if idx + 1 < len(chapter_bookmarks):
+ content_end = chapter_bookmarks[idx + 1][0]
+ else:
+ content_end = mog_end
+
+ paragraphs = extract_paragraphs(all_paragraphs, content_start, content_end)
+
+ ch = {
+ "book": "mog",
+ "chapter": ch_num,
+ "title": ch_title,
+ "type": "chapter",
+ "paragraphs": paragraphs,
+ }
+ chapters.append(ch)
+
+ # Step 6: Build chapter listing for intro.json
+ chapter_listing = [
+ {"number": ch["chapter"], "title": ch["title"]} for ch in chapters
+ ]
+
+ return {
+ "intro_segments": intro_segments,
+ "chapters": chapters,
+ "chapter_listing": chapter_listing,
+ }
+
+
+# ---------------------------------------------------------------------------
+# Output
+# ---------------------------------------------------------------------------
+
+
+def write_json(data, filepath):
+ """Write data as formatted JSON to filepath, creating directories as needed."""
+ os.makedirs(os.path.dirname(filepath), exist_ok=True)
+ with open(filepath, "w", encoding="utf-8") as f:
+ json.dump(data, f, ensure_ascii=False, indent=2)
+
+
+def main():
+ base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+ html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html")
+ output_base = os.path.join(base_dir, "lib", "jsons", "books", "mog")
+
+ if not os.path.exists(html_path):
+ print(f"ERROR: {html_path} not found!")
+ sys.exit(1)
+
+ print(f"Reading {html_path}...")
+ with open(html_path, "r", encoding="utf-8") as f:
+ soup = BeautifulSoup(f, "html.parser")
+
+ print("Extracting Message of Godhead (1955)...\n")
+ result = extract_mog(soup)
+
+ intro_segments = result["intro_segments"]
+ chapters = result["chapters"]
+ chapter_listing = result["chapter_listing"]
+
+ # --- Write intro.json ---
+ intro_meta = {
+ "book": "mog",
+ "title": "Message of Godhead",
+ "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
+ "year": 1955,
+ "description": "An early handwritten work presenting the essence of Bhagavad-gītā's teachings on transcendental knowledge, karma-yoga, and the Personality of Godhead.",
+ "totalChapters": 3,
+ "chapters": chapter_listing,
+ }
+ intro_path = os.path.join(output_base, "intro.json")
+ write_json(intro_meta, intro_path)
+ print(f"\n Written: {intro_path}")
+
+ # --- Write intro segments ---
+ print(f"\n--- Intro Segments ---")
+ for seg in intro_segments:
+ filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
+ write_json(seg, filepath)
+ print(
+ f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}"
+ )
+
+ # --- Write chapters ---
+ print(f"\n--- Chapters ---")
+ for ch in chapters:
+ ch_num_str = str(ch["chapter"]).zfill(2)
+ filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
+ write_json(ch, filepath)
+ print(
+ f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
+ f"{ch['title']}"
+ )
+
+ # --- Summary ---
+ total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments)
+ total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
+ total_paras = total_intro_paras + total_chapter_paras
+
+ print(f"\n{'=' * 60}")
+ print(f" EXTRACTION SUMMARY")
+ print(f"{'=' * 60}")
+ print(f" Intro segments: {len(intro_segments)}")
+ for seg in intro_segments:
+ print(
+ f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs"
+ )
+ print(f" Chapters: {len(chapters)}")
+ for ch in chapters:
+ print(
+ f" Ch {ch['chapter']}: {len(ch['paragraphs'])} paragraphs — {ch['title']}"
+ )
+ print(f" Total paragraphs: {total_paras}")
+ print(f" Intro: {total_intro_paras}")
+ print(f" Chapters: {total_chapter_paras}")
+ if chapters:
+ print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
+ print(f" Output directory: {output_base}/")
+
+ # --- Samples ---
+ print(f"\n--- Sample: Front Pages ---")
+ if intro_segments:
+ front = intro_segments[0]
+ for i, p in enumerate(front["paragraphs"][:5]):
+ preview = re.sub(r"<[^>]+>", "", p)[:120]
+ print(f" [{i}] {preview}")
+
+ if chapters:
+ print(f"\n--- Sample: Chapter 1 ({chapters[0]['title']}) ---")
+ ch1 = chapters[0]
+ for i, p in enumerate(ch1["paragraphs"][:3]):
+ preview = re.sub(r"<[^>]+>", "", p)[:120]
+ print(f" [{i}] {preview}...")
+
+ if len(chapters) >= 2:
+ print(f"\n--- Sample: Chapter 2 ({chapters[1]['title']}) ---")
+ ch2 = chapters[1]
+ for i, p in enumerate(ch2["paragraphs"][:3]):
+ preview = re.sub(r"<[^>]+>", "", p)[:120]
+ print(f" [{i}] {preview}...")
+
+ if len(chapters) >= 3:
+ print(f"\n--- Sample: Chapter 3 ({chapters[2]['title']}) ---")
+ ch3 = chapters[2]
+ for i, p in enumerate(ch3["paragraphs"][:3]):
+ preview = re.sub(r"<[^>]+>", "", p)[:120]
+ print(f" [{i}] {preview}...")
+
+ # --- Quality Checks ---
+ print(f"\n--- Quality Checks ---")
+
+ # Check all 3 chapters present
+ expected_chapters = set(range(1, 4))
+ actual_chapters = set(ch["chapter"] for ch in chapters)
+ missing = expected_chapters - actual_chapters
+ extra = actual_chapters - expected_chapters
+ if missing:
+ print(f" ⚠ Missing chapters: {sorted(missing)}")
+ if extra:
+ print(f" ⚠ Extra chapters: {sorted(extra)}")
+ if not missing and not extra:
+ print(f" ✅ All 3 chapters present")
+
+ # Check intro segments
+ if intro_segments:
+ print(
+ f" ✅ Front pages segment present ({len(intro_segments[0]['paragraphs'])} paragraphs)"
+ )
+ else:
+ print(f" ⚠ Front pages segment missing!")
+
+ # Check empty chapters
+ empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
+ if empty_chapters:
+ print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
+ else:
+ print(f" ✅ No empty chapters")
+
+ # Check for "——The Author." in chapter 1
+ if chapters:
+ ch1_text = " ".join(chapters[0]["paragraphs"])
+ if "The Author" in ch1_text:
+ print(f" ✅ '——The Author.' attribution found in Chapter 1")
+ else:
+ print(f" ⚠ '——The Author.' attribution NOT found in Chapter 1")
+
+ # Check chapter 2 has "(Devanāgarī)" line
+ if len(chapters) >= 2:
+ ch2_text = " ".join(chapters[1]["paragraphs"])
+ if "Devan" in ch2_text:
+ print(f" ✅ '(Devanāgarī)' line found in Chapter 2")
+ else:
+ print(f" ℹ '(Devanāgarī)' line not found in Chapter 2 (may be normal)")
+
+ # Verify no Bhakti-rasāmṛta-sindhu content leaked in
+ all_text = ""
+ for ch in chapters:
+ all_text += " ".join(ch["paragraphs"])
+ if "Text 22" in all_text or "BHAKTI RASAMRITA" in all_text.upper():
+ print(f" ⚠ Content from Bhakti-rasāmṛta-sindhu may have leaked into MOG!")
+ else:
+ print(f" ✅ No content leak from next work detected")
+
+ # Check last chapter ends cleanly
+ if chapters:
+ last_ch = chapters[-1]
+ last_para = last_ch["paragraphs"][-1] if last_ch["paragraphs"] else ""
+ last_preview = re.sub(r"<[^>]+>", "", last_para)[:80]
+ print(f' ℹ Last paragraph of Ch {last_ch["chapter"]}: "{last_preview}..."')
+
+ print(f"\nDone! ✨")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/od_analysis_bg_extract.py b/scripts/od_analysis_bg_extract.py
new file mode 100644
index 00000000..07f76443
--- /dev/null
+++ b/scripts/od_analysis_bg_extract.py
@@ -0,0 +1,303 @@
+#!/usr/bin/env python3
+"""Extract "An Analysis of Bhagwat Geeta" from others-before-1977.html.
+
+This document is a structured analysis organized into three numbered sections
+(topics) with numbered points and verse references. It is plain English text
+(no ScaGoudy decoding needed).
+
+Source: public/html/others-before-1977.html
+ - Title: An Analysis of Bhagwat Geeta items
+ - Section 2: "Man's Duty Toward Godhead" — 34 items
+ - Section 3: "Man's ultimate necessity and perfection of life" — 3 items
+ - End boundary: Solution of Present Crisises... (line ~15276)
+
+CSS classes:
+ p116 — Title (bold)
+ p39 — Section headers ("1: Mans eternal relation with Godhead")
+ p5 — Duplicate section title (bold, with ) — SKIP
+ p6 — Body paragraphs (plain English)
+ p34 — Math calculation at the end — SKIP
+ p11 — Next document title (end boundary)
+
+Output (prose book structure):
+ lib/jsons/other-documents/analysis-of-bhagavad-gita/
+ intro.json # metadata
+ chapters/
+ 01.json # Chapter 1
+ 02.json # Chapter 2
+ 03.json # Chapter 3
+
+Usage:
+ python3 scripts/od_analysis_bg_extract.py
+"""
+
+import json
+import os
+import re
+import sys
+
+from bs4 import BeautifulSoup, Tag
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+
+BOOK_CODE = "od-analysis-bg"
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def get_css_class(tag: Tag) -> str:
+ """Get the first CSS class of a tag, or empty string."""
+ classes = tag.get("class", [])
+ return classes[0] if classes else ""
+
+
+def is_empty(tag: Tag) -> bool:
+ """Check if a tag is empty or whitespace-only."""
+ text = tag.get_text().strip()
+ return not text or text == "\xa0"
+
+
+def clean_text(tag: Tag) -> str:
+ """Extract inner HTML from a tag, strip span class artifacts, normalize whitespace."""
+ inner = "".join(str(child) for child in tag.children)
+ # Remove span tags with CSS classes (RTF conversion artifacts) but keep their content
+ inner = re.sub(
+ r' An Analysis of Bhagwat Geeta tags after the title until we hit the end boundary
+ chapters = []
+ current_chapter = None
+
+ tag = title_tag.find_next_sibling()
+ while tag:
+ if not isinstance(tag, Tag) or tag.name != "p":
+ tag = tag.find_next_sibling()
+ continue
+
+ css = get_css_class(tag)
+ text = tag.get_text().strip()
+
+ # End boundary: next document starts
+ if css == "p11" and "Solution of Present Crisises" in text:
+ break
+
+ # Skip empty tags
+ if is_empty(tag):
+ tag = tag.find_next_sibling()
+ continue
+
+ # Section header: 1: Mans eternal relation with Godhead
+ if css == "p6" and current_chapter is not None:
+ cleaned = clean_text(tag)
+ if cleaned:
+ current_chapter["paragraphs"].append(cleaned)
+ tag = tag.find_next_sibling()
+ continue
+
+ # Any other class — skip
+ tag = tag.find_next_sibling()
+
+ return chapters
+
+
+# ---------------------------------------------------------------------------
+# Quality check
+# ---------------------------------------------------------------------------
+
+
+def quality_check(chapters: list[dict]):
+ """Validate the extracted chapters."""
+ print("\n🔍 Quality check:\n")
+
+ print(f" Chapters: {len(chapters)}")
+ assert len(chapters) == 3, f"Expected 3 chapters, got {len(chapters)}"
+
+ total_paragraphs = 0
+ for ch in chapters:
+ count = len(ch["paragraphs"])
+ total_paragraphs += count
+ print(f' Chapter {ch["number"]}: "{ch["title"]}" — {count} paragraphs')
+
+ # Spot-check first paragraph
+ if ch["paragraphs"]:
+ first = ch["paragraphs"][0]
+ print(f" First: {first[:80]}...")
+
+ # Spot-check last paragraph
+ if len(ch["paragraphs"]) > 1:
+ last = ch["paragraphs"][-1]
+ print(f" Last: {last[:80]}...")
+
+ print(f"\n Total paragraphs: {total_paragraphs}")
+
+ # Expected counts (from manual inspection):
+ # Section 1: 45 items (numbered 1-45)
+ # Section 2: 34 items (numbered 1-34)
+ # Section 3: 3 items (numbered 1-3)
+ expected_total = 82 # 45 + 34 + 3
+ if total_paragraphs == expected_total:
+ print(f" ✅ Paragraph count matches expected: {expected_total}")
+ else:
+ print(f" ⚠️ Expected {expected_total} paragraphs, got {total_paragraphs}")
+ # This is a warning, not a hard failure — some items may have sub-items
+ # or the HTML structure may slightly differ
+
+ # Check that paragraphs are not empty
+ for ch in chapters:
+ for i, p in enumerate(ch["paragraphs"]):
+ assert p.strip(), f"Empty paragraph in chapter {ch['number']}, index {i}"
+
+ print("\n ✅ All checks passed!")
+
+
+# ---------------------------------------------------------------------------
+# Output
+# ---------------------------------------------------------------------------
+
+
+def write_output(chapters: list[dict], output_dir: str):
+ """Write intro.json and per-chapter JSON files."""
+
+ # Build intro.json
+ intro = {
+ "book": BOOK_CODE,
+ "title": "An Analysis of Bhagwat Geeta",
+ "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
+ "year": None,
+ "description": "A structured analysis of Bhagavad-gītā organized into three topics with numbered points and verse references.",
+ "totalChapters": len(chapters),
+ "chapters": [{"number": ch["number"], "title": ch["title"]} for ch in chapters],
+ }
+
+ write_json(intro, os.path.join(output_dir, "intro.json"))
+
+ # Write per-chapter files
+ chapters_dir = os.path.join(output_dir, "chapters")
+ for ch in chapters:
+ chapter_data = {
+ "book": BOOK_CODE,
+ "chapter": ch["number"],
+ "title": ch["title"],
+ "type": "chapter",
+ "paragraphs": ch["paragraphs"],
+ }
+ filename = f"{ch['number']:02d}.json"
+ write_json(chapter_data, os.path.join(chapters_dir, filename))
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+
+def main():
+ base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+ html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html")
+ output_dir = os.path.join(
+ base_dir, "lib", "jsons", "other-documents", "analysis-of-bhagavad-gita"
+ )
+
+ if not os.path.exists(html_path):
+ print(f"❌ HTML file not found: {html_path}")
+ sys.exit(1)
+
+ # Clean up old single-file JSON if it exists
+ old_json = os.path.join(
+ base_dir, "lib", "jsons", "other-documents", "analysis-of-bhagavad-gita.json"
+ )
+ if os.path.exists(old_json):
+ os.remove(old_json)
+ print(f" 🗑️ Removed old single-file JSON: {old_json}")
+
+ print(f"📂 Reading {html_path}...")
+ with open(html_path, "r", encoding="utf-8") as f:
+ soup = BeautifulSoup(f, "html.parser")
+
+ print("\n📝 Extracting 'An Analysis of Bhagwat Geeta'...\n")
+ chapters = extract_chapters(soup)
+
+ quality_check(chapters)
+
+ write_output(chapters, output_dir)
+ print(f"\n🎉 Done! Output directory: {output_dir}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/pk_extract.py b/scripts/pk_extract.py
new file mode 100644
index 00000000..4c3dd67f
--- /dev/null
+++ b/scripts/pk_extract.py
@@ -0,0 +1,533 @@
+#!/usr/bin/env python3
+"""Extract Prayers of King Kulaśekhara (1958) from others-before-1977.html.
+
+The book has 6 chapters (one per verse/prayer):
+ 1. Prayer 1 — "Śrī Vallabha" (prayer to Mukunda)
+ 2. Prayer 2 — "Jayatu Jayatu" (glories to Devakī-nandana)
+ 3. Prayer 3 — "Mukunda Mūrdhnā" (prayer for remembrance)
+ 4. Prayer 4 — "Nāhaṁ Vande" (neither heaven nor hell)
+ 5. Prayer 5 — "Nāsthā Dharme" (unflinching devotion)
+ 6. Prayer 6 — "Divi Vā Bhuvi" (remembrance at death)
+
+Output:
+ lib/jsons/books/pk/intro.json — Book metadata
+ lib/jsons/books/pk/chapters/*.json — Chapters (01–06)
+
+Usage:
+ python3 scripts/pk_extract.py
+"""
+
+import json
+import os
+import re
+import sys
+
+from bs4 import BeautifulSoup, Tag
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def decode_html_content(tag: Tag) -> str:
+ """Extract inner HTML from a tag, preserve formatting tags, strip spans."""
+ inner = "".join(str(child) for child in tag.children)
+ # Remove span tags with CSS classes (RTF conversion artifacts)
+ inner = re.sub(
+ r' containing "Prayers of King Kula"
+ End: that starts the next work ("Shaktipujah")
+ """
+ start_idx = None
+ end_idx = len(all_elements)
+
+ for i, el in enumerate(all_elements):
+ if el.name != "p":
+ continue
+ cls = get_css_class(el)
+ text = decode_text(el)
+
+ # Find start: p4 containing "Prayers of King"
+ if start_idx is None:
+ if cls == DOC_HEADER_CLASS and "Prayers of King" in text:
+ start_idx = i
+ continue
+ continue
+
+ # Find end: p125 that starts a different work
+ if cls == NEXT_WORK_BOUNDARY_CLASS:
+ end_idx = i
+ break
+
+ if start_idx is None:
+ print(
+ "ERROR: Could not find 'Prayers of King Kulaçekhara' in others-before-1977.html!"
+ )
+ sys.exit(1)
+
+ return start_idx, end_idx
+
+
+def find_chapter_bookmarks(all_elements: list, start_idx: int, end_idx: int) -> list:
+ """Find all PK chapter bookmark tags (p17) in the PK range.
+
+ Returns list of (element_index, verse_number).
+ """
+ bookmarks = []
+ for i in range(start_idx, end_idx):
+ el = all_elements[i]
+ if el.name != "p":
+ continue
+ cls = get_css_class(el)
+ if cls == CHAPTER_BOOKMARK_CLASS:
+ text = decode_text(el).strip()
+ if "Prayers of King" in text or "Kulaçekhara" in text.replace("ç", "ç"):
+ # Extract the verse number from the end
+ match = re.search(r":\s*(\d+)\s*$", text)
+ if match:
+ verse_num = int(match.group(1))
+ bookmarks.append((i, verse_num))
+ return bookmarks
+
+
+# ---------------------------------------------------------------------------
+# Table synonym extraction
+# ---------------------------------------------------------------------------
+
+
+def extract_synonyms_from_table(table: Tag) -> str:
+ """Extract synonyms from a with "word=meaning" format.
+ Convert to "word—meaning; word—meaning; ..." format.
+ """
+ entries = []
+ for row in table.find_all("tr"):
+ for cell in row.find_all("td"):
+ # Get all tags inside the cell
+ for p in cell.find_all("p"):
+ text = decode_text(p).strip()
+ if not text or text == "\xa0":
+ continue
+ # Must contain '=' to be a synonym entry
+ if "=" not in text:
+ continue
+ # Replace = with em-dash
+ text = text.replace("=", "—", 1)
+ # Clean trailing whitespace/period if needed
+ text = text.strip()
+ entries.append(text)
+
+ if not entries:
+ return ""
+ return "; ".join(entries)
+
+
+# ---------------------------------------------------------------------------
+# Paragraph extraction for a chapter
+# ---------------------------------------------------------------------------
+
+
+def extract_chapter_paragraphs(
+ all_elements: list,
+ start_idx: int,
+ end_idx: int,
+) -> list:
+ """Extract paragraphs for a chapter (verse).
+
+ Handles section headers (Wordings., Synonyms., Translation., Purport.),
+ body paragraphs, and synonym tables.
+
+ Returns a list of HTML paragraph strings.
+ """
+ result = []
+
+ # Build a set of tags that are inside tables, so we don't double-process
+ table_p_tags = set()
+ for i in range(start_idx, end_idx):
+ el = all_elements[i]
+ if el.name == "table":
+ for p in el.find_all("p"):
+ table_p_tags.add(id(p))
+
+ for i in range(start_idx, end_idx):
+ el = all_elements[i]
+
+ # Handle tables (synonyms)
+ if el.name == "table":
+ table_cls = get_css_class(el)
+ if table_cls == "t1":
+ synonyms_text = extract_synonyms_from_table(el)
+ if synonyms_text:
+ result.append(synonyms_text)
+ continue
+
+ # Only process tags from here
+ if el.name != "p":
+ continue
+
+ # Skip tags that are inside tables (already processed)
+ if id(el) in table_p_tags:
+ continue
+
+ cls = get_css_class(el)
+
+ # Skip empty paragraphs
+ if is_empty(el):
+ continue
+
+ # Skip page numbers
+ if cls in PAGE_NUMBER_CLASSES:
+ continue
+
+ # Skip verse markers
+ if cls in VERSE_MARKER_CLASSES:
+ text = decode_text(el)
+ # p9 can also be "(Devanägaré)" or prayer number markers — skip all
+ continue
+
+ # Skip devanagari markers
+ if cls == DEVANAGARI_MARKER_CLASS:
+ continue
+
+ # Skip chapter bookmarks
+ if cls == CHAPTER_BOOKMARK_CLASS:
+ continue
+
+ # Skip document header and title (only at very start)
+ if cls == DOC_HEADER_CLASS:
+ continue
+ if cls == TITLE_CLASS:
+ continue
+
+ # Skip empty/break classes
+ if cls in SKIP_CLASSES:
+ continue
+
+ # Section headers: Wordings., Synonyms., Translation., Purport.
+ if cls in SECTION_HEADER_CLASSES:
+ text = decode_text(el).strip()
+ # Normalize section header names
+ text_lower = text.lower().rstrip(".")
+ if "wording" in text_lower:
+ result.append("Wordings")
+ elif (
+ "synonym" in text_lower
+ or "synomym" in text_lower
+ or "synonysm" in text_lower
+ ):
+ result.append("Synonyms")
+ elif "translation" in text_lower:
+ result.append("Translation")
+ elif "purport" in text_lower:
+ result.append("Purport")
+ else:
+ # Unknown header — include as bold
+ html = decode_html_content(el)
+ if html:
+ result.append(f"{html}")
+ continue
+
+ # Body paragraphs
+ if cls in BODY_CLASSES:
+ html = decode_html_content(el)
+ if html:
+ result.append(html)
+ continue
+
+ # Fallback: try to extract content from unknown classes
+ html = decode_html_content(el)
+ if html:
+ result.append(html)
+
+ return result
+
+
+# ---------------------------------------------------------------------------
+# Main extraction logic
+# ---------------------------------------------------------------------------
+
+
+def extract_pk(soup: BeautifulSoup) -> dict:
+ """Extract the entire Prayers of King Kulaśekhara from the parsed HTML.
+
+ Returns a dict with:
+ "chapters": list of chapter dicts
+ "chapter_listing": list of {number, title} for intro.json
+ """
+ # We need both and tags total")
+
+ # Find section boundaries
+ sections = find_section_boundaries(all_tags)
+ print(
+ f" The Beginning: tags[{sections['beg_start']}] → tags[{sections['beg_end']}]"
+ )
+ for key, val in sorted(sections.items()):
+ print(f" {key}: {val}")
+
+ # ---------------------------------------------------------------------------
+ # Extract Introduction
+ # ---------------------------------------------------------------------------
+ intro_start = sections["introduction_start"]
+ intro_end = sections["ch1_start"]
+ intro_paragraphs = extract_prose_section(all_tags, intro_start, intro_end)
+ print(f"\n Introduction: {len(intro_paragraphs)} paragraphs")
+
+ # ---------------------------------------------------------------------------
+ # Extract Chapter 1: The Upper West Side
+ # ---------------------------------------------------------------------------
+ ch1_start = sections["ch1_start"]
+ ch1_end = sections["ch2_start"]
+ ch1_summary, ch1_entries = extract_day_entries(all_tags, ch1_start, ch1_end)
+ print(
+ f" Ch 1 (Upper West Side): {len(ch1_summary)} summary paras, {len(ch1_entries)} day entries"
+ )
+
+ # ---------------------------------------------------------------------------
+ # Extract Chapter 2: The Bowery
+ # ---------------------------------------------------------------------------
+ ch2_start = sections["ch2_start"]
+ ch2_end = sections["ch3_start"]
+ ch2_summary, ch2_entries = extract_day_entries(all_tags, ch2_start, ch2_end)
+ print(
+ f" Ch 2 (The Bowery): {len(ch2_summary)} summary paras, {len(ch2_entries)} day entries"
+ )
+
+ # ---------------------------------------------------------------------------
+ # Extract Chapter 3: 26 Second Avenue
+ # ---------------------------------------------------------------------------
+ ch3_start = sections["ch3_start"]
+ ch3_end = sections["ch4_start"]
+ ch3_summary, ch3_entries = extract_day_entries(all_tags, ch3_start, ch3_end)
+ print(
+ f" Ch 3 (26 Second Ave): {len(ch3_summary)} summary paras, {len(ch3_entries)} day entries"
+ )
+
+ # ---------------------------------------------------------------------------
+ # Extract Memoranda and Accounts
+ # ---------------------------------------------------------------------------
+ ch4_start = sections["ch4_start"]
+ ch4_end = sections["ch5_start"]
+ memo_paragraphs = extract_prose_section(all_tags, ch4_start, ch4_end)
+ print(f" Memoranda: {len(memo_paragraphs)} paragraphs")
+
+ # ---------------------------------------------------------------------------
+ # Extract Epilogue
+ # ---------------------------------------------------------------------------
+ ch5_start = sections["ch5_start"]
+ ch5_end = sections["ch6_start"]
+ epilogue_paragraphs = extract_prose_section(
+ all_tags,
+ ch5_start,
+ ch5_end,
+ skip_classes={"p4", "p11", "p3", "p5", "p153", "p24"},
+ )
+ print(f" Epilogue: {len(epilogue_paragraphs)} paragraphs")
+
+ # ---------------------------------------------------------------------------
+ # Extract Excerpts
+ # ---------------------------------------------------------------------------
+ ch6_start = sections["ch6_start"]
+ ch6_end = sections["beg_end"]
+ excerpt_preamble, excerpt_list = extract_excerpts(all_tags, ch6_start, ch6_end)
+ print(
+ f" Excerpts: {len(excerpt_preamble)} preamble paras, {len(excerpt_list)} excerpts"
+ )
+ for ex in excerpt_list:
+ print(
+ f" - {ex['slug']}: {ex['title'][:60]}... ({len(ex['paragraphs'])} paras)"
+ )
+
+ # ---------------------------------------------------------------------------
+ # Group day entries by month within each chapter
+ # ---------------------------------------------------------------------------
+
+ def group_by_month(entries):
+ """Group day entries by month, preserving order."""
+ groups = [] # List of (month_abbr, month_full, entries_list)
+ current_month = None
+ current_group = None
+
+ for entry in entries:
+ m = entry.get("month")
+ if m and m != current_month:
+ current_month = m
+ current_group = {
+ "month": m,
+ "month_full": MONTH_FULL.get(m, m),
+ "entries": [],
+ }
+ groups.append(current_group)
+ if current_group is not None:
+ current_group["entries"].append(entry)
+ else:
+ # Entry without month (like "Notice") — create special group
+ current_group = {
+ "month": None,
+ "month_full": "Other",
+ "entries": [entry],
+ }
+ groups.append(current_group)
+ current_group = None
+ current_month = None
+
+ return groups
+
+ ch1_months = group_by_month(ch1_entries)
+ ch2_months = group_by_month(ch2_entries)
+ ch3_months = group_by_month(ch3_entries)
+
+ # ---------------------------------------------------------------------------
+ # Ensure unique slugs across all entries
+ # ---------------------------------------------------------------------------
+ all_slugs = set()
+
+ def ensure_unique_slug(slug):
+ if slug in all_slugs:
+ suffix = 2
+ while f"{slug}-{suffix}" in all_slugs:
+ suffix += 1
+ slug = f"{slug}-{suffix}"
+ all_slugs.add(slug)
+ return slug
+
+ # Reserve standalone slugs
+ for s in [
+ "introduction",
+ "upper-west-side-summary",
+ "bowery-summary",
+ "second-avenue-summary",
+ "memoranda",
+ "epilogue",
+ "excerpts-preamble",
+ ]:
+ all_slugs.add(s)
+
+ # Ensure unique slugs for day entries
+ for entries in [ch1_entries, ch2_entries, ch3_entries]:
+ for entry in entries:
+ entry["slug"] = ensure_unique_slug(entry["slug"])
+
+ # Ensure unique slugs for excerpts
+ for ex in excerpt_list:
+ ex["slug"] = ensure_unique_slug(ex["slug"])
+
+ # ---------------------------------------------------------------------------
+ # Build intro.json with hierarchical chapters
+ # ---------------------------------------------------------------------------
+
+ intro_chapters = []
+
+ # Introduction (standalone)
+ intro_chapters.append({"slug": "introduction", "title": "Introduction"})
+
+ # Chapter 1: The Upper West Side
+ ch1_sub = [{"slug": "upper-west-side-summary", "title": "Summary"}]
+ for mg in ch1_months:
+ ch1_sub.append(
+ {
+ "title": mg["month_full"],
+ "subChapters": [
+ {"slug": e["slug"], "title": e["title"]} for e in mg["entries"]
+ ],
+ }
+ )
+ intro_chapters.append(
+ {
+ "title": "The Upper West Side (Jan–Mar 1966)",
+ "subChapters": ch1_sub,
+ }
+ )
+
+ # Chapter 2: The Bowery
+ ch2_sub = [{"slug": "bowery-summary", "title": "Summary"}]
+ for mg in ch2_months:
+ ch2_sub.append(
+ {
+ "title": mg["month_full"],
+ "subChapters": [
+ {"slug": e["slug"], "title": e["title"]} for e in mg["entries"]
+ ],
+ }
+ )
+ intro_chapters.append(
+ {
+ "title": "The Bowery (Apr–Jun 1966)",
+ "subChapters": ch2_sub,
+ }
+ )
+
+ # Chapter 3: 26 Second Avenue
+ ch3_sub = [{"slug": "second-avenue-summary", "title": "Summary"}]
+ for mg in ch3_months:
+ if mg["month"] is not None:
+ ch3_sub.append(
+ {
+ "title": mg["month_full"],
+ "subChapters": [
+ {"slug": e["slug"], "title": e["title"]} for e in mg["entries"]
+ ],
+ }
+ )
+ else:
+ # Non-month entries like "Notice"
+ for e in mg["entries"]:
+ ch3_sub.append({"slug": e["slug"], "title": e["title"]})
+ intro_chapters.append(
+ {
+ "title": "26 Second Avenue (Jul–Nov 1966)",
+ "subChapters": ch3_sub,
+ }
+ )
+
+ # Memoranda
+ intro_chapters.append({"slug": "memoranda", "title": "Memoranda and Accounts"})
+
+ # Epilogue
+ intro_chapters.append({"slug": "epilogue", "title": "Epilogue"})
+
+ # Excerpts — group under a parent heading
+ excerpt_sub = []
+ if excerpt_preamble:
+ excerpt_sub.append({"slug": "excerpts-preamble", "title": "Preamble"})
+ for ex in excerpt_list:
+ # Shorten title for sidebar
+ short_title = ex["title"]
+ if len(short_title) > 80:
+ short_title = short_title[:77] + "..."
+ excerpt_sub.append({"slug": ex["slug"], "title": short_title})
+ intro_chapters.append(
+ {
+ "title": "Excerpts",
+ "subChapters": excerpt_sub,
+ }
+ )
+
+ # Count total chapters
+ total_chapters = 0
+ for ch in intro_chapters:
+ if "slug" in ch:
+ total_chapters += 1
+ if "subChapters" in ch:
+ for sc in ch["subChapters"]:
+ if "slug" in sc:
+ total_chapters += 1
+ if "subChapters" in sc:
+ total_chapters += len(sc["subChapters"])
+
+ intro_json = {
+ "book": "the-beginning",
+ "title": "The Beginning — The 1966 New York Journal",
+ "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
+ "year": 1966,
+ "description": "Śrīla Prabhupāda's personal journal documenting the founding of the Hare Kṛṣṇa movement in New York City, from January to November 1966.",
+ "totalChapters": total_chapters,
+ "chapters": intro_chapters,
+ }
+
+ # ---------------------------------------------------------------------------
+ # Write all JSON files
+ # ---------------------------------------------------------------------------
+
+ # Clean output directory
+ if os.path.exists(chapters_dir):
+ for f in os.listdir(chapters_dir):
+ os.remove(os.path.join(chapters_dir, f))
+
+ # intro.json
+ write_json(intro_json, os.path.join(output_dir, "intro.json"))
+ print(f"\n Written: intro.json ({total_chapters} total chapters)")
+
+ chapter_num = 1
+ files_written = 0
+
+ # Introduction
+ write_json(
+ {
+ "book": "the-beginning",
+ "chapter": chapter_num,
+ "title": "Introduction",
+ "type": "introduction",
+ "slug": "introduction",
+ "paragraphs": intro_paragraphs,
+ },
+ os.path.join(chapters_dir, "introduction.json"),
+ )
+ print(f" Written: introduction.json ({len(intro_paragraphs)} paras)")
+ chapter_num += 1
+ files_written += 1
+
+ # Chapter 1 summary
+ write_json(
+ {
+ "book": "the-beginning",
+ "chapter": chapter_num,
+ "title": "The Upper West Side — Summary",
+ "type": "summary",
+ "slug": "upper-west-side-summary",
+ "paragraphs": ch1_summary,
+ },
+ os.path.join(chapters_dir, "upper-west-side-summary.json"),
+ )
+ print(f" Written: upper-west-side-summary.json ({len(ch1_summary)} paras)")
+ chapter_num += 1
+ files_written += 1
+
+ # Chapter 1 day entries
+ for entry in ch1_entries:
+ write_json(
+ {
+ "book": "the-beginning",
+ "chapter": chapter_num,
+ "title": entry["title"],
+ "type": "chapter",
+ "slug": entry["slug"],
+ "paragraphs": entry["paragraphs"],
+ },
+ os.path.join(chapters_dir, f"{entry['slug']}.json"),
+ )
+ chapter_num += 1
+ files_written += 1
+
+ print(f" Written: {len(ch1_entries)} Chapter 1 day entry files")
+
+ # Chapter 2 summary
+ write_json(
+ {
+ "book": "the-beginning",
+ "chapter": chapter_num,
+ "title": "The Bowery — Summary",
+ "type": "summary",
+ "slug": "bowery-summary",
+ "paragraphs": ch2_summary,
+ },
+ os.path.join(chapters_dir, "bowery-summary.json"),
+ )
+ print(f" Written: bowery-summary.json ({len(ch2_summary)} paras)")
+ chapter_num += 1
+ files_written += 1
+
+ # Chapter 2 day entries
+ for entry in ch2_entries:
+ write_json(
+ {
+ "book": "the-beginning",
+ "chapter": chapter_num,
+ "title": entry["title"],
+ "type": "chapter",
+ "slug": entry["slug"],
+ "paragraphs": entry["paragraphs"],
+ },
+ os.path.join(chapters_dir, f"{entry['slug']}.json"),
+ )
+ chapter_num += 1
+ files_written += 1
+
+ print(f" Written: {len(ch2_entries)} Chapter 2 day entry files")
+
+ # Chapter 3 summary
+ write_json(
+ {
+ "book": "the-beginning",
+ "chapter": chapter_num,
+ "title": "26 Second Avenue — Summary",
+ "type": "summary",
+ "slug": "second-avenue-summary",
+ "paragraphs": ch3_summary,
+ },
+ os.path.join(chapters_dir, "second-avenue-summary.json"),
+ )
+ print(f" Written: second-avenue-summary.json ({len(ch3_summary)} paras)")
+ chapter_num += 1
+ files_written += 1
+
+ # Chapter 3 day entries
+ for entry in ch3_entries:
+ write_json(
+ {
+ "book": "the-beginning",
+ "chapter": chapter_num,
+ "title": entry["title"],
+ "type": "chapter",
+ "slug": entry["slug"],
+ "paragraphs": entry["paragraphs"],
+ },
+ os.path.join(chapters_dir, f"{entry['slug']}.json"),
+ )
+ chapter_num += 1
+ files_written += 1
+
+ print(f" Written: {len(ch3_entries)} Chapter 3 day entry files")
+
+ # Memoranda
+ write_json(
+ {
+ "book": "the-beginning",
+ "chapter": chapter_num,
+ "title": "Memoranda and Accounts",
+ "type": "chapter",
+ "slug": "memoranda",
+ "paragraphs": memo_paragraphs,
+ },
+ os.path.join(chapters_dir, "memoranda.json"),
+ )
+ print(f" Written: memoranda.json ({len(memo_paragraphs)} paras)")
+ chapter_num += 1
+ files_written += 1
+
+ # Epilogue
+ write_json(
+ {
+ "book": "the-beginning",
+ "chapter": chapter_num,
+ "title": "Epilogue",
+ "type": "chapter",
+ "slug": "epilogue",
+ "paragraphs": epilogue_paragraphs,
+ },
+ os.path.join(chapters_dir, "epilogue.json"),
+ )
+ print(f" Written: epilogue.json ({len(epilogue_paragraphs)} paras)")
+ chapter_num += 1
+ files_written += 1
+
+ # Excerpts preamble
+ if excerpt_preamble:
+ write_json(
+ {
+ "book": "the-beginning",
+ "chapter": chapter_num,
+ "title": "Excerpts — Preamble",
+ "type": "chapter",
+ "slug": "excerpts-preamble",
+ "paragraphs": excerpt_preamble,
+ },
+ os.path.join(chapters_dir, "excerpts-preamble.json"),
+ )
+ print(f" Written: excerpts-preamble.json ({len(excerpt_preamble)} paras)")
+ chapter_num += 1
+ files_written += 1
+
+ # Excerpt entries
+ for ex in excerpt_list:
+ write_json(
+ {
+ "book": "the-beginning",
+ "chapter": chapter_num,
+ "title": ex["title"],
+ "type": "excerpt",
+ "slug": ex["slug"],
+ "paragraphs": ex["paragraphs"],
+ },
+ os.path.join(chapters_dir, f"{ex['slug']}.json"),
+ )
+ chapter_num += 1
+ files_written += 1
+
+ print(f" Written: {len(excerpt_list)} excerpt files")
+
+ # ---------------------------------------------------------------------------
+ # Summary
+ # ---------------------------------------------------------------------------
+ print(f"\n{'=' * 60}")
+ print(f" EXTRACTION SUMMARY")
+ print(f"{'=' * 60}")
+ print(f" Introduction: {len(intro_paragraphs)} paragraphs")
+ print(
+ f" Ch 1 (Upper West Side): {len(ch1_summary)} summary + {len(ch1_entries)} days"
+ )
+ for mg in ch1_months:
+ print(f" {mg['month_full']}: {len(mg['entries'])} entries")
+ print(f" Ch 2 (The Bowery): {len(ch2_summary)} summary + {len(ch2_entries)} days")
+ for mg in ch2_months:
+ print(f" {mg['month_full']}: {len(mg['entries'])} entries")
+ print(
+ f" Ch 3 (26 Second Ave): {len(ch3_summary)} summary + {len(ch3_entries)} days"
+ )
+ for mg in ch3_months:
+ mname = mg["month_full"] if mg["month"] else "Other"
+ print(f" {mname}: {len(mg['entries'])} entries")
+ print(f" Memoranda: {len(memo_paragraphs)} paragraphs")
+ print(f" Epilogue: {len(epilogue_paragraphs)} paragraphs")
+ print(f" Excerpts: {len(excerpt_list)} excerpts")
+ print(f" Total chapter files: {files_written}")
+ print(f" Total chapters (for intro.json): {total_chapters}")
+ print(f"{'=' * 60}")
+ print(f"\nDone! ✨")
+
+
+if __name__ == "__main__":
+ main()
and reference tags like [Bg. 7.14].
+ """
+ inner = "".join(str(child) for child in tag.children)
+ # Remove span tags with CSS classes — keep content
+ inner = re.sub(
+ r'
tags)
+ # Replace multiple spaces with single space
+ inner = re.sub(r"[ ]+", " ", inner).strip()
+ # Remove trailing
tags
+ inner = re.sub(r"({inner}'
+ return ""
+
+
+def decode_text(tag: Tag) -> str:
+ """Get plain text from a tag."""
+ return tag.get_text().strip()
+
+
+def get_css_class(tag: Tag) -> str:
+ """Get the first CSS class of a tag, or empty string."""
+ classes = tag.get("class", [])
+ return classes[0] if classes else ""
+
+
+def is_empty(tag: Tag) -> bool:
+ """Check if a tag is empty or whitespace-only."""
+ text = tag.get_text().strip()
+ return not text or text == "\xa0"
+
+
+# ---------------------------------------------------------------------------
+# CSS class categorization for BK
+# ---------------------------------------------------------------------------
+
+# Book title header
+BOOK_TITLE_CLASS = "p56"
+
+# Chapter bookmark headers (used for boundary detection)
+CHAPTER_BOOKMARK_CLASS = "p17"
+
+# Chapter title (bold) — skip from body
+CHAPTER_TITLE_CLASS = "p5"
+
+# Body paragraph classes (all need ScaGoudy decoding)
+BODY_CLASSES = frozenset(["p6", "p12", "p38"])
+
+# Sanskrit verse line classes (wrap as centered italic)
+VERSE_CLASSES = frozenset(["p9", "p19", "p59", "p23", "p36", "p60"])
+
+# Reference marker classes — include as paragraphs
+REFERENCE_CLASSES = frozenset(["p10", "p61"])
+
+# Footnote classes — include as paragraphs
+FOOTNOTE_CLASSES = frozenset(["p57", "p58"])
+
+# Classes to skip entirely
+SKIP_CLASSES = frozenset(["p43", "p24"])
+
+# TOC entry class
+TOC_CLASS = "p2"
+
+
+# ---------------------------------------------------------------------------
+# Boundary detection
+# ---------------------------------------------------------------------------
+
+
+def find_bk_boundaries(all_paragraphs: list) -> tuple:
+ """Find the start and end paragraph indices of the BK content.
+
+ Start:
."""
+ inner = "".join(str(child) for child in tag.children)
+ # Remove span tags with CSS classes (RTF conversion artifacts) but keep their content
+ inner = re.sub(
+ r'
tags)
+ inner = re.sub(r"\s+", " ", inner).strip()
+ return inner
+
+
+def write_json(data, filepath):
+ """Write data as formatted JSON to filepath, creating directories as needed."""
+ os.makedirs(os.path.dirname(filepath), exist_ok=True)
+ with open(filepath, "w", encoding="utf-8") as f:
+ json.dump(data, f, ensure_ascii=False, indent=2)
+
+
+def is_divider(tag: Tag) -> bool:
+ """Check if a p9 tag is just a divider line (underscores) or a page header."""
+ text = tag.get_text().strip()
+ # Dividers are lines of underscores
+ if re.match(r"^_+$", text):
+ return True
+ return False
+
+
+def is_page_header(tag: Tag) -> bool:
+ """Check if a p9 tag is a page header like 'Essays on Shri Chaitanya Charitamrita Page 30'."""
+ text = tag.get_text().strip()
+ if re.search(r"Page\s+\d+", text):
+ return True
+ return False
+
+
+def is_editor_note(tag: Tag) -> bool:
+ """Check if a paragraph is an editor note like '[This file has been proofread...]'."""
+ text = tag.get_text().strip()
+ if text.startswith("[") and (
+ "proofread" in text.lower() or "—Ed" in text or "- Ed" in text or ".—Ed" in text
+ ):
+ return True
+ return False
+
+
+# ---------------------------------------------------------------------------
+# Essay configurations
+# ---------------------------------------------------------------------------
+
+AUTHOR = "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda"
+
+ESSAYS = [
+ {
+ "slug": "the-spiritual-masters",
+ "title": "The Spiritual Masters",
+ "author": AUTHOR,
+ "year": 1960,
+ "group": "cc-essays",
+ "order": 1,
+ },
+ {
+ "slug": "the-absolute-truth",
+ "title": "The Absolute Truth",
+ "author": AUTHOR,
+ "year": 1957,
+ "group": "cc-essays",
+ "order": 2,
+ },
+ {
+ "slug": "causes-of-lord-caitanyas-incarnation",
+ "title": "The Causes of Lord Caitanya's Incarnation",
+ "author": AUTHOR,
+ "year": 1957,
+ "group": "cc-essays",
+ "order": 3,
+ },
+ {
+ "slug": "confidential-reasons-for-lord-caitanyas-appearance",
+ "title": "The Confidential Reasons for Lord Caitanya's Appearance",
+ "author": AUTHOR,
+ "year": 1957,
+ "group": "cc-essays",
+ "order": 4,
+ },
+ {
+ "slug": "glories-of-lord-nityananda",
+ "title": "The Glories of Lord Nityānanda",
+ "author": AUTHOR,
+ "year": 1957,
+ "group": "cc-essays",
+ "order": 5,
+ },
+]
+
+# ---------------------------------------------------------------------------
+# Extraction logic
+# ---------------------------------------------------------------------------
+
+
+def parse_all_paragraphs(html_path: str) -> list[Tag]:
+ """Parse the entire HTML and return all
)
+ html = re.sub(r"[ \t]+", " ", html)
+ html = html.strip()
+ # Remove leading
tags
+ html = re.sub(r"^(
tags
+ inner = re.sub(r" and convert to text.
+
+ Each cell contains a
elements in document order
+ all_elements = list(soup.find_all(["p", "table"]))
+
+ # Step 1: Find PK boundaries
+ pk_start, pk_end = find_pk_boundaries(all_elements)
+ print(f" PK content: elements {pk_start} to {pk_end}")
+ print(f" Total elements in range: {pk_end - pk_start}")
+
+ # Step 2: Find all chapter bookmarks (p17)
+ bookmarks = find_chapter_bookmarks(all_elements, pk_start, pk_end)
+ print(f" Found {len(bookmarks)} PK chapter bookmarks:")
+ for pos, verse_num in bookmarks:
+ print(f" Element {pos}: Verse {verse_num}")
+
+ # If we didn't find 6 bookmarks, also look for the first verse that starts
+ # before any p17 bookmark (verse 1 may start with p108 [1])
+ # Actually, verse 1's bookmark IS in the HTML. Let's verify.
+ if len(bookmarks) < 6:
+ print(f" WARNING: Expected 6 bookmarks, found {len(bookmarks)}")
+
+ # Step 3: Build chapter ranges
+ # Each bookmark marks the START of a verse section.
+ # Content for verse N runs from bookmark N to bookmark N+1 (or pk_end).
+ chapters = []
+ for idx, (bk_pos, verse_num) in enumerate(bookmarks):
+ # Content starts after the bookmark
+ content_start = bk_pos + 1
+
+ # Content ends at next bookmark or pk_end
+ if idx + 1 < len(bookmarks):
+ content_end = bookmarks[idx + 1][0]
+ else:
+ content_end = pk_end
+
+ paragraphs = extract_chapter_paragraphs(
+ all_elements, content_start, content_end
+ )
+
+ ch = {
+ "book": "pk",
+ "chapter": verse_num,
+ "title": f"Prayer {verse_num}",
+ "type": "chapter",
+ "paragraphs": paragraphs,
+ }
+ chapters.append(ch)
+
+ # Step 4: Build chapter listing for intro.json
+ chapter_listing = [
+ {"number": ch["chapter"], "title": ch["title"]} for ch in chapters
+ ]
+
+ return {
+ "chapters": chapters,
+ "chapter_listing": chapter_listing,
+ }
+
+
+# ---------------------------------------------------------------------------
+# Output
+# ---------------------------------------------------------------------------
+
+
+def write_json(data, filepath):
+ """Write data as formatted JSON to filepath, creating directories as needed."""
+ os.makedirs(os.path.dirname(filepath), exist_ok=True)
+ with open(filepath, "w", encoding="utf-8") as f:
+ json.dump(data, f, ensure_ascii=False, indent=2)
+
+
+def main():
+ base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+ html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html")
+ output_base = os.path.join(base_dir, "lib", "jsons", "books", "pk")
+
+ if not os.path.exists(html_path):
+ print(f"ERROR: {html_path} not found!")
+ sys.exit(1)
+
+ print(f"Reading {html_path}...")
+ with open(html_path, "r", encoding="utf-8") as f:
+ soup = BeautifulSoup(f, "html.parser")
+
+ print("Extracting Prayers of King Kulaśekhara (1958)...\n")
+ result = extract_pk(soup)
+
+ chapters = result["chapters"]
+ chapter_listing = result["chapter_listing"]
+
+ # --- Write intro.json ---
+ intro_meta = {
+ "book": "pk",
+ "title": "Prayers of King Kulaśekhara (1958)",
+ "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
+ "year": 1958,
+ "description": "Early handwritten translations and purports of the Mukunda-mālā-stotra prayers (verses 1–6).",
+ "totalChapters": 6,
+ "chapters": chapter_listing,
+ }
+ intro_path = os.path.join(output_base, "intro.json")
+ write_json(intro_meta, intro_path)
+ print(f"\n Written: {intro_path}")
+
+ # --- Write chapters ---
+ print(f"\n--- Chapters ---")
+ for ch in chapters:
+ ch_num_str = str(ch["chapter"]).zfill(2)
+ filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
+ write_json(ch, filepath)
+ print(
+ f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
+ f"{ch['title']}"
+ )
+
+ # --- Summary ---
+ total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
+
+ print(f"\n{'=' * 60}")
+ print(f" EXTRACTION SUMMARY")
+ print(f"{'=' * 60}")
+ print(f" Chapters: {len(chapters)}")
+ for ch in chapters:
+ print(
+ f" Ch {ch['chapter']}: {len(ch['paragraphs'])} paragraphs — {ch['title']}"
+ )
+ print(f" Total paragraphs: {total_chapter_paras}")
+ if chapters:
+ print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
+ print(f" Output directory: {output_base}/")
+
+ # --- Samples ---
+ for ch_idx, ch in enumerate(chapters):
+ print(f"\n--- Sample: Chapter {ch['chapter']} ({ch['title']}) ---")
+ for i, p in enumerate(ch["paragraphs"][:5]):
+ preview = re.sub(r"<[^>]+>", "", p)[:120]
+ print(f" [{i}] {preview}")
+ if len(ch["paragraphs"]) > 5:
+ print(f" ... ({len(ch['paragraphs'])} total paragraphs)")
+
+ # --- Quality Checks ---
+ print(f"\n--- Quality Checks ---")
+
+ # Check all 6 chapters present
+ expected_chapters = set(range(1, 7))
+ actual_chapters = set(ch["chapter"] for ch in chapters)
+ missing = expected_chapters - actual_chapters
+ extra = actual_chapters - expected_chapters
+ if missing:
+ print(f" ⚠ Missing chapters: {sorted(missing)}")
+ if extra:
+ print(f" ⚠ Extra chapters: {sorted(extra)}")
+ if not missing and not extra:
+ print(f" ✅ All 6 chapters present")
+
+ # Check empty chapters
+ empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
+ if empty_chapters:
+ print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
+ else:
+ print(f" ✅ No empty chapters")
+
+ # Check each chapter has expected sections
+ for ch in chapters:
+ ch_text = " ".join(ch["paragraphs"])
+ has_wordings = "Wordings" in ch_text
+ has_synonyms = "Synonyms" in ch_text
+ has_translation = "Translation" in ch_text
+ has_purport = "Purport" in ch_text
+ sections = []
+ if has_wordings:
+ sections.append("W")
+ if has_synonyms:
+ sections.append("S")
+ if has_translation:
+ sections.append("T")
+ if has_purport:
+ sections.append("P")
+ status = "✅" if len(sections) == 4 else "⚠"
+ print(f" {status} Ch {ch['chapter']}: sections [{', '.join(sections)}]")
+
+ # Check synonym entries use em-dash format
+ synonym_count = 0
+ for ch in chapters:
+ for p in ch["paragraphs"]:
+ if "—" in p and "" not in p:
+ synonym_count += 1
+ break
+ print(f" ℹ Chapters with em-dash synonyms: {synonym_count}/6")
+
+ # Verify no Shaktipujah content leaked in
+ all_text = ""
+ for ch in chapters:
+ all_text += " ".join(ch["paragraphs"])
+ if "Shaktipujah" in all_text or "Baidya Nath" in all_text:
+ print(f" ⚠ Content from Shaktipujah may have leaked into PK!")
+ else:
+ print(f" ✅ No content leak from next work detected")
+
+ # Check last chapter ends cleanly
+ if chapters:
+ last_ch = chapters[-1]
+ last_para = last_ch["paragraphs"][-1] if last_ch["paragraphs"] else ""
+ last_preview = re.sub(r"<[^>]+>", "", last_para)[:80]
+ print(f' ℹ Last paragraph of Ch {last_ch["chapter"]}: "{last_preview}..."')
+
+ print(f"\nDone! ✨")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/seed-bk.ts b/scripts/seed-bk.ts
new file mode 100644
index 00000000..6c69c817
--- /dev/null
+++ b/scripts/seed-bk.ts
@@ -0,0 +1,169 @@
+/**
+ * seed-bk.ts — Push Bhagavāner Kathā JSON data into MongoDB.
+ *
+ * Uses the native MongoDB driver (same pattern as seed-cbhc.ts) to avoid
+ * Prisma replica-set write requirements.
+ *
+ * BK is a simple prose book with no volume structure and no intro segments.
+ * We use volume=0 as the sentinel for "no volumes".
+ *
+ * Collections written:
+ * - book_meta → one doc { book: "bk", data:
tags
+ html = re.sub(r"^(
":
+ paragraphs.append(html)
+
+ return paragraphs
+
+
+# ---------------------------------------------------------------------------
+# Extract day entries from a chapter section
+# ---------------------------------------------------------------------------
+
+
+def extract_day_entries(tags, start_idx, end_idx):
+ """Extract day entries from a chapter section.
+ Returns (summary_paragraphs, day_entries_list).
+ """
+ summary_paragraphs = []
+ day_entries = []
+ current_entry = None
+ in_summary = True # Start in summary mode until first p39 day entry
+
+ skip_classes = {"p4", "p11", "p3", "p5", "p153"}
+
+ def finalize_entry():
+ nonlocal current_entry
+ if current_entry and current_entry["paragraphs"]:
+ day_entries.append(current_entry)
+ current_entry = None
+
+ for i in range(start_idx, end_idx):
+ tag = tags[i]
+ cls = get_css_class(tag)
+ text = tag.get_text().strip()
+
+ if cls in skip_classes:
+ continue
+
+ # p39 = day bookmark
+ if cls == "p39":
+ finalize_entry()
+ in_summary = False
+
+ # Check if it's a special entry like "Notice" or "Memoranda"
+ if text.lower() == "notice":
+ current_entry = {
+ "day_name": None,
+ "date": None,
+ "month": None,
+ "slug": "notice",
+ "title": "Notice",
+ "paragraphs": [],
+ }
+ continue
+ elif text.lower() == "memoranda":
+ current_entry = {
+ "day_name": None,
+ "date": None,
+ "month": None,
+ "slug": "memoranda",
+ "title": "Memoranda",
+ "paragraphs": [],
+ }
+ continue
+
+ day_name, date_num, month_abbr = parse_day_month(text)
+ if day_name and date_num and month_abbr:
+ slug = f"{month_abbr}-{date_num}"
+ title = f"{day_name} {date_num}"
+ current_entry = {
+ "day_name": day_name,
+ "date": date_num,
+ "month": month_abbr,
+ "slug": slug,
+ "title": title,
+ "paragraphs": [],
+ }
+ else:
+ # Can't parse — might be an excerpt title or special entry
+ # Treat as a titled entry
+ slug_text = re.sub(r"[^a-z0-9]+", "-", text.lower().strip())[:50].strip(
+ "-"
+ )
+ current_entry = {
+ "day_name": None,
+ "date": None,
+ "month": None,
+ "slug": slug_text or "unknown",
+ "title": text,
+ "paragraphs": [],
+ }
+ continue
+
+ # p8 = bold day header (display version of p39)
+ if cls == "p8":
+ # This is the bold display version — skip if we already have the entry from p39
+ if current_entry is not None:
+ continue
+ # If we're in summary mode, this might be the chapter date range or summary header
+ if in_summary:
+ html = inner_html(tag)
+ if html:
+ summary_paragraphs.append(html)
+ continue
+ continue
+
+ # p24 = empty line (skip)
+ if cls == "p24":
+ continue
+
+ # p29 = bold subheading (like "NOTICE")
+ if cls == "p29":
+ if current_entry is not None:
+ html = inner_html(tag)
+ if html:
+ current_entry["paragraphs"].append(html)
+ continue
+
+ # Collect body paragraphs
+ html = inner_html(tag)
+ if not html or html == "
":
+ continue
+
+ if in_summary:
+ summary_paragraphs.append(html)
+ elif current_entry is not None:
+ current_entry["paragraphs"].append(html)
+
+ finalize_entry()
+ return summary_paragraphs, day_entries
+
+
+# ---------------------------------------------------------------------------
+# Extract excerpts
+# ---------------------------------------------------------------------------
+
+
+def extract_excerpts(tags, start_idx, end_idx):
+ """Extract individual excerpts from the Excerpts section.
+ Each excerpt starts with a p39 bookmark.
+ Returns a list of { slug, title, paragraphs }.
+ """
+ excerpts = []
+ current_excerpt = None
+ in_preamble = True # Before first p39 excerpt
+
+ preamble_paragraphs = []
+ skip_classes = {"p4", "p11", "p3", "p5", "p153"}
+ seen_slugs = set()
+
+ def finalize_excerpt():
+ nonlocal current_excerpt
+ if current_excerpt and current_excerpt["paragraphs"]:
+ excerpts.append(current_excerpt)
+ current_excerpt = None
+
+ for i in range(start_idx, end_idx):
+ tag = tags[i]
+ cls = get_css_class(tag)
+ text = tag.get_text().strip()
+
+ if cls in skip_classes:
+ continue
+
+ # p39 = excerpt title bookmark
+ if cls == "p39":
+ finalize_excerpt()
+ in_preamble = False
+
+ title = decode_scagoudy(text)
+ slug = slugify_excerpt(title)
+
+ # Ensure unique slug
+ if slug in seen_slugs:
+ suffix = 2
+ while f"{slug}-{suffix}" in seen_slugs:
+ suffix += 1
+ slug = f"{slug}-{suffix}"
+ seen_slugs.add(slug)
+
+ current_excerpt = {
+ "slug": slug,
+ "title": title,
+ "paragraphs": [],
+ }
+ continue
+
+ # p8 = bold excerpt header (display version) — skip if we have the excerpt
+ if cls == "p8":
+ if current_excerpt is not None:
+ # Skip the bold header — we already have the title from p39
+ continue
+ if in_preamble:
+ html = inner_html(tag)
+ if html:
+ preamble_paragraphs.append(html)
+ continue
+
+ if cls == "p24":
+ continue
+
+ html = inner_html(tag)
+ if not html or html == "
":
+ continue
+
+ if in_preamble:
+ preamble_paragraphs.append(html)
+ elif current_excerpt is not None:
+ # For conversation lines (p74), format with bold speaker
+ # The inner_html already preserves tags
+ current_excerpt["paragraphs"].append(html)
+
+ finalize_excerpt()
+
+ return preamble_paragraphs, excerpts
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+
+def main():
+ base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+ html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html")
+ output_dir = os.path.join(base_dir, "lib", "jsons", "journals", "the-beginning")
+ chapters_dir = os.path.join(output_dir, "chapters")
+
+ if not os.path.exists(html_path):
+ print(f"ERROR: {html_path} not found!")
+ sys.exit(1)
+
+ print(f"Reading {html_path}...")
+ with open(html_path, "r", encoding="utf-8") as f:
+ content = f.read()
+
+ soup = BeautifulSoup(content, "html.parser")
+ all_tags = soup.find_all("p")
+ print(f" Parsed {len(all_tags)}