krispire/spsays-training
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add ssr

Browse Source
This commit is contained in:
gosangam
2026-04-17 22:51:58 +05:30
parent d88ababd76
commit 0aab1d23ec
97 changed files with 29891 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/bbd_extract.py
+665
View File
@@ -0,0 +1,665 @@
#!/usr/bin/env python3
"""Extract Beyond Birth and Death (1974) from other.html.
The book has 1 intro segment (front-pages) and 5 chapters.
Output:
lib/jsons/books/bbd/intro.json — Book metadata
lib/jsons/books/bbd/intro/*.json — Intro segments
lib/jsons/books/bbd/chapters/*.json — Chapters
Usage:
python3 scripts/bbd_extract.py
"""
import json
import os
import re
import sys
from bs4 import BeautifulSoup, Tag
# ---------------------------------------------------------------------------
# ScaGoudy → IAST Unicode mapping
# ---------------------------------------------------------------------------
SCAGOUDY_TO_IAST = {
"ä": "ā",
"Ä": "Ā",
"å": "ṛ",
"Å": "Ṛ",
"ñ": "ṣ",
"Ñ": "Ṣ",
"ö": "ṭ",
"Ö": "Ṭ",
"ë": "ṇ",
"Ë": "Ṇ",
"ò": "ḍ",
"Ò": "Ḍ",
"ç": "ś",
"Ç": "Ś",
"ì": "ṅ",
"ï": "ñ",
"ü": "ū",
"Ü": "Ū",
"é": "ī",
"É": "Ī",
"à": "ṁ",
"ù": "ḥ",
"è": "ṝ",
"û": "ṁl",
"ÿ": "ḷ",
}
def decode_scagoudy(text: str) -> str:
"""Replace ScaGoudy-encoded characters with proper IAST Unicode."""
for src, dst in SCAGOUDY_TO_IAST.items():
text = text.replace(src, dst)
return text
def decode_html_content(tag: Tag) -> str:
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
inner = "".join(str(child) for child in tag.children)
inner = decode_scagoudy(inner)
# Remove span tags with CSS classes (RTF conversion artifacts)
inner = re.sub(
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
)
# Clean up Apple-tab-span leftovers (tab characters)
inner = inner.replace("\t", " ")
# Normalize whitespace (but preserve <br> tags)
inner = re.sub(r"\s+", " ", inner).strip()
return inner
def decode_text(tag: Tag) -> str:
"""Get plain text from a tag with ScaGoudy decoding."""
return decode_scagoudy(tag.get_text().strip())
def get_css_class(tag: Tag) -> str:
"""Get the first CSS class of a tag, or empty string."""
classes = tag.get("class", [])
return classes[0] if classes else ""
def is_empty(tag: Tag) -> bool:
"""Check if a tag is empty or whitespace-only."""
text = tag.get_text().strip()
return not text or text == "\xa0"
def is_dash_line(text: str) -> bool:
"""Return True if text is only dashes, spaces, and/or whitespace."""
cleaned = text.strip()
if not cleaned:
return False
return all(c in "-—–\u2014\u2013 \t" for c in cleaned)
# ---------------------------------------------------------------------------
# CSS class categorization for BBD
# ---------------------------------------------------------------------------
# Book title header class
BOOK_HEADER_CLASS = "p12"
# Chapter bookmark class
CHAPTER_BOOKMARK_CLASS = "p13"
# Body paragraph classes (primary content)
BODY_CLASSES = frozenset(["p6"])
# Sanskrit verse pāda classes (italic, centered)
VERSE_PADA_CLASSES = frozenset(["p53"])
# Bold quote paragraphs — include as body
BOLD_QUOTE_CLASSES = frozenset(["p56", "p30"])
# Verse translations / body text
TRANSLATION_CLASS = "p44"
# Chapter heading (bold)
CHAPTER_HEADING_CLASS = "p27"
# Title page text
TITLE_PAGE_CLASS = "p3"
# Table of contents
TOC_CLASS = "p80"
# END marker class
END_MARKER_CLASS = "p81"
# Classes to skip entirely (empty/decorative)
SKIP_CLASSES = frozenset(["p9", "p70"])
# ---------------------------------------------------------------------------
# Extraction helpers
# ---------------------------------------------------------------------------
def find_bbd_boundaries(all_paragraphs: list) -> tuple:
"""Find the start and end paragraph indices of the BBD content.
Start: <p class="p12"> containing "Beyond Birth and Death" and "1974"
End: <p class="p81"><b>END</b></p> or next <p class="p12"> after start
"""
start_idx = None
end_idx = len(all_paragraphs)
for i, p in enumerate(all_paragraphs):
cls = get_css_class(p)
if cls != BOOK_HEADER_CLASS:
continue
text = decode_text(p)
if "Beyond Birth and Death" in text and "1974" in text:
start_idx = i
elif start_idx is not None:
# Next p12 after BBD start = end of BBD
end_idx = i
break
if start_idx is None:
print(
"ERROR: Could not find 'Beyond Birth and Death (1974)' header in other.html!"
)
sys.exit(1)
# Also check for END marker (p81 with text "END") as end boundary
for i in range(start_idx + 1, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
if cls == END_MARKER_CLASS:
text = p.get_text().strip()
if text.upper() == "END":
end_idx = i
break
return start_idx, end_idx
def find_chapter_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
"""Find all BBD chapter bookmark tags (p13) in the BBD range.
Returns list of (paragraph_index, chapter_number, chapter_title).
"""
bookmarks = []
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
if cls == CHAPTER_BOOKMARK_CLASS:
text = decode_text(p)
m = re.match(r"BBD\s+(\d+):\s+(.+)", text.strip())
if m:
ch_num = int(m.group(1))
ch_title = m.group(2).strip()
bookmarks.append((i, ch_num, ch_title))
return bookmarks
def extract_paragraphs(
all_paragraphs: list,
start_idx: int,
end_idx: int,
is_front_pages: bool = False,
) -> list:
"""Extract paragraphs for a section (chapter or intro segment).
Handles all CSS classes within the BBD content.
Groups consecutive p53 verse pādas into centered italic divs.
"""
result = []
verse_accumulator = []
def flush_verse():
if verse_accumulator:
joined = "<br>".join(verse_accumulator)
result.append(f'<div style="text-align:center"><i>{joined}</i></div>')
verse_accumulator.clear()
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
# Skip empty paragraphs
if is_empty(p):
continue
# Skip noise classes
if cls in SKIP_CLASSES:
continue
# Skip bookmark tags (p13)
if cls == CHAPTER_BOOKMARK_CLASS:
continue
# Skip book title header (p12)
if cls == BOOK_HEADER_CLASS:
continue
# Skip END marker (p81)
if cls == END_MARKER_CLASS:
continue
# Check for dash lines — skip them
text = decode_text(p)
if is_dash_line(text):
continue
# Sanskrit verse pādas (p53) — accumulate into centered block
if cls in VERSE_PADA_CLASSES:
html = decode_html_content(p)
if html:
# Remove inner <i> tags before re-wrapping
stripped = re.sub(r"</?i>", "", html)
verse_accumulator.append(stripped)
continue
# Non-verse paragraph — flush any accumulated verse first
flush_verse()
# Chapter heading (p27) — include as paragraph
if cls == CHAPTER_HEADING_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Title page text (p3) — include
if cls == TITLE_PAGE_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Table of contents (p80) — include in front-pages
if cls == TOC_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Bold quote paragraphs (p56, p30) — include as body
if cls in BOLD_QUOTE_CLASSES:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Verse translation / body text (p44)
if cls == TRANSLATION_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Body paragraphs (p6)
if cls in BODY_CLASSES:
html = decode_html_content(p)
if html:
# Detect Sanskrit verses disguised as body paragraphs:
# they start with <i> and contain <br> (multi-line italic = verse)
stripped_html = html.strip()
if stripped_html.startswith("<i>") and "<br" in stripped_html:
# This is a verse — wrap in centered italic div
inner = re.sub(r"</?i>", "", stripped_html)
result.append(
f'<div style="text-align:center"><i>{inner}</i></div>'
)
else:
result.append(html)
continue
# Any other class — try to extract content (be permissive)
html = decode_html_content(p)
if html and not is_dash_line(text):
result.append(html)
# Flush any trailing verse
flush_verse()
return result
# ---------------------------------------------------------------------------
# Main extraction logic
# ---------------------------------------------------------------------------
def extract_bbd(soup: BeautifulSoup) -> dict:
"""Extract the entire Beyond Birth and Death (1974) from the parsed HTML.
Returns a dict with:
"intro_segments": list of intro segment dicts
"chapters": list of chapter dicts
"chapter_listing": list of {number, title} for intro.json
"""
all_paragraphs = list(soup.find_all("p"))
# Step 1: Find BBD boundaries
bbd_start, bbd_end = find_bbd_boundaries(all_paragraphs)
print(f" BBD content: paragraphs {bbd_start} to {bbd_end}")
print(f" Total paragraphs in range: {bbd_end - bbd_start}")
# Step 2: Find all chapter bookmarks
chapter_bookmarks = find_chapter_bookmarks(all_paragraphs, bbd_start, bbd_end)
print(f" Found {len(chapter_bookmarks)} BBD chapter bookmarks")
for pos, ch_num, ch_title in chapter_bookmarks:
print(f" [chapter {ch_num}] paragraph {pos}: {ch_title}")
# Step 3: Extract front pages
# From book start (bbd_start) to first chapter bookmark (BBD 1)
intro_segments = []
if chapter_bookmarks:
front_pages_end = chapter_bookmarks[0][0]
else:
front_pages_end = bbd_end
front_pages_paragraphs = extract_paragraphs(
all_paragraphs, bbd_start, front_pages_end, is_front_pages=True
)
intro_segments.append(
{
"slug": "front-pages",
"title": "Front Pages",
"type": "intro-segment",
"book": "bbd",
"order": 1,
"paragraphs": front_pages_paragraphs,
}
)
# Step 4: Extract chapters
chapters = []
for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_bookmarks):
content_start = ch_pos + 1
# Content ends at next chapter bookmark or bbd_end
if idx + 1 < len(chapter_bookmarks):
content_end = chapter_bookmarks[idx + 1][0]
else:
content_end = bbd_end
paragraphs = extract_paragraphs(all_paragraphs, content_start, content_end)
ch = {
"book": "bbd",
"chapter": ch_num,
"title": ch_title,
"type": "chapter",
"paragraphs": paragraphs,
}
chapters.append(ch)
# Step 5: Build chapter listing for intro.json
chapter_listing = [
{"number": ch["chapter"], "title": ch["title"]} for ch in chapters
]
return {
"intro_segments": intro_segments,
"chapters": chapters,
"chapter_listing": chapter_listing,
}
# ---------------------------------------------------------------------------
# Output
# ---------------------------------------------------------------------------
def write_json(data, filepath):
"""Write data as formatted JSON to filepath, creating directories as needed."""
os.makedirs(os.path.dirname(filepath), exist_ok=True)
with open(filepath, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False, indent=2)
def main():
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
html_path = os.path.join(base_dir, "public", "html", "other.html")
output_base = os.path.join(base_dir, "lib", "jsons", "books", "bbd")
if not os.path.exists(html_path):
print(f"ERROR: {html_path} not found!")
sys.exit(1)
print(f"Reading {html_path}...")
with open(html_path, "r", encoding="utf-8") as f:
soup = BeautifulSoup(f, "html.parser")
print("Extracting Beyond Birth and Death (1974)...\n")
result = extract_bbd(soup)
intro_segments = result["intro_segments"]
chapters = result["chapters"]
chapter_listing = result["chapter_listing"]
# --- Write intro.json ---
intro_meta = {
"book": "bbd",
"title": "Beyond Birth and Death",
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
"year": 1974,
"description": "An introduction to the science of the self, transcending the cycle of birth and death through Kṛṣṇa consciousness.",
"totalChapters": 5,
"chapters": chapter_listing,
}
intro_path = os.path.join(output_base, "intro.json")
write_json(intro_meta, intro_path)
print(f"\n Written: {intro_path}")
# --- Write intro segments ---
print(f"\n--- Intro Segments ---")
for seg in intro_segments:
filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
write_json(seg, filepath)
print(
f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}"
)
# --- Write chapters ---
print(f"\n--- Chapters ---")
for ch in chapters:
ch_num_str = str(ch["chapter"]).zfill(2)
filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
write_json(ch, filepath)
print(
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
f"{ch['title']}"
)
# --- Summary ---
total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments)
total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
total_paras = total_intro_paras + total_chapter_paras
print(f"\n{'=' * 60}")
print(f" EXTRACTION SUMMARY")
print(f"{'=' * 60}")
print(f" Intro segments: {len(intro_segments)}")
print(f" Chapters: {len(chapters)}")
print(f" Total paragraphs: {total_paras}")
print(f" Intro: {total_intro_paras}")
print(f" Chapters: {total_chapter_paras}")
if chapters:
print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
print(f" Output directory: {output_base}/")
# --- Samples ---
print(f"\n--- Sample: Front Pages ---")
if intro_segments:
front = intro_segments[0]
for i, p in enumerate(front["paragraphs"][:5]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
if chapters:
print(f"\n--- Sample: Chapter 1 ({chapters[0]['title']}) ---")
ch1 = chapters[0]
for i, p in enumerate(ch1["paragraphs"][:5]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
if chapters and len(chapters) > 1:
print(
f"\n--- Sample: Chapter {chapters[-1]['chapter']} ({chapters[-1]['title']}) ---"
)
last = chapters[-1]
for i, p in enumerate(last["paragraphs"][:5]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
# --- Quality Checks ---
print(f"\n--- Quality Checks ---")
# Check all 5 chapters present
expected_chapters = set(range(1, 6))
actual_chapters = set(ch["chapter"] for ch in chapters)
missing = expected_chapters - actual_chapters
extra = actual_chapters - expected_chapters
if missing:
print(f" ⚠ Missing chapters: {sorted(missing)}")
if extra:
print(f" ⚠ Extra chapters: {sorted(extra)}")
if not missing and not extra:
print(f" ✅ All 5 chapters present")
# Check all 1 intro segment present
expected_intros = {"front-pages"}
actual_intros = {s["slug"] for s in intro_segments}
missing_intros = expected_intros - actual_intros
extra_intros = actual_intros - expected_intros
if missing_intros:
print(f" ⚠ Missing intro segments: {missing_intros}")
if extra_intros:
print(f" ⚠ Extra intro segments: {extra_intros}")
if not missing_intros and not extra_intros:
print(f" ✅ All 1 intro segments present")
# Check empty chapters
empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
if empty_chapters:
print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
else:
print(f" ✅ No empty chapters")
# Check empty intro segments
empty_intros = [s["slug"] for s in intro_segments if len(s["paragraphs"]) == 0]
if empty_intros:
print(f" ⚠ Empty intro segments: {empty_intros}")
else:
print(f" ✅ No empty intro segments")
# Check ScaGoudy decoding
print(f"\n--- ScaGoudy Decoding Check ---")
all_text = ""
for ch in chapters:
all_text += " ".join(ch["paragraphs"][:20])
for seg in intro_segments:
all_text += " ".join(seg["paragraphs"][:10])
# Check for properly decoded words
if "Kṛṣṇa" in all_text:
print(f" ✅ 'Kṛṣṇa' found (ScaGoudy properly decoded)")
else:
print(f" ⚠ 'Kṛṣṇa' not found — ScaGoudy decoding may have failed")
if "Bhagavad-gītā" in all_text:
print(f" ✅ 'Bhagavad-gītā' found (ScaGoudy properly decoded)")
elif "gītā" in all_text or "Gītā" in all_text:
print(f" ✅ 'gītā' found (ScaGoudy properly decoded)")
else:
print(f" ⚠ 'Gītā' not found — ScaGoudy decoding may have failed")
if "Prabhupāda" in all_text:
print(f" ✅ 'Prabhupāda' found (ScaGoudy properly decoded)")
# Check for un-decoded ScaGoudy remnants (common encoded chars)
scagoudy_remnants = 0
for ch in chapters:
for p in ch["paragraphs"]:
if "Kåñëa" in p or "Prabhupäda" in p:
scagoudy_remnants += 1
for seg in intro_segments:
for p in seg["paragraphs"]:
if "Kåñëa" in p or "Prabhupäda" in p:
scagoudy_remnants += 1
if scagoudy_remnants == 0:
print(f" ✅ No ScaGoudy remnants detected")
else:
print(f" ⚠ {scagoudy_remnants} paragraphs with ScaGoudy remnants")
# Check for centered verse blocks
verse_block_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
if '<div style="text-align:center"><i>' in p:
verse_block_count += 1
if verse_block_count > 0:
print(f" ✅ {verse_block_count} centered Sanskrit verse blocks found")
else:
print(f" ⚠ No centered verse blocks found — verse grouping may have failed")
# Check for residual span tags
span_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
if 'class="' in p and "<span" in p:
span_count += 1
for seg in intro_segments:
for p in seg["paragraphs"]:
if 'class="' in p and "<span" in p:
span_count += 1
if span_count == 0:
print(f" ✅ No residual span class artifacts found")
else:
print(f" ⚠ {span_count} paragraphs still contain span class artifacts")
# Check for dash-line paragraphs that shouldn't be there
dash_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
plain = re.sub(r"<[^>]+>", "", p)
if is_dash_line(plain):
dash_count += 1
for seg in intro_segments:
for p in seg["paragraphs"]:
plain = re.sub(r"<[^>]+>", "", p)
if is_dash_line(plain):
dash_count += 1
if dash_count == 0:
print(f" ✅ No dashed-line paragraphs in output")
else:
print(f" ⚠ {dash_count} dashed-line paragraphs found in output")
# Show a verse block sample
print(f"\n--- Sample: Verse Block ---")
for ch in chapters:
for p in ch["paragraphs"]:
if '<div style="text-align:center"><i>' in p:
preview = re.sub(r"<[^>]+>", "", p)[:120]
print(f" {preview}...")
break
else:
continue
break
# Chapter title decoding check
print(f"\n--- Chapter Title Decoding Check ---")
for ch in chapters:
title = ch["title"]
if "å" in title or "ë" in title or "ñ" in title or "ä" in title:
print(f" ⚠ Chapter {ch['chapter']} title has un-decoded ScaGoudy: {title}")
else:
print(f" ✅ Ch {ch['chapter']}: {title}")
print(f"\nDone! ✨")
if __name__ == "__main__":
main()
scripts/ds_extract.py
+1082
View File
File diff suppressed because it is too large Load Diff
scripts/ej_extract.py
+750
View File
@@ -0,0 +1,750 @@
#!/usr/bin/env python3
"""Extract Easy Journey to Other Planets (1970 Edition) from other.html.
The book has 2 intro segments (front-pages, preface) and 2 chapters.
Output:
lib/jsons/books/ej/intro.json — Book metadata
lib/jsons/books/ej/intro/*.json — Intro segments
lib/jsons/books/ej/chapters/*.json — Chapters
Usage:
python3 scripts/ej_extract.py
"""
import json
import os
import re
import sys
from bs4 import BeautifulSoup, Tag
# ---------------------------------------------------------------------------
# ScaGoudy → IAST Unicode mapping
# ---------------------------------------------------------------------------
SCAGOUDY_TO_IAST = {
"ä": "ā",
"Ä": "Ā",
"å": "ṛ",
"Å": "Ṛ",
"ñ": "ṣ",
"Ñ": "Ṣ",
"ö": "ṭ",
"Ö": "Ṭ",
"ë": "ṇ",
"Ë": "Ṇ",
"ò": "ḍ",
"Ò": "Ḍ",
"ç": "ś",
"Ç": "Ś",
"ì": "ṅ",
"ï": "ñ",
"ü": "ū",
"Ü": "Ū",
"é": "ī",
"É": "Ī",
"à": "ṁ",
"ù": "ḥ",
"è": "ṝ",
"û": "ṁl",
"ÿ": "ḷ",
}
def decode_scagoudy(text: str) -> str:
"""Replace ScaGoudy-encoded characters with proper IAST Unicode."""
for src, dst in SCAGOUDY_TO_IAST.items():
text = text.replace(src, dst)
return text
def decode_html_content(tag: Tag) -> str:
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
inner = "".join(str(child) for child in tag.children)
inner = decode_scagoudy(inner)
# Remove span tags with CSS classes (RTF conversion artifacts)
inner = re.sub(
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
)
# Clean up Apple-tab-span leftovers (tab characters)
inner = inner.replace("\t", " ")
# Normalize whitespace (but preserve <br> tags)
inner = re.sub(r"\s+", " ", inner).strip()
return inner
def decode_text(tag: Tag) -> str:
"""Get plain text from a tag with ScaGoudy decoding."""
return decode_scagoudy(tag.get_text().strip())
def get_css_class(tag: Tag) -> str:
"""Get the first CSS class of a tag, or empty string."""
classes = tag.get("class", [])
return classes[0] if classes else ""
def is_empty(tag: Tag) -> bool:
"""Check if a tag is empty or whitespace-only."""
text = tag.get_text().strip()
return not text or text == "\xa0"
def is_dash_line(text: str) -> bool:
"""Return True if text is only dashes, spaces, and/or whitespace."""
cleaned = text.strip()
if not cleaned:
return False
return all(c in "-—–\u2014\u2013 \t" for c in cleaned)
# ---------------------------------------------------------------------------
# CSS class categorization for EJ'70
# ---------------------------------------------------------------------------
# Book title header class
BOOK_HEADER_CLASS = "p12"
# Section bookmark class (Preface, Glossary)
SECTION_BOOKMARK_CLASS = "p64"
# Chapter bookmark class
CHAPTER_BOOKMARK_CLASS = "p49"
# Body paragraph classes (primary content)
BODY_CLASSES = frozenset(["p45", "p63", "p72", "p71"])
# Sanskrit verse pāda classes (italic, centered)
VERSE_PADA_CLASSES = frozenset(["p10", "p73"])
# Section heading (bold) — "Preface", "1 Anti-material Worlds", etc.
SECTION_HEADING_CLASS = "p31"
# Front-pages specific classes
BOOK_LIST_CLASS = "p68"
TITLE_PAGE_CLASS = "p3"
BOOK_TITLE_CLASS = "p69"
# Price text — skip
PRICE_CLASS = "p42"
# Glossary header — end boundary marker
GLOSSARY_HEADER_CLASS = "p19"
# Classes to skip entirely (empty/decorative)
SKIP_CLASSES = frozenset(["p9", "p70"])
# ---------------------------------------------------------------------------
# Intro segment definitions
# ---------------------------------------------------------------------------
INTRO_SEGMENTS = [
{
"slug": "front-pages",
"title": "Front Pages",
"order": 1,
"boundary_type": "start", # from book start to first section bookmark
},
{
"slug": "preface",
"title": "Preface",
"order": 2,
"bookmark_text": "EJ'70:Preface",
},
]
# ---------------------------------------------------------------------------
# Extraction helpers
# ---------------------------------------------------------------------------
def find_ej_boundaries(all_paragraphs: list) -> tuple:
"""Find the start and end paragraph indices of the EJ'70 content.
Start: <p class="p12"> containing "Easy Journey" and "1970"
End: next <p class="p12"> after start, or the EJ'70:Glossary bookmark
"""
start_idx = None
end_idx = len(all_paragraphs)
for i, p in enumerate(all_paragraphs):
cls = get_css_class(p)
if cls != BOOK_HEADER_CLASS:
continue
text = decode_text(p)
if "Easy Journey" in text and "1970" in text:
start_idx = i
elif start_idx is not None:
# Next p12 after EJ start = end of EJ
end_idx = i
break
if start_idx is None:
print(
"ERROR: Could not find 'Easy Journey to other planets (1970)' header in other.html!"
)
sys.exit(1)
# Also check for the EJ'70:Glossary bookmark as an end boundary
for i in range(start_idx + 1, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
if cls == SECTION_BOOKMARK_CLASS:
text = decode_text(p)
if "EJ'70:Glossary" in text:
end_idx = i
break
return start_idx, end_idx
def find_all_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
"""Find all bookmark tags (p64 and p49) in the EJ range.
Returns list of (paragraph_index, bookmark_text, bookmark_type).
bookmark_type is 'section' for p64, 'chapter' for p49.
"""
bookmarks = []
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
if cls == SECTION_BOOKMARK_CLASS:
text = decode_text(p)
if "EJ'70:" in text:
bookmarks.append((i, text, "section"))
elif cls == CHAPTER_BOOKMARK_CLASS:
text = decode_text(p)
if "EJ'70:" in text:
bookmarks.append((i, text, "chapter"))
return bookmarks
def parse_chapter_bookmark(text: str) -> tuple:
"""Parse a chapter bookmark like "EJ'70:1 Anti-material Worlds".
Returns (chapter_number, title) or None if not a chapter bookmark.
"""
m = re.match(r"EJ'70:(\d+)\s+(.+)", text.strip())
if m:
return int(m.group(1)), m.group(2).strip()
return None
def parse_section_bookmark(text: str) -> str:
"""Parse a section bookmark like "EJ'70:Preface" or "EJ'70:Glossary".
Returns the section name or None.
"""
m = re.match(r"EJ'70:(\w+)", text.strip())
if m:
return m.group(1).strip()
return None
def extract_paragraphs(
all_paragraphs: list,
start_idx: int,
end_idx: int,
is_front_pages: bool = False,
) -> list:
"""Extract paragraphs for a section (chapter or intro segment).
Handles all CSS classes within the EJ content.
Groups consecutive p10/p73 verse pādas into centered italic divs.
"""
result = []
verse_accumulator = []
def flush_verse():
if verse_accumulator:
joined = "<br>".join(verse_accumulator)
result.append(f'<div style="text-align:center"><i>{joined}</i></div>')
verse_accumulator.clear()
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
# Skip empty paragraphs
if is_empty(p):
continue
# Skip noise classes
if cls in SKIP_CLASSES:
continue
# Skip bookmark tags (both p64 and p49)
if cls in (SECTION_BOOKMARK_CLASS, CHAPTER_BOOKMARK_CLASS):
continue
# Skip book title header (p12)
if cls == BOOK_HEADER_CLASS:
continue
# Skip price text (p42)
if cls == PRICE_CLASS:
continue
# Skip glossary header (p19)
if cls == GLOSSARY_HEADER_CLASS:
continue
# Check for dash lines — skip them
text = decode_text(p)
if is_dash_line(text):
continue
# Sanskrit verse pādas (p10, p73) — accumulate into centered block
if cls in VERSE_PADA_CLASSES:
html = decode_html_content(p)
if html:
# Remove inner <i> tags before re-wrapping
stripped = re.sub(r"</?i>", "", html)
verse_accumulator.append(stripped)
continue
# Non-verse paragraph — flush any accumulated verse first
flush_verse()
# Section heading (p31) — include as paragraph
if cls == SECTION_HEADING_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Book title bold (p69) — include in front-pages
if cls == BOOK_TITLE_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Title page text (p3) — include in front-pages
if cls == TITLE_PAGE_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Book list items (p68) — include in front-pages
if cls == BOOK_LIST_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Body paragraphs (p45, p63, p72, p71)
if cls in BODY_CLASSES:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Any other class — try to extract content (be permissive)
html = decode_html_content(p)
if html and not is_dash_line(text):
result.append(html)
# Flush any trailing verse
flush_verse()
return result
# ---------------------------------------------------------------------------
# Main extraction logic
# ---------------------------------------------------------------------------
def extract_ej(soup: BeautifulSoup) -> dict:
"""Extract the entire Easy Journey to Other Planets (1970) from the parsed HTML.
Returns a dict with:
"intro_segments": list of intro segment dicts
"chapters": list of chapter dicts
"chapter_listing": list of {number, title} for intro.json
"""
all_paragraphs = list(soup.find_all("p"))
# Step 1: Find EJ boundaries
ej_start, ej_end = find_ej_boundaries(all_paragraphs)
print(f" EJ'70 content: paragraphs {ej_start} to {ej_end}")
print(f" Total paragraphs in range: {ej_end - ej_start}")
# Step 2: Find all bookmarks
bookmarks = find_all_bookmarks(all_paragraphs, ej_start, ej_end)
print(f" Found {len(bookmarks)} EJ'70 bookmarks")
for pos, text, btype in bookmarks:
print(f" [{btype}] paragraph {pos}: {text}")
# Step 3: Categorize bookmarks into section vs chapter
section_bookmarks = [] # (para_idx, bookmark_text)
chapter_bookmarks = [] # (para_idx, chapter_num, title)
for pos, text, btype in bookmarks:
ch = parse_chapter_bookmark(text)
if ch:
chapter_bookmarks.append((pos, ch[0], ch[1]))
else:
section_name = parse_section_bookmark(text)
if section_name and section_name != "Glossary":
section_bookmarks.append((pos, text))
print(f" Section bookmarks: {len(section_bookmarks)}")
print(f" Chapter bookmarks: {len(chapter_bookmarks)}")
# Step 4: Collect all boundary positions for range detection
# These are all the positions where a new section/chapter starts
all_boundary_positions = sorted(
[pos for pos, _ in section_bookmarks] + [pos for pos, _, _ in chapter_bookmarks]
)
# Step 5: Extract intro segments
intro_segments = []
# --- Front Pages ---
# From book start (ej_start) to first bookmark (Preface)
front_pages_end = all_boundary_positions[0] if all_boundary_positions else ej_end
front_pages_paragraphs = extract_paragraphs(
all_paragraphs, ej_start, front_pages_end, is_front_pages=True
)
intro_segments.append(
{
"slug": "front-pages",
"title": "Front Pages",
"type": "intro-segment",
"book": "ej",
"order": 1,
"paragraphs": front_pages_paragraphs,
}
)
# --- Preface ---
# From EJ'70:Preface bookmark to first chapter bookmark
preface_pos = None
for pos, text in section_bookmarks:
if "Preface" in text:
preface_pos = pos
break
if preface_pos is not None:
# Content starts after the bookmark
preface_start = preface_pos + 1
# Content ends at the first chapter bookmark
preface_end = ej_end
for s in all_boundary_positions:
if s > preface_pos:
preface_end = s
break
preface_paragraphs = extract_paragraphs(
all_paragraphs, preface_start, preface_end
)
intro_segments.append(
{
"slug": "preface",
"title": "Preface",
"type": "intro-segment",
"book": "ej",
"order": 2,
"paragraphs": preface_paragraphs,
}
)
else:
print(" WARNING: Could not find Preface bookmark!")
# Step 6: Extract chapters
chapters = []
for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_bookmarks):
content_start = ch_pos + 1
# Content ends at next chapter bookmark or ej_end
if idx + 1 < len(chapter_bookmarks):
content_end = chapter_bookmarks[idx + 1][0]
else:
content_end = ej_end
paragraphs = extract_paragraphs(all_paragraphs, content_start, content_end)
ch = {
"book": "ej",
"chapter": ch_num,
"title": ch_title,
"type": "chapter",
"paragraphs": paragraphs,
}
chapters.append(ch)
# Step 7: Build chapter listing for intro.json
chapter_listing = [
{"number": ch["chapter"], "title": ch["title"]} for ch in chapters
]
return {
"intro_segments": intro_segments,
"chapters": chapters,
"chapter_listing": chapter_listing,
}
# ---------------------------------------------------------------------------
# Output
# ---------------------------------------------------------------------------
def write_json(data, filepath):
"""Write data as formatted JSON to filepath, creating directories as needed."""
os.makedirs(os.path.dirname(filepath), exist_ok=True)
with open(filepath, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False, indent=2)
def main():
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
html_path = os.path.join(base_dir, "public", "html", "other.html")
output_base = os.path.join(base_dir, "lib", "jsons", "books", "ej")
if not os.path.exists(html_path):
print(f"ERROR: {html_path} not found!")
sys.exit(1)
print(f"Reading {html_path}...")
with open(html_path, "r", encoding="utf-8") as f:
soup = BeautifulSoup(f, "html.parser")
print("Extracting Easy Journey to Other Planets (1970 Edition)...\n")
result = extract_ej(soup)
intro_segments = result["intro_segments"]
chapters = result["chapters"]
chapter_listing = result["chapter_listing"]
# --- Write intro.json ---
intro_meta = {
"book": "ej",
"title": "Easy Journey to Other Planets",
"subtitle": "By Practice of Supreme Yoga",
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
"year": 1970,
"description": "How one can transfer oneself to other planets by the process of bhakti-yoga.",
"totalChapters": 2,
"chapters": chapter_listing,
}
intro_path = os.path.join(output_base, "intro.json")
write_json(intro_meta, intro_path)
print(f"\n Written: {intro_path}")
# --- Write intro segments ---
print(f"\n--- Intro Segments ---")
for seg in intro_segments:
filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
write_json(seg, filepath)
print(
f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}"
)
# --- Write chapters ---
print(f"\n--- Chapters ---")
for ch in chapters:
ch_num_str = str(ch["chapter"]).zfill(2)
filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
write_json(ch, filepath)
print(
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
f"{ch['title']}"
)
# --- Summary ---
total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments)
total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
total_paras = total_intro_paras + total_chapter_paras
print(f"\n{'=' * 60}")
print(f" EXTRACTION SUMMARY")
print(f"{'=' * 60}")
print(f" Intro segments: {len(intro_segments)}")
print(f" Chapters: {len(chapters)}")
print(f" Total paragraphs: {total_paras}")
print(f" Intro: {total_intro_paras}")
print(f" Chapters: {total_chapter_paras}")
if chapters:
print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
print(f" Output directory: {output_base}/")
# --- Samples ---
print(f"\n--- Sample: Front Pages ---")
if intro_segments:
front = intro_segments[0]
for i, p in enumerate(front["paragraphs"][:5]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
if len(intro_segments) > 1:
print(f"\n--- Sample: Preface ---")
preface = intro_segments[1]
for i, p in enumerate(preface["paragraphs"][:3]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
if chapters:
print(f"\n--- Sample: Chapter 1 ({chapters[0]['title']}) ---")
ch1 = chapters[0]
for i, p in enumerate(ch1["paragraphs"][:5]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
if chapters and len(chapters) > 1:
print(
f"\n--- Sample: Chapter {chapters[-1]['chapter']} ({chapters[-1]['title']}) ---"
)
last = chapters[-1]
for i, p in enumerate(last["paragraphs"][:5]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
# --- Quality Checks ---
print(f"\n--- Quality Checks ---")
# Check all 2 chapters present
expected_chapters = set(range(1, 3))
actual_chapters = set(ch["chapter"] for ch in chapters)
missing = expected_chapters - actual_chapters
extra = actual_chapters - expected_chapters
if missing:
print(f" ⚠ Missing chapters: {sorted(missing)}")
if extra:
print(f" ⚠ Extra chapters: {sorted(extra)}")
if not missing and not extra:
print(f" ✅ All 2 chapters present")
# Check all 2 intro segments present
expected_intros = {"front-pages", "preface"}
actual_intros = {s["slug"] for s in intro_segments}
missing_intros = expected_intros - actual_intros
extra_intros = actual_intros - expected_intros
if missing_intros:
print(f" ⚠ Missing intro segments: {missing_intros}")
if extra_intros:
print(f" ⚠ Extra intro segments: {extra_intros}")
if not missing_intros and not extra_intros:
print(f" ✅ All 2 intro segments present")
# Check empty chapters
empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
if empty_chapters:
print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
else:
print(f" ✅ No empty chapters")
# Check empty intro segments
empty_intros = [s["slug"] for s in intro_segments if len(s["paragraphs"]) == 0]
if empty_intros:
print(f" ⚠ Empty intro segments: {empty_intros}")
else:
print(f" ✅ No empty intro segments")
# Check ScaGoudy decoding
print(f"\n--- ScaGoudy Decoding Check ---")
all_text = ""
for ch in chapters:
all_text += " ".join(ch["paragraphs"][:20])
for seg in intro_segments:
all_text += " ".join(seg["paragraphs"][:10])
# Check for properly decoded words
if "Kṛṣṇa" in all_text:
print(f" ✅ 'Kṛṣṇa' found (ScaGoudy properly decoded)")
else:
print(f" ⚠ 'Kṛṣṇa' not found — ScaGoudy decoding may have failed")
if "Bhagavad-gītā" in all_text:
print(f" ✅ 'Bhagavad-gītā' found (ScaGoudy properly decoded)")
elif "Gītā" in all_text or "gītā" in all_text:
print(f" ✅ 'gītā' found (ScaGoudy properly decoded)")
else:
print(f" ⚠ 'Gītā' not found — ScaGoudy decoding may have failed")
if "Prabhupāda" in all_text:
print(f" ✅ 'Prabhupāda' found (ScaGoudy properly decoded)")
# Check for un-decoded ScaGoudy remnants (common encoded chars)
scagoudy_remnants = 0
# Look for patterns that are very likely ScaGoudy artifacts
# e.g., "Kåñëa" instead of "Kṛṣṇa"
for ch in chapters:
for p in ch["paragraphs"]:
if "Kåñëa" in p or "Prabhupäda" in p:
scagoudy_remnants += 1
for seg in intro_segments:
for p in seg["paragraphs"]:
if "Kåñëa" in p or "Prabhupäda" in p:
scagoudy_remnants += 1
if scagoudy_remnants == 0:
print(f" ✅ No ScaGoudy remnants detected")
else:
print(f" ⚠ {scagoudy_remnants} paragraphs with ScaGoudy remnants")
# Check for centered verse blocks
verse_block_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
if '<div style="text-align:center"><i>' in p:
verse_block_count += 1
if verse_block_count > 0:
print(f" ✅ {verse_block_count} centered Sanskrit verse blocks found")
else:
print(f" ⚠ No centered verse blocks found — verse grouping may have failed")
# Check for residual span tags
span_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
if 'class="' in p and "<span" in p:
span_count += 1
for seg in intro_segments:
for p in seg["paragraphs"]:
if 'class="' in p and "<span" in p:
span_count += 1
if span_count == 0:
print(f" ✅ No residual span class artifacts found")
else:
print(f" ⚠ {span_count} paragraphs still contain span class artifacts")
# Check for dash-line paragraphs that shouldn't be there
dash_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
plain = re.sub(r"<[^>]+>", "", p)
if is_dash_line(plain):
dash_count += 1
for seg in intro_segments:
for p in seg["paragraphs"]:
plain = re.sub(r"<[^>]+>", "", p)
if is_dash_line(plain):
dash_count += 1
if dash_count == 0:
print(f" ✅ No dashed-line paragraphs in output")
else:
print(f" ⚠ {dash_count} dashed-line paragraphs found in output")
# Show a verse block sample
print(f"\n--- Sample: Verse Block ---")
for ch in chapters:
for p in ch["paragraphs"]:
if '<div style="text-align:center"><i>' in p:
preview = re.sub(r"<[^>]+>", "", p)[:120]
print(f" {preview}...")
break
else:
continue
break
print(f"\nDone! ✨")
if __name__ == "__main__":
main()
scripts/ek_extract.py
+655
View File
@@ -0,0 +1,655 @@
#!/usr/bin/env python3
"""Extract Elevation to Kṛṣṇa Consciousness (1973) from other.html.
The book has 1 intro segment (front-pages) and 6 chapters.
Output:
lib/jsons/books/ek/intro.json — Book metadata
lib/jsons/books/ek/intro/*.json — Intro segments
lib/jsons/books/ek/chapters/*.json — Chapters
Usage:
python3 scripts/ek_extract.py
"""
import json
import os
import re
import sys
from bs4 import BeautifulSoup, Tag
# ---------------------------------------------------------------------------
# ScaGoudy → IAST Unicode mapping
# ---------------------------------------------------------------------------
SCAGOUDY_TO_IAST = {
"ä": "ā",
"Ä": "Ā",
"å": "ṛ",
"Å": "Ṛ",
"ñ": "ṣ",
"Ñ": "Ṣ",
"ö": "ṭ",
"Ö": "Ṭ",
"ë": "ṇ",
"Ë": "Ṇ",
"ò": "ḍ",
"Ò": "Ḍ",
"ç": "ś",
"Ç": "Ś",
"ì": "ṅ",
"ï": "ñ",
"ü": "ū",
"Ü": "Ū",
"é": "ī",
"É": "Ī",
"à": "ṁ",
"ù": "ḥ",
"è": "ṝ",
"û": "ṁl",
"ÿ": "ḷ",
}
def decode_scagoudy(text: str) -> str:
"""Replace ScaGoudy-encoded characters with proper IAST Unicode."""
for src, dst in SCAGOUDY_TO_IAST.items():
text = text.replace(src, dst)
return text
def decode_html_content(tag: Tag) -> str:
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
inner = "".join(str(child) for child in tag.children)
inner = decode_scagoudy(inner)
# Remove span tags with CSS classes (RTF conversion artifacts)
inner = re.sub(
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
)
# Clean up Apple-tab-span leftovers (tab characters)
inner = inner.replace("\t", " ")
# Normalize whitespace (but preserve <br> tags)
inner = re.sub(r"\s+", " ", inner).strip()
return inner
def decode_text(tag: Tag) -> str:
"""Get plain text from a tag with ScaGoudy decoding."""
return decode_scagoudy(tag.get_text().strip())
def get_css_class(tag: Tag) -> str:
"""Get the first CSS class of a tag, or empty string."""
classes = tag.get("class", [])
return classes[0] if classes else ""
def is_empty(tag: Tag) -> bool:
"""Check if a tag is empty or whitespace-only."""
text = tag.get_text().strip()
return not text or text == "\xa0"
def is_dash_line(text: str) -> bool:
"""Return True if text is only dashes, spaces, and/or whitespace."""
cleaned = text.strip()
if not cleaned:
return False
return all(c in "-—–\u2014\u2013 \t" for c in cleaned)
# ---------------------------------------------------------------------------
# CSS class categorization for EK
# ---------------------------------------------------------------------------
# Book title header class
BOOK_HEADER_CLASS = "p12"
# Chapter bookmark class
CHAPTER_BOOKMARK_CLASS = "p13"
# Body paragraph classes (primary content)
BODY_CLASSES = frozenset(["p6"])
# Sanskrit verse pāda classes (italic, centered)
VERSE_PADA_CLASSES = frozenset(["p53"])
# Verse translations / body text
TRANSLATION_CLASS = "p44"
# Chapter heading (bold)
CHAPTER_HEADING_CLASS = "p27"
# Title page text
TITLE_PAGE_CLASS = "p3"
# Source reference
SOURCE_REF_CLASS = "p74"
# Glossary header — end boundary marker
GLOSSARY_HEADER_CLASS = "p19"
# Classes to skip entirely (empty/decorative)
SKIP_CLASSES = frozenset(["p9", "p70"])
# ---------------------------------------------------------------------------
# Extraction helpers
# ---------------------------------------------------------------------------
def find_ek_boundaries(all_paragraphs: list) -> tuple:
"""Find the start and end paragraph indices of the EK content.
Start: <p class="p12"> containing "Elevation to" and "1973"
End: next <p class="p12"> after start, or <p class="p19">GLOSSARY</p>
"""
start_idx = None
end_idx = len(all_paragraphs)
for i, p in enumerate(all_paragraphs):
cls = get_css_class(p)
if cls != BOOK_HEADER_CLASS:
continue
text = decode_text(p)
if "Elevation" in text and "1973" in text:
start_idx = i
elif start_idx is not None:
# Next p12 after EK start = end of EK
end_idx = i
break
if start_idx is None:
print(
"ERROR: Could not find 'Elevation to Kṛṣṇa Consciousness (1973)' header in other.html!"
)
sys.exit(1)
# Also check for GLOSSARY as end boundary (p19 with text "GLOSSARY")
for i in range(start_idx + 1, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
if cls == GLOSSARY_HEADER_CLASS:
text = p.get_text().strip()
if text.upper().startswith("GLOSSARY"):
end_idx = i
break
return start_idx, end_idx
def find_chapter_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
"""Find all EK chapter bookmark tags (p13) in the EK range.
Returns list of (paragraph_index, chapter_number, chapter_title).
"""
bookmarks = []
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
if cls == CHAPTER_BOOKMARK_CLASS:
text = decode_text(p)
m = re.match(r"EK\s+(\d+):\s+(.+)", text.strip())
if m:
ch_num = int(m.group(1))
ch_title = m.group(2).strip()
bookmarks.append((i, ch_num, ch_title))
return bookmarks
def extract_paragraphs(
all_paragraphs: list,
start_idx: int,
end_idx: int,
is_front_pages: bool = False,
) -> list:
"""Extract paragraphs for a section (chapter or intro segment).
Handles all CSS classes within the EK content.
Groups consecutive p53 verse pādas into centered italic divs.
"""
result = []
verse_accumulator = []
def flush_verse():
if verse_accumulator:
joined = "<br>".join(verse_accumulator)
result.append(f'<div style="text-align:center"><i>{joined}</i></div>')
verse_accumulator.clear()
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
# Skip empty paragraphs
if is_empty(p):
continue
# Skip noise classes
if cls in SKIP_CLASSES:
continue
# Skip bookmark tags (p13)
if cls == CHAPTER_BOOKMARK_CLASS:
continue
# Skip book title header (p12)
if cls == BOOK_HEADER_CLASS:
continue
# Skip glossary header (p19)
if cls == GLOSSARY_HEADER_CLASS:
continue
# Check for dash lines — skip them
text = decode_text(p)
if is_dash_line(text):
continue
# Sanskrit verse pādas (p53) — accumulate into centered block
if cls in VERSE_PADA_CLASSES:
html = decode_html_content(p)
if html:
# Remove inner <i> tags before re-wrapping
stripped = re.sub(r"</?i>", "", html)
verse_accumulator.append(stripped)
continue
# Non-verse paragraph — flush any accumulated verse first
flush_verse()
# Chapter heading (p27) — include as paragraph
if cls == CHAPTER_HEADING_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Title page text (p3) — include
if cls == TITLE_PAGE_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Source reference (p74) — include as paragraph
if cls == SOURCE_REF_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Verse translation / body text (p44)
if cls == TRANSLATION_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Body paragraphs (p6)
if cls in BODY_CLASSES:
html = decode_html_content(p)
if html:
# Detect Sanskrit verses disguised as body paragraphs:
# they start with <i> and contain <br> (multi-line italic = verse)
stripped_html = html.strip()
if stripped_html.startswith("<i>") and "<br" in stripped_html:
# This is a verse — wrap in centered italic div
inner = re.sub(r"</?i>", "", stripped_html)
result.append(
f'<div style="text-align:center"><i>{inner}</i></div>'
)
else:
result.append(html)
continue
# Any other class — try to extract content (be permissive)
html = decode_html_content(p)
if html and not is_dash_line(text):
result.append(html)
# Flush any trailing verse
flush_verse()
return result
# ---------------------------------------------------------------------------
# Main extraction logic
# ---------------------------------------------------------------------------
def extract_ek(soup: BeautifulSoup) -> dict:
"""Extract the entire Elevation to Kṛṣṇa Consciousness (1973) from the parsed HTML.
Returns a dict with:
"intro_segments": list of intro segment dicts
"chapters": list of chapter dicts
"chapter_listing": list of {number, title} for intro.json
"""
all_paragraphs = list(soup.find_all("p"))
# Step 1: Find EK boundaries
ek_start, ek_end = find_ek_boundaries(all_paragraphs)
print(f" EK content: paragraphs {ek_start} to {ek_end}")
print(f" Total paragraphs in range: {ek_end - ek_start}")
# Step 2: Find all chapter bookmarks
chapter_bookmarks = find_chapter_bookmarks(all_paragraphs, ek_start, ek_end)
print(f" Found {len(chapter_bookmarks)} EK chapter bookmarks")
for pos, ch_num, ch_title in chapter_bookmarks:
print(f" [chapter {ch_num}] paragraph {pos}: {ch_title}")
# Step 3: Extract front pages
# From book start (ek_start) to first chapter bookmark (EK 1)
intro_segments = []
if chapter_bookmarks:
front_pages_end = chapter_bookmarks[0][0]
else:
front_pages_end = ek_end
front_pages_paragraphs = extract_paragraphs(
all_paragraphs, ek_start, front_pages_end, is_front_pages=True
)
intro_segments.append(
{
"slug": "front-pages",
"title": "Front Pages",
"type": "intro-segment",
"book": "ek",
"order": 1,
"paragraphs": front_pages_paragraphs,
}
)
# Step 4: Extract chapters
chapters = []
for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_bookmarks):
content_start = ch_pos + 1
# Content ends at next chapter bookmark or ek_end
if idx + 1 < len(chapter_bookmarks):
content_end = chapter_bookmarks[idx + 1][0]
else:
content_end = ek_end
paragraphs = extract_paragraphs(all_paragraphs, content_start, content_end)
ch = {
"book": "ek",
"chapter": ch_num,
"title": ch_title,
"type": "chapter",
"paragraphs": paragraphs,
}
chapters.append(ch)
# Step 5: Build chapter listing for intro.json
chapter_listing = [
{"number": ch["chapter"], "title": ch["title"]} for ch in chapters
]
return {
"intro_segments": intro_segments,
"chapters": chapters,
"chapter_listing": chapter_listing,
}
# ---------------------------------------------------------------------------
# Output
# ---------------------------------------------------------------------------
def write_json(data, filepath):
"""Write data as formatted JSON to filepath, creating directories as needed."""
os.makedirs(os.path.dirname(filepath), exist_ok=True)
with open(filepath, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False, indent=2)
def main():
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
html_path = os.path.join(base_dir, "public", "html", "other.html")
output_base = os.path.join(base_dir, "lib", "jsons", "books", "ek")
if not os.path.exists(html_path):
print(f"ERROR: {html_path} not found!")
sys.exit(1)
print(f"Reading {html_path}...")
with open(html_path, "r", encoding="utf-8") as f:
soup = BeautifulSoup(f, "html.parser")
print("Extracting Elevation to Kṛṣṇa Consciousness (1973)...\n")
result = extract_ek(soup)
intro_segments = result["intro_segments"]
chapters = result["chapters"]
chapter_listing = result["chapter_listing"]
# --- Write intro.json ---
intro_meta = {
"book": "ek",
"title": "Elevation to Kṛṣṇa Consciousness",
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
"year": 1973,
"description": "How to elevate oneself from material consciousness to the spiritual platform of Kṛṣṇa consciousness.",
"totalChapters": 6,
"chapters": chapter_listing,
}
intro_path = os.path.join(output_base, "intro.json")
write_json(intro_meta, intro_path)
print(f"\n Written: {intro_path}")
# --- Write intro segments ---
print(f"\n--- Intro Segments ---")
for seg in intro_segments:
filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
write_json(seg, filepath)
print(
f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}"
)
# --- Write chapters ---
print(f"\n--- Chapters ---")
for ch in chapters:
ch_num_str = str(ch["chapter"]).zfill(2)
filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
write_json(ch, filepath)
print(
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
f"{ch['title']}"
)
# --- Summary ---
total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments)
total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
total_paras = total_intro_paras + total_chapter_paras
print(f"\n{'=' * 60}")
print(f" EXTRACTION SUMMARY")
print(f"{'=' * 60}")
print(f" Intro segments: {len(intro_segments)}")
print(f" Chapters: {len(chapters)}")
print(f" Total paragraphs: {total_paras}")
print(f" Intro: {total_intro_paras}")
print(f" Chapters: {total_chapter_paras}")
if chapters:
print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
print(f" Output directory: {output_base}/")
# --- Samples ---
print(f"\n--- Sample: Front Pages ---")
if intro_segments:
front = intro_segments[0]
for i, p in enumerate(front["paragraphs"][:5]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
if chapters:
print(f"\n--- Sample: Chapter 1 ({chapters[0]['title']}) ---")
ch1 = chapters[0]
for i, p in enumerate(ch1["paragraphs"][:5]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
if chapters and len(chapters) > 1:
print(
f"\n--- Sample: Chapter {chapters[-1]['chapter']} ({chapters[-1]['title']}) ---"
)
last = chapters[-1]
for i, p in enumerate(last["paragraphs"][:5]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
# --- Quality Checks ---
print(f"\n--- Quality Checks ---")
# Check all 6 chapters present
expected_chapters = set(range(1, 7))
actual_chapters = set(ch["chapter"] for ch in chapters)
missing = expected_chapters - actual_chapters
extra = actual_chapters - expected_chapters
if missing:
print(f" ⚠ Missing chapters: {sorted(missing)}")
if extra:
print(f" ⚠ Extra chapters: {sorted(extra)}")
if not missing and not extra:
print(f" ✅ All 6 chapters present")
# Check all 1 intro segment present
expected_intros = {"front-pages"}
actual_intros = {s["slug"] for s in intro_segments}
missing_intros = expected_intros - actual_intros
extra_intros = actual_intros - expected_intros
if missing_intros:
print(f" ⚠ Missing intro segments: {missing_intros}")
if extra_intros:
print(f" ⚠ Extra intro segments: {extra_intros}")
if not missing_intros and not extra_intros:
print(f" ✅ All 1 intro segments present")
# Check empty chapters
empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
if empty_chapters:
print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
else:
print(f" ✅ No empty chapters")
# Check empty intro segments
empty_intros = [s["slug"] for s in intro_segments if len(s["paragraphs"]) == 0]
if empty_intros:
print(f" ⚠ Empty intro segments: {empty_intros}")
else:
print(f" ✅ No empty intro segments")
# Check ScaGoudy decoding
print(f"\n--- ScaGoudy Decoding Check ---")
all_text = ""
for ch in chapters:
all_text += " ".join(ch["paragraphs"][:20])
for seg in intro_segments:
all_text += " ".join(seg["paragraphs"][:10])
# Check for properly decoded words
if "Kṛṣṇa" in all_text:
print(f" ✅ 'Kṛṣṇa' found (ScaGoudy properly decoded)")
else:
print(f" ⚠ 'Kṛṣṇa' not found — ScaGoudy decoding may have failed")
if "Bhagavad-gītā" in all_text:
print(f" ✅ 'Bhagavad-gītā' found (ScaGoudy properly decoded)")
elif "gītā" in all_text or "Gītā" in all_text:
print(f" ✅ 'gītā' found (ScaGoudy properly decoded)")
else:
print(f" ⚠ 'Gītā' not found — ScaGoudy decoding may have failed")
if "Prabhupāda" in all_text:
print(f" ✅ 'Prabhupāda' found (ScaGoudy properly decoded)")
# Check for un-decoded ScaGoudy remnants (common encoded chars)
scagoudy_remnants = 0
for ch in chapters:
for p in ch["paragraphs"]:
if "Kåñëa" in p or "Prabhupäda" in p:
scagoudy_remnants += 1
for seg in intro_segments:
for p in seg["paragraphs"]:
if "Kåñëa" in p or "Prabhupäda" in p:
scagoudy_remnants += 1
if scagoudy_remnants == 0:
print(f" ✅ No ScaGoudy remnants detected")
else:
print(f" ⚠ {scagoudy_remnants} paragraphs with ScaGoudy remnants")
# Check for centered verse blocks
verse_block_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
if '<div style="text-align:center"><i>' in p:
verse_block_count += 1
if verse_block_count > 0:
print(f" ✅ {verse_block_count} centered Sanskrit verse blocks found")
else:
print(f" ⚠ No centered verse blocks found — verse grouping may have failed")
# Check for residual span tags
span_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
if 'class="' in p and "<span" in p:
span_count += 1
for seg in intro_segments:
for p in seg["paragraphs"]:
if 'class="' in p and "<span" in p:
span_count += 1
if span_count == 0:
print(f" ✅ No residual span class artifacts found")
else:
print(f" ⚠ {span_count} paragraphs still contain span class artifacts")
# Check for dash-line paragraphs that shouldn't be there
dash_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
plain = re.sub(r"<[^>]+>", "", p)
if is_dash_line(plain):
dash_count += 1
for seg in intro_segments:
for p in seg["paragraphs"]:
plain = re.sub(r"<[^>]+>", "", p)
if is_dash_line(plain):
dash_count += 1
if dash_count == 0:
print(f" ✅ No dashed-line paragraphs in output")
else:
print(f" ⚠ {dash_count} dashed-line paragraphs found in output")
# Show a verse block sample
print(f"\n--- Sample: Verse Block ---")
for ch in chapters:
for p in ch["paragraphs"]:
if '<div style="text-align:center"><i>' in p:
preview = re.sub(r"<[^>]+>", "", p)[:120]
print(f" {preview}...")
break
else:
continue
break
# Chapter title decoding check
print(f"\n--- Chapter Title Decoding Check ---")
for ch in chapters:
title = ch["title"]
if "å" in title or "ë" in title or "ñ" in title or "ä" in title:
print(f" ⚠ Chapter {ch['chapter']} title has un-decoded ScaGoudy: {title}")
else:
print(f" ✅ Ch {ch['chapter']}: {title}")
print(f"\nDone! ✨")
if __name__ == "__main__":
main()
scripts/hkh_extract.py
+932
View File
@@ -0,0 +1,932 @@
#!/usr/bin/env python3
"""Extract The Kṛṣṇa Consciousness Handbook (1971) from other.html.
The book has 2 intro segments (front-pages, introduction) and 7 chapters.
Main chapter bookmarks (p13 class):
- HKH: Introduction
- HKH 1: A Brief History of ISKCON
- HKH 2: Teachings of ISKCON
- HKH 3: Brief Biography of the Spiritual Master
- HKH 4: Paramparā
- HKH 5: Ministers for Kṛṣṇa Consciousness: ...
- HKH 6: Transcendental Aspects
- HKH 7: The Qualifications and Curriculum of the Divinity Student
Sub-section bookmarks (p49 class):
- HKH 6a–6f within chapter 6
- HKH 7a–7l within chapter 7
- HKH 7m is the END boundary (book catalog, not handbook content)
Output:
lib/jsons/books/hkh/intro.json — Book metadata
lib/jsons/books/hkh/intro/front-pages.json
lib/jsons/books/hkh/intro/introduction.json
lib/jsons/books/hkh/chapters/01.json through 07.json
Usage:
python3 scripts/hkh_extract.py
"""
import json
import os
import re
import sys
from bs4 import BeautifulSoup, Tag
# ---------------------------------------------------------------------------
# ScaGoudy → IAST Unicode mapping
# ---------------------------------------------------------------------------
SCAGOUDY_TO_IAST = {
"ä": "ā",
"Ä": "Ā",
"å": "ṛ",
"Å": "Ṛ",
"ñ": "ṣ",
"Ñ": "Ṣ",
"ö": "ṭ",
"Ö": "Ṭ",
"ë": "ṇ",
"Ë": "Ṇ",
"ò": "ḍ",
"Ò": "Ḍ",
"ç": "ś",
"Ç": "Ś",
"ì": "ṅ",
"ï": "ñ",
"ü": "ū",
"Ü": "Ū",
"é": "ī",
"É": "Ī",
"à": "ṁ",
"ù": "ḥ",
"è": "ṝ",
"û": "ṁl",
"ÿ": "ḷ",
}
def decode_scagoudy(text: str) -> str:
"""Replace ScaGoudy-encoded characters with proper IAST Unicode."""
for src, dst in SCAGOUDY_TO_IAST.items():
text = text.replace(src, dst)
return text
def decode_html_content(tag: Tag) -> str:
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
inner = "".join(str(child) for child in tag.children)
inner = decode_scagoudy(inner)
# Remove span tags with CSS classes (RTF conversion artifacts)
inner = re.sub(
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
)
# Clean up Apple-tab-span leftovers (tab characters)
inner = inner.replace("\t", " ")
# Normalize whitespace (but preserve <br> tags)
inner = re.sub(r"\s+", " ", inner).strip()
return inner
def decode_text(tag: Tag) -> str:
"""Get plain text from a tag with ScaGoudy decoding."""
return decode_scagoudy(tag.get_text().strip())
def get_css_class(tag: Tag) -> str:
"""Get the first CSS class of a tag, or empty string."""
classes = tag.get("class", [])
return classes[0] if classes else ""
def is_empty(tag: Tag) -> bool:
"""Check if a tag is empty or whitespace-only."""
text = tag.get_text().strip()
return not text or text == "\xa0"
def is_dash_line(text: str) -> bool:
"""Return True if text is only dashes, spaces, and/or whitespace."""
cleaned = text.strip()
if not cleaned:
return False
return all(c in "-—–\u2014\u2013 \t" for c in cleaned)
# ---------------------------------------------------------------------------
# CSS class categorization for HKH
# ---------------------------------------------------------------------------
# Book title header class
BOOK_HEADER_CLASS = "p12"
# Chapter bookmark class (main chapters)
CHAPTER_BOOKMARK_CLASS = "p13"
# Sub-section bookmark class
SUBSECTION_BOOKMARK_CLASS = "p49"
# Sanskrit verse pāda classes (italic, centered)
VERSE_PADA_CLASSES = frozenset(["p53"])
# Classes to skip entirely
SKIP_CLASSES = frozenset(["p9", "p70"])
# Body paragraph classes — be permissive, handle everything not otherwise special
BODY_CLASSES = frozenset(
[
"p6",
"p67",
"p104",
"p101",
"p46",
"p48",
"p103",
"p105",
"p36",
"p37",
"p24",
"p44",
"p28",
"p109",
"p110",
"p111",
"p112",
"p113",
"p114",
"p115",
"p95",
"p82",
"p25",
"p118",
]
)
# ---------------------------------------------------------------------------
# Extraction helpers
# ---------------------------------------------------------------------------
def find_hkh_boundaries(all_paragraphs: list) -> tuple:
"""Find the start and end paragraph indices of the HKH content.
Start: <p class="p12"> containing "Krsna Consciousness Handbook" and "1971"
End: <p class="p49"> containing "HKH 7m:" (book catalog section)
"""
start_idx = None
end_idx = len(all_paragraphs)
for i, p in enumerate(all_paragraphs):
cls = get_css_class(p)
text = p.get_text().strip()
# Find start
if (
cls == BOOK_HEADER_CLASS
and "Krsna Consciousness Handbook" in text
and "1971" in text
):
start_idx = i
# Find end: p49 containing "HKH 7m:"
if start_idx is not None and cls == SUBSECTION_BOOKMARK_CLASS:
if re.search(r"HKH\s+7m:", text):
end_idx = i
break
if start_idx is None:
print(
"ERROR: Could not find 'The Krsna Consciousness Handbook (1971)' header in other.html!"
)
sys.exit(1)
return start_idx, end_idx
def find_chapter_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
"""Find all HKH chapter/intro bookmark tags (p13) in the HKH range.
Returns list of (paragraph_index, chapter_number_or_None, chapter_title, bookmark_type).
bookmark_type is "intro" for "HKH: Introduction" and "chapter" for numbered chapters.
"""
bookmarks = []
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
if cls != CHAPTER_BOOKMARK_CLASS:
continue
text = decode_text(p)
# Check for introduction bookmark
if re.match(r"HKH:\s+Introduction", text.strip()):
bookmarks.append((i, None, "Introduction", "intro"))
continue
# Check for numbered chapter bookmark (handle multi-line titles)
# Collapse newlines for matching
collapsed = re.sub(r"\s+", " ", text.strip())
m = re.match(r"HKH\s+(\d+):\s+(.+)", collapsed)
if m:
ch_num = int(m.group(1))
ch_title = m.group(2).strip()
bookmarks.append((i, ch_num, ch_title, "chapter"))
return bookmarks
def find_subsection_bookmarks(
all_paragraphs: list, start_idx: int, end_idx: int
) -> list:
"""Find all HKH sub-section bookmark tags (p49) in a range.
Returns list of (paragraph_index, subsection_code, subsection_title).
e.g. (12149, "6a", "Śikhā")
"""
bookmarks = []
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
if cls != SUBSECTION_BOOKMARK_CLASS:
continue
text = decode_text(p)
# Match HKH Nx: Title patterns
collapsed = re.sub(r"\s+", " ", text.strip())
m = re.match(r"HKH\s+(\d+[a-z]):\s*(.*)", collapsed)
if m:
code = m.group(1)
title = m.group(2).strip()
# Clean up title — remove surrounding <i> artifacts and leading/trailing whitespace
title = re.sub(r"</?i>", "", title).strip()
bookmarks.append((i, code, title))
return bookmarks
def extract_paragraphs(
all_paragraphs: list,
start_idx: int,
end_idx: int,
is_front_pages: bool = False,
subsection_bookmarks: list = None,
) -> list:
"""Extract paragraphs for a section (chapter or intro segment).
Handles all CSS classes within the HKH content.
Groups consecutive p53 verse pādas into centered italic divs.
Sub-section bookmarks (p49) are converted to bold heading paragraphs.
"""
result = []
verse_accumulator = []
if subsection_bookmarks is None:
subsection_bookmarks = []
# Build a set of subsection bookmark indices for quick lookup
subsection_indices = {b[0] for b in subsection_bookmarks}
def flush_verse():
if verse_accumulator:
joined = "<br>".join(verse_accumulator)
result.append(f'<div style="text-align:center"><i>{joined}</i></div>')
verse_accumulator.clear()
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
# Skip empty paragraphs
if is_empty(p):
continue
# Skip noise classes
if cls in SKIP_CLASSES:
continue
# Skip book title header (p12)
if cls == BOOK_HEADER_CLASS:
continue
# Skip chapter bookmark tags (p13) — boundaries only
if cls == CHAPTER_BOOKMARK_CLASS:
continue
# Handle sub-section bookmarks (p49) — convert to bold headings
if cls == SUBSECTION_BOOKMARK_CLASS:
flush_verse()
if i in subsection_indices:
# Find the matching subsection bookmark
for bm in subsection_bookmarks:
if bm[0] == i:
sub_title = bm[2]
if sub_title:
result.append(f"<b>{sub_title}</b>")
break
# Skip p49 tags that aren't in subsection_bookmarks (shouldn't happen in range)
continue
# Check for dash lines — skip them
text = decode_text(p)
if is_dash_line(text):
continue
# Sanskrit verse pādas (p53) — accumulate into centered block
if cls in VERSE_PADA_CLASSES:
html = decode_html_content(p)
if html:
# Remove inner <i> tags before re-wrapping
stripped = re.sub(r"</?i>", "", html)
verse_accumulator.append(stripped)
continue
# Non-verse paragraph — flush any accumulated verse first
flush_verse()
# Section headings (p31, p27) — include as paragraphs
if cls in ("p31", "p27"):
html = decode_html_content(p)
if html:
result.append(html)
continue
# Misc text (p19, p21) — include
if cls in ("p19", "p21"):
html = decode_html_content(p)
if html:
result.append(html)
continue
# Contents heading (p69) — include
if cls == "p69":
html = decode_html_content(p)
if html:
result.append(html)
continue
# Contact info (p102) — include in front pages
if cls == "p102":
html = decode_html_content(p)
if html:
result.append(html)
continue
# Body paragraphs and everything else — be permissive
html = decode_html_content(p)
if html and not is_dash_line(text):
# Detect Sanskrit verses disguised as body paragraphs:
# they start with <i> and contain <br> (multi-line italic = verse)
stripped_html = html.strip()
if stripped_html.startswith("<i>") and "<br" in stripped_html:
# This is a verse — wrap in centered italic div
inner = re.sub(r"</?i>", "", stripped_html)
result.append(f'<div style="text-align:center"><i>{inner}</i></div>')
else:
result.append(html)
# Flush any trailing verse
flush_verse()
return result
# ---------------------------------------------------------------------------
# Clean chapter titles
# ---------------------------------------------------------------------------
# Map of chapter number to clean Unicode title
CLEAN_TITLES = {
1: "A Brief History of ISKCON",
2: "Teachings of ISKCON",
3: "Brief Biography of the Spiritual Master",
4: "Paramparā",
5: "Ministers for Kṛṣṇa Consciousness",
6: "Transcendental Aspects",
7: "The Qualifications and Curriculum of the Divinity Student",
}
def clean_chapter_title(ch_num: int, raw_title: str) -> str:
"""Return a clean Unicode title for the chapter."""
if ch_num in CLEAN_TITLES:
return CLEAN_TITLES[ch_num]
# Fallback: decode ScaGoudy from raw title
return decode_scagoudy(raw_title)
# ---------------------------------------------------------------------------
# Main extraction logic
# ---------------------------------------------------------------------------
def extract_hkh(soup: BeautifulSoup) -> dict:
"""Extract the entire Kṛṣṇa Consciousness Handbook (1971) from the parsed HTML.
Chapter 6 ("Transcendental Aspects") sub-sections (Śikhā, Beads, Ārātrika,
Garlands, Deities, Diet) are promoted to their own chapters (6-11), and the
old chapter 7 becomes chapter 12. This gives 12 total chapters.
Returns a dict with:
"intro_segments": list of intro segment dicts
"chapters": list of chapter dicts
"chapter_listing": list of {number, title} for intro.json
"""
all_paragraphs = list(soup.find_all("p"))
# Step 1: Find HKH boundaries
hkh_start, hkh_end = find_hkh_boundaries(all_paragraphs)
print(f" HKH content: paragraphs {hkh_start} to {hkh_end}")
print(f" Total paragraphs in range: {hkh_end - hkh_start}")
# Step 2: Find all chapter bookmarks
chapter_bookmarks = find_chapter_bookmarks(all_paragraphs, hkh_start, hkh_end)
print(f" Found {len(chapter_bookmarks)} HKH chapter bookmarks")
for pos, ch_num, ch_title, bm_type in chapter_bookmarks:
if bm_type == "intro":
print(f" [introduction] paragraph {pos}: {ch_title}")
else:
print(f" [chapter {ch_num}] paragraph {pos}: {ch_title}")
# Step 3: Find all sub-section bookmarks in the full range
all_subsections = find_subsection_bookmarks(all_paragraphs, hkh_start, hkh_end)
print(f" Found {len(all_subsections)} sub-section bookmarks")
# Step 4: Separate intro bookmark from chapter bookmarks
intro_bookmark = None
numbered_bookmarks = []
for bm in chapter_bookmarks:
if bm[3] == "intro":
intro_bookmark = bm
else:
numbered_bookmarks.append(bm)
# Step 5: Extract front pages
intro_segments = []
if intro_bookmark:
front_pages_end = intro_bookmark[0]
elif numbered_bookmarks:
front_pages_end = numbered_bookmarks[0][0]
else:
front_pages_end = hkh_end
front_pages_paragraphs = extract_paragraphs(
all_paragraphs, hkh_start, front_pages_end, is_front_pages=True
)
intro_segments.append(
{
"slug": "front-pages",
"title": "Front Pages",
"type": "intro-segment",
"book": "hkh",
"order": 1,
"paragraphs": front_pages_paragraphs,
}
)
# Step 6: Extract introduction segment
if intro_bookmark:
intro_start = intro_bookmark[0] + 1
if numbered_bookmarks:
intro_end = numbered_bookmarks[0][0]
else:
intro_end = hkh_end
intro_paragraphs = extract_paragraphs(all_paragraphs, intro_start, intro_end)
intro_segments.append(
{
"slug": "introduction",
"title": "Introduction",
"type": "intro-segment",
"book": "hkh",
"order": 2,
"paragraphs": intro_paragraphs,
}
)
# Step 7: Build the effective chapter list.
#
# Chapters 1-5 stay as-is from p13 bookmarks.
# Chapter 6 ("Transcendental Aspects") is SPLIT: its p49 sub-sections
# (6a=Śikhā, 6b=Beads, 6c=Ārātrika, 6d=Garlands, 6e=Deities, 6f=Diet)
# each become their own chapter (renumbered 6-11).
# Old chapter 7 is also SPLIT: 7a+7b stay together as "The Qualifications
# and Curriculum of the Divinity Student" (chapter 12), and 7c-7l each
# become their own chapters (13-21). Total: 21 chapters.
# Identify the chapter-6 and chapter-7 bookmarks
ch6_bm = None
ch7_bm = None
for bm in numbered_bookmarks:
if bm[1] == 6:
ch6_bm = bm
elif bm[1] == 7:
ch7_bm = bm
# Find chapter 6's sub-section bookmarks (6a through 6f)
ch6_subsections = []
if ch6_bm and ch7_bm:
ch6_range_start = ch6_bm[0]
ch6_range_end = ch7_bm[0]
ch6_subsections = [
s for s in all_subsections if ch6_range_start <= s[0] < ch6_range_end
]
# Find chapter 7's sub-section bookmarks (7a through 7l)
ch7_subsections = []
if ch7_bm:
ch7_range_start = ch7_bm[0]
ch7_range_end = hkh_end
ch7_subsections = [
s for s in all_subsections if ch7_range_start <= s[0] < ch7_range_end
]
# Sub-sections to skip entirely:
# 7a = "Entrance Qualifications", 7b = "Curriculum" (administrative)
# 7h = "ISKCON Press" (organizational, not spiritual content)
CH7_SKIP_CODES = {"7a", "7b", "7h"}
ch7_split = [
s
for s in ch7_subsections
if s[1].split(":")[0].strip().replace("HKH ", "") not in CH7_SKIP_CODES
]
chapters = []
output_ch_num = 0
for idx, (ch_pos, ch_num, ch_title, _) in enumerate(numbered_bookmarks):
content_start = ch_pos + 1
if idx + 1 < len(numbered_bookmarks):
content_end = numbered_bookmarks[idx + 1][0]
else:
content_end = hkh_end
# --- Special handling for chapter 6: split into sub-section chapters ---
if ch_num == 6 and ch6_subsections:
for sub_idx, (sub_pos, sub_code, sub_title) in enumerate(ch6_subsections):
output_ch_num += 1
sub_start = sub_pos + 1
if sub_idx + 1 < len(ch6_subsections):
sub_end = ch6_subsections[sub_idx + 1][0]
else:
sub_end = content_end
paragraphs = extract_paragraphs(all_paragraphs, sub_start, sub_end)
clean_title = decode_scagoudy(sub_title).strip()
chapters.append(
{
"book": "hkh",
"chapter": output_ch_num,
"title": clean_title,
"type": "chapter",
"paragraphs": paragraphs,
}
)
continue
# --- Special handling for chapter 7: skip 7a+7b (curriculum), split 7c-7l ---
if ch_num == 7 and ch7_split:
# Skip excluded sub-sections (7a, 7b, 7h).
# Only emit remaining sub-sections as separate chapters.
for sub_idx, (sub_pos, sub_code, sub_title) in enumerate(ch7_split):
output_ch_num += 1
sub_start = sub_pos + 1
if sub_idx + 1 < len(ch7_split):
sub_end = ch7_split[sub_idx + 1][0]
else:
sub_end = content_end
paragraphs = extract_paragraphs(all_paragraphs, sub_start, sub_end)
clean_title = decode_scagoudy(sub_title).strip()
chapters.append(
{
"book": "hkh",
"chapter": output_ch_num,
"title": clean_title,
"type": "chapter",
"paragraphs": paragraphs,
}
)
continue
# --- Normal chapters (1-5) ---
output_ch_num += 1
paragraphs = extract_paragraphs(
all_paragraphs,
content_start,
content_end,
)
clean_title = clean_chapter_title(ch_num, ch_title)
chapters.append(
{
"book": "hkh",
"chapter": output_ch_num,
"title": clean_title,
"type": "chapter",
"paragraphs": paragraphs,
}
)
# Step 8: Build chapter listing for intro.json
chapter_listing = [
{"number": ch["chapter"], "title": ch["title"]} for ch in chapters
]
return {
"intro_segments": intro_segments,
"chapters": chapters,
"chapter_listing": chapter_listing,
}
# ---------------------------------------------------------------------------
# Output
# ---------------------------------------------------------------------------
def write_json(data, filepath):
"""Write data as formatted JSON to filepath, creating directories as needed."""
os.makedirs(os.path.dirname(filepath), exist_ok=True)
with open(filepath, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False, indent=2)
def main():
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
html_path = os.path.join(base_dir, "public", "html", "other.html")
output_base = os.path.join(base_dir, "lib", "jsons", "books", "hkh")
if not os.path.exists(html_path):
print(f"ERROR: {html_path} not found!")
sys.exit(1)
print(f"Reading {html_path}...")
with open(html_path, "r", encoding="utf-8") as f:
soup = BeautifulSoup(f, "html.parser")
print("Extracting The Kṛṣṇa Consciousness Handbook (1971)...\n")
result = extract_hkh(soup)
intro_segments = result["intro_segments"]
chapters = result["chapters"]
chapter_listing = result["chapter_listing"]
# --- Write intro.json ---
intro_meta = {
"book": "hkh",
"title": "The Kṛṣṇa Consciousness Handbook",
"author": "International Society for Krishna Consciousness",
"year": 1971,
"description": "The first annual handbook of ISKCON, covering the history, teachings, curriculum, and practices of Kṛṣṇa consciousness.",
"totalChapters": 7,
"chapters": chapter_listing,
}
intro_path = os.path.join(output_base, "intro.json")
write_json(intro_meta, intro_path)
print(f"\n Written: {intro_path}")
# --- Write intro segments ---
print(f"\n--- Intro Segments ---")
for seg in intro_segments:
filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
write_json(seg, filepath)
print(
f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}"
)
# --- Write chapters ---
print(f"\n--- Chapters ---")
for ch in chapters:
ch_num_str = str(ch["chapter"]).zfill(2)
filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
write_json(ch, filepath)
print(
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
f"{ch['title']}"
)
# --- Summary ---
total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments)
total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
total_paras = total_intro_paras + total_chapter_paras
print(f"\n{'=' * 60}")
print(f" EXTRACTION SUMMARY")
print(f"{'=' * 60}")
print(f" Intro segments: {len(intro_segments)}")
print(f" Chapters: {len(chapters)}")
print(f" Total paragraphs: {total_paras}")
print(f" Intro: {total_intro_paras}")
print(f" Chapters: {total_chapter_paras}")
if chapters:
print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
print(f" Output directory: {output_base}/")
# --- Samples ---
print(f"\n--- Sample: Front Pages ---")
if intro_segments:
front = intro_segments[0]
for i, p in enumerate(front["paragraphs"][:5]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
if len(intro_segments) > 1:
print(f"\n--- Sample: Introduction ---")
intro_seg = intro_segments[1]
for i, p in enumerate(intro_seg["paragraphs"][:5]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
if chapters:
print(f"\n--- Sample: Chapter 1 ({chapters[0]['title']}) ---")
ch1 = chapters[0]
for i, p in enumerate(ch1["paragraphs"][:5]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
if chapters and len(chapters) > 1:
print(
f"\n--- Sample: Chapter {chapters[-1]['chapter']} ({chapters[-1]['title']}) ---"
)
last = chapters[-1]
for i, p in enumerate(last["paragraphs"][:5]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
# --- Quality Checks ---
print(f"\n--- Quality Checks ---")
# Check all 7 chapters present
expected_chapters = set(range(1, 8))
actual_chapters = set(ch["chapter"] for ch in chapters)
missing = expected_chapters - actual_chapters
extra = actual_chapters - expected_chapters
if missing:
print(f" ⚠ Missing chapters: {sorted(missing)}")
if extra:
print(f" ⚠ Extra chapters: {sorted(extra)}")
if not missing and not extra:
print(f" ✅ All 7 chapters present")
# Check all 2 intro segments present
expected_intros = {"front-pages", "introduction"}
actual_intros = {s["slug"] for s in intro_segments}
missing_intros = expected_intros - actual_intros
extra_intros = actual_intros - expected_intros
if missing_intros:
print(f" ⚠ Missing intro segments: {missing_intros}")
if extra_intros:
print(f" ⚠ Extra intro segments: {extra_intros}")
if not missing_intros and not extra_intros:
print(f" ✅ All 2 intro segments present")
# Check empty chapters
empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
if empty_chapters:
print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
else:
print(f" ✅ No empty chapters")
# Check empty intro segments
empty_intros = [s["slug"] for s in intro_segments if len(s["paragraphs"]) == 0]
if empty_intros:
print(f" ⚠ Empty intro segments: {empty_intros}")
else:
print(f" ✅ No empty intro segments")
# Check ScaGoudy decoding
print(f"\n--- ScaGoudy Decoding Check ---")
all_text = ""
for ch in chapters:
all_text += " ".join(ch["paragraphs"][:20])
for seg in intro_segments:
all_text += " ".join(seg["paragraphs"][:10])
# Check for properly decoded words
if "Kṛṣṇa" in all_text:
print(f" ✅ 'Kṛṣṇa' found (ScaGoudy properly decoded)")
else:
print(f" ⚠ 'Kṛṣṇa' not found — ScaGoudy decoding may have failed")
if "Bhagavad-gītā" in all_text:
print(f" ✅ 'Bhagavad-gītā' found (ScaGoudy properly decoded)")
elif "gītā" in all_text or "Gītā" in all_text:
print(f" ✅ 'gītā' found (ScaGoudy properly decoded)")
else:
print(f" ⚠ 'Gītā' not found — ScaGoudy decoding may have failed")
if "Prabhupāda" in all_text:
print(f" ✅ 'Prabhupāda' found (ScaGoudy properly decoded)")
if "ISKCON" in all_text:
print(f" ✅ 'ISKCON' found")
# Check for un-decoded ScaGoudy remnants (common encoded chars)
scagoudy_remnants = 0
for ch in chapters:
for p in ch["paragraphs"]:
if "Kåñëa" in p or "Prabhupäda" in p:
scagoudy_remnants += 1
for seg in intro_segments:
for p in seg["paragraphs"]:
if "Kåñëa" in p or "Prabhupäda" in p:
scagoudy_remnants += 1
if scagoudy_remnants == 0:
print(f" ✅ No ScaGoudy remnants detected")
else:
print(f" ⚠ {scagoudy_remnants} paragraphs with ScaGoudy remnants")
# Check for centered verse blocks
verse_block_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
if '<div style="text-align:center"><i>' in p:
verse_block_count += 1
if verse_block_count > 0:
print(f" ✅ {verse_block_count} centered Sanskrit verse blocks found")
else:
print(f" ⚠ No centered verse blocks found — verse grouping may have failed")
# Check for residual span tags
span_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
if 'class="' in p and "<span" in p:
span_count += 1
for seg in intro_segments:
for p in seg["paragraphs"]:
if 'class="' in p and "<span" in p:
span_count += 1
if span_count == 0:
print(f" ✅ No residual span class artifacts found")
else:
print(f" ⚠ {span_count} paragraphs still contain span class artifacts")
# Check for dash-line paragraphs that shouldn't be there
dash_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
plain = re.sub(r"<[^>]+>", "", p)
if is_dash_line(plain):
dash_count += 1
for seg in intro_segments:
for p in seg["paragraphs"]:
plain = re.sub(r"<[^>]+>", "", p)
if is_dash_line(plain):
dash_count += 1
if dash_count == 0:
print(f" ✅ No dashed-line paragraphs in output")
else:
print(f" ⚠ {dash_count} dashed-line paragraphs found in output")
# Show a verse block sample
print(f"\n--- Sample: Verse Block ---")
for ch in chapters:
for p in ch["paragraphs"]:
if '<div style="text-align:center"><i>' in p:
preview = re.sub(r"<[^>]+>", "", p)[:120]
print(f" {preview}...")
break
else:
continue
break
# Check sub-section headings in chapters 6 and 7
print(f"\n--- Sub-section Heading Check ---")
for ch in chapters:
if ch["chapter"] in (6, 7):
bold_headings = [
p
for p in ch["paragraphs"]
if p.startswith("<b>")
and p.endswith("</b>")
and len(re.sub(r"<[^>]+>", "", p)) < 80
]
print(
f" Chapter {ch['chapter']}: {len(bold_headings)} sub-section headings found"
)
for h in bold_headings[:5]:
preview = re.sub(r"<[^>]+>", "", h)
print(f" → {preview}")
# Chapter title decoding check
print(f"\n--- Chapter Title Decoding Check ---")
for ch in chapters:
title = ch["title"]
if "å" in title or "ë" in title or "ñ" in title or "ä" in title:
print(f" ⚠ Chapter {ch['chapter']} title has un-decoded ScaGoudy: {title}")
else:
print(f" ✅ Ch {ch['chapter']}: {title}")
print(f"\nDone! ✨")
if __name__ == "__main__":
main()
scripts/iso_extract.py
+817
View File
@@ -0,0 +1,817 @@
#!/usr/bin/env python3
"""Extract Śrī Īśopaniṣad (1969 Edition) verses from other.html into structured JSON files."""
import json
import os
import re
import sys
from bs4 import BeautifulSoup, Tag
from indic_transliteration import sanscript
from indic_transliteration.sanscript import transliterate
# --- ScaGoudy → IAST Unicode mapping ---
SCAGOUDY_TO_IAST = {
"ä": "ā",
"Ä": "Ā",
"å": "ṛ",
"Å": "Ṛ",
"ñ": "ṣ",
"Ñ": "Ṣ",
"ö": "ṭ",
"Ö": "Ṭ",
"ë": "ṇ",
"Ë": "Ṇ",
"ò": "ḍ",
"Ò": "Ḍ",
"ç": "ś",
"Ç": "Ś",
"ì": "ṅ",
"ï": "ñ",
"ü": "ū",
"Ü": "Ū",
"é": "ī",
"É": "Ī",
"à": "ṁ",
"ù": "ḥ",
"è": "ṝ", # long vocalic r (U+1E5D)
"û": "ṁl", # anusvara + l (ScaGoudy ligature glyph)
"ÿ": "ḷ", # vocalic l (U+1E37)
}
# --- Mantra number to title mapping ---
MANTRA_TITLES = {
0: "Invocation",
1: "Mantra One",
2: "Mantra Two",
3: "Mantra Three",
4: "Mantra Four",
5: "Mantra Five",
6: "Mantra Six",
7: "Mantra Seven",
8: "Mantra Eight",
9: "Mantra Nine",
10: "Mantra Ten",
11: "Mantra Eleven",
12: "Mantra Twelve",
13: "Mantra Thirteen",
14: "Mantra Fourteen",
15: "Mantra Fifteen",
16: "Mantra Sixteen",
17: "Mantra Seventeen",
18: "Mantra Eighteen",
}
# --- CSS class constants ---
BOOK_HEADER_CLASS = "p12"
BOOKMARK_CLASS = "p13"
VERSE_BOOKMARK_CLASS = "p49"
VERSE_HEADING_CLASS = "p50"
DEVANAGARI_CLASS = "p51" # RM Devanagari — SKIP
TEXT_LABEL_CLASS = "p52"
INVOCATION_TRANSLIT_CLASS = "p53"
PADA_TRANSLIT_CLASS = "p5"
SYNONYM_CLASSES = ("p54", "p59")
TRANSLATION_LABEL_CLASS = "p55"
TRANSLATION_TEXT_CLASS = "p56"
PURPORT_LABEL_CLASS = "p57"
PURPORT_CLASSES = ("p48", "p58", "p23", "p60", "p63", "p46", "p47", "p61", "p62")
SECTION_HEADING_CLASS = "p31"
BODY_CLASSES = ("p6", "p45")
SKIP_CLASSES = ("p9", "p15", "p19")
def decode_scagoudy(text: str) -> str:
"""Convert ScaGoudy-encoded text to proper IAST Unicode."""
for src, dst in SCAGOUDY_TO_IAST.items():
text = text.replace(src, dst)
return text
def decode_html_content(tag: Tag) -> str:
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
inner = "".join(str(child) for child in tag.children)
inner = decode_scagoudy(inner)
# Remove span tags with CSS classes (artifacts from RTF conversion)
inner = re.sub(
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
)
# Clean up Apple-tab-span leftovers (tab characters)
inner = inner.replace("\t", " ")
# Normalize whitespace
inner = re.sub(r"\s+", " ", inner).strip()
return inner
def decode_text(tag: Tag) -> str:
"""Extract plain text from a tag and decode ScaGoudy."""
text = tag.get_text()
return decode_scagoudy(text).strip()
def normalize_devanagari_anusvara(text: str) -> str:
"""Replace explicit class nasals before same-class consonants with anusvāra.
Only normalize ङ् and ञ् — these are almost always written with anusvāra.
Keep ण्ड, न्त, म्ब etc. as explicit conjuncts (standard spelling).
"""
replacements = [
# ङ् before velars (क, ख, ग, घ)
(r"ङ्([कखगघ])", r"ं\1"),
# ञ् before palatals (च, छ, ज, झ)
(r"ञ्([चछजझ])", r"ं\1"),
]
for pattern, repl in replacements:
text = re.sub(pattern, repl, text)
return text
def join_virama_words(text: str) -> str:
"""Join Devanagari words where the previous word ends with virāma (्).
Handles both virāma+consonant and virāma+vowel cases.
Uses [ \\t]+ instead of \\s+ to avoid joining across newlines.
"""
# Join virāma + space + consonant (क U+0915 through ह U+0939)
text = re.sub(r"्[ \t]+([क-ह])", r"्\1", text)
# Join virāma + space + independent vowel
_VOWEL_TO_MATRA = {
"अ": "", # inherent vowel — just drop virāma and the अ
"आ": "ा",
"इ": "ि",
"ई": "ी",
"उ": "ु",
"ऊ": "ू",
"ऋ": "ृ",
"ॠ": "ॄ",
"ऌ": "ॢ",
"ॡ": "ॣ",
"ए": "े",
"ऐ": "ै",
"ओ": "ो",
"औ": "ौ",
}
def _virama_vowel_replace(m: re.Match[str]) -> str:
vowel = m.group(1)
return _VOWEL_TO_MATRA.get(vowel, vowel)
text = re.sub(r"्[ \t]+([अ-औॠॡ])", _virama_vowel_replace, text)
return text
def iast_to_devanagari(iast_text: str) -> str:
"""Convert IAST transliteration to Devanagari script."""
lines = iast_text.split("\n")
result_lines = []
for line in lines:
# Remove hyphens between words for Devanagari
clean = line.replace("-", "")
# Normalize ṁ (U+1E41) to ṃ (U+1E43) for proper anusvāra
clean = clean.replace("ṁ", "ṃ")
deva = transliterate(clean, sanscript.IAST, sanscript.DEVANAGARI)
deva = normalize_devanagari_anusvara(deva)
deva = join_virama_words(deva)
result_lines.append(deva)
return "\n".join(result_lines)
def parse_synonyms(text: str) -> list:
"""Parse synonym text into structured array.
Input format: "word—meaning; word—meaning; word—meaning"
"""
result = []
entries = text.split(";")
for entry in entries:
entry = entry.strip()
if not entry:
continue
if "—" in entry:
parts = entry.split("—", 1)
elif "--" in entry:
parts = entry.split("--", 1)
else:
continue
if len(parts) == 2:
word = parts[0].strip()
meaning = parts[1].strip()
# Clean word: remove trailing/leading punctuation
word = word.strip("., ")
if word and meaning:
result.append({"word": word, "meanings": [meaning]})
return result
def pair_padas(padas: list) -> str:
"""Pair pādas into verse lines (2 pādas per line).
A śloka has 4 pādas displayed as 2 lines. If there's an odd number,
the first pāda stands alone (usually a speaker attribution).
"""
if not padas:
return ""
lines = []
i = 0
# If odd number of pādas, first one is standalone
if len(padas) % 2 == 1:
lines.append(padas[0])
i = 1
# Pair remaining pādas
while i < len(padas) - 1:
lines.append(padas[i] + " " + padas[i + 1])
i += 2
# Handle any trailing single pāda
if i < len(padas):
lines.append(padas[i])
return "\n".join(lines)
def get_css_class(tag: Tag) -> str:
"""Get the primary CSS class of a tag."""
classes = tag.get("class", [])
return classes[0] if classes else ""
def is_empty(tag: Tag) -> bool:
"""Check if a tag is empty or whitespace-only."""
text = tag.get_text().strip()
return not text or text == "\xa0"
def is_dashed_line(text: str) -> bool:
"""Check if text is a dashed-line separator."""
clean = text.strip().replace("-", "").replace("—", "").strip()
return clean == ""
def parse_verse_bookmark(text: str) -> int | None:
"""Parse a verse bookmark like 'Iso Invocation-1969' or 'Iso 1-1969'.
Returns mantra number (0 for Invocation, 1-18 for mantras), or None.
"""
text = text.strip()
if text == "Iso Invocation-1969":
return 0
m = re.match(r"Iso\s+(\d+)-1969", text)
if m:
return int(m.group(1))
return None
# ---------------------------------------------------------------------------
# Boundary detection
# ---------------------------------------------------------------------------
def find_iso_boundaries(all_paragraphs: list) -> tuple[int, int]:
"""Find the start and end paragraph indices of the Iso content.
Start: <p class="p12"> containing "Īśopaniṣad" (ScaGoudy encoded)
End: next <p class="p12"> after start (start of next book), or end of file
"""
start_idx = None
end_idx = len(all_paragraphs)
for i, p in enumerate(all_paragraphs):
cls = get_css_class(p)
if cls != BOOK_HEADER_CLASS:
continue
text = p.get_text().strip()
# The ScaGoudy-encoded title contains "Éçopaniñad"
if "opani" in text.lower() and "1969" in text:
start_idx = i
elif start_idx is not None:
# Next p12 after Iso start = end of Iso
end_idx = i
break
if start_idx is None:
print(
"ERROR: Could not find 'Śrī Īśopaniṣad - 1969 Edition' header in other.html!"
)
sys.exit(1)
return start_idx, end_idx
def find_section_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> dict:
"""Find the key p13 bookmarks for Iso section boundaries.
Returns dict with bookmark text -> paragraph index.
"""
bookmarks = {}
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
if cls == BOOKMARK_CLASS:
text = decode_text(p)
if "Iso" in text and "1969" in text:
bookmarks[text] = i
return bookmarks
def find_verse_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
"""Find all p49 verse bookmark tags in the Iso range.
Returns list of (paragraph_index, mantra_number).
"""
bookmarks = []
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
if cls == VERSE_BOOKMARK_CLASS:
text = decode_text(p)
mantra_num = parse_verse_bookmark(text)
if mantra_num is not None:
bookmarks.append((i, mantra_num))
return bookmarks
# ---------------------------------------------------------------------------
# Intro segment extraction
# ---------------------------------------------------------------------------
def extract_intro_segment(
all_paragraphs: list, start_idx: int, end_idx: int
) -> list[str]:
"""Extract paragraphs for an intro segment (front pages or about the author).
Decodes ScaGoudy, preserves HTML formatting, skips dashed lines and empty paragraphs.
"""
result = []
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
if is_empty(p):
continue
# Skip bookmark and header classes
if cls in (BOOKMARK_CLASS, BOOK_HEADER_CLASS, VERSE_BOOKMARK_CLASS):
continue
# Skip noise classes
if cls in SKIP_CLASSES:
continue
html = decode_html_content(p)
if not html:
continue
# Skip dashed-line separators
plain = re.sub(r"<[^>]+>", "", html).strip()
if is_dashed_line(plain):
continue
result.append(html)
return result
# ---------------------------------------------------------------------------
# Verse extraction state machine
# ---------------------------------------------------------------------------
def extract_verses(all_paragraphs: list, verses_start: int, verses_end: int) -> list:
"""Extract verses from the Iso Texts section using a state machine.
Returns list of verse dicts.
"""
verses = []
current_verse = None
state = "IDLE"
for i in range(verses_start, verses_end):
p = all_paragraphs[i]
cls = get_css_class(p)
# --- Verse bookmark (p49) ---
if cls == VERSE_BOOKMARK_CLASS:
# Finalize previous verse if any
if current_verse and state != "IDLE":
_finalize_verse(current_verse)
verses.append(current_verse)
text = decode_text(p)
mantra_num = parse_verse_bookmark(text)
if mantra_num is not None:
title = MANTRA_TITLES.get(mantra_num, f"Mantra {mantra_num}")
if mantra_num == 0:
reference = "Invocation"
else:
reference = f"Īśo {mantra_num}"
current_verse = {
"book": "iso",
"book_title": "Śrī Īśopaniṣad",
"mantra": mantra_num,
"title": title,
"reference": reference,
"devanagari": "",
"transliteration": "",
"_padas": [],
"synonyms": [],
"translation": "",
"purport": [],
}
state = "VERSE_START"
continue
if not current_verse:
continue
# --- Verse heading (p50) ---
if cls == VERSE_HEADING_CLASS:
# Just confirms the mantra title; skip
continue
# --- RM Devanagari (p51) — SKIP ---
if cls == DEVANAGARI_CLASS:
continue
# --- TEXT label (p52) — skip ---
if cls == TEXT_LABEL_CLASS:
state = "TRANSLITERATION"
continue
# --- Invocation transliteration (p53) — single <p> with <br> ---
if cls == INVOCATION_TRANSLIT_CLASS and current_verse["mantra"] == 0:
state = "TRANSLITERATION"
# Split on <br> tags to get individual pādas
# Get inner HTML and split on <br> variants
inner_html = "".join(str(child) for child in p.children)
# Remove <i> tags
inner_html = re.sub(r"</?i>", "", inner_html)
# Split on <br> or <br/> or <br />
raw_padas = re.split(r"<br\s*/?>", inner_html)
for pada in raw_padas:
# Decode ScaGoudy on plain text
pada_text = BeautifulSoup(pada, "html.parser").get_text()
pada_text = decode_scagoudy(pada_text).strip()
# Remove trailing period
pada_text = pada_text.rstrip(".")
if pada_text:
current_verse["_padas"].append(pada_text)
continue
# --- Pāda transliteration lines (p5) ---
if cls == PADA_TRANSLIT_CLASS and state in (
"VERSE_START",
"TRANSLITERATION",
):
state = "TRANSLITERATION"
line = decode_text(p)
# Remove trailing period
line = line.rstrip(".")
# Strip leading/trailing whitespace
line = line.strip()
if line:
current_verse["_padas"].append(line)
continue
# --- Synonyms (p54, p59) ---
if cls in SYNONYM_CLASSES and current_verse:
state = "SYNONYMS"
syn_text = decode_text(p)
# Remove leading/trailing italic markers that might come through
syn_text = syn_text.strip()
parsed = parse_synonyms(syn_text)
current_verse["synonyms"].extend(parsed)
continue
# --- TRANSLATION label (p55) ---
if cls == TRANSLATION_LABEL_CLASS and current_verse:
state = "TRANSLATION"
continue
# --- Translation text (p56) ---
if cls == TRANSLATION_TEXT_CLASS and current_verse and state == "TRANSLATION":
html = decode_html_content(p)
# Remove bold wrapper if present (the whole translation is bold)
html = re.sub(r"^<b>(.*)</b>$", r"\1", html, flags=re.DOTALL)
# Remove trailing source references like [& Bṛhad-āraṇyaka Upaniṣad 5.1]
# These are in span.s4 tags — they may have survived decode_html_content
html = html.strip()
current_verse["translation"] = html
state = "POST_TRANSLATION"
continue
# --- PURPORT label (p57) ---
if cls == PURPORT_LABEL_CLASS and current_verse:
state = "PURPORT"
continue
# --- Purport paragraphs ---
if state == "PURPORT" and current_verse and cls in PURPORT_CLASSES:
html = decode_html_content(p)
if html:
# Skip dashed-line separators
plain = re.sub(r"<[^>]+>", "", html).strip()
if not is_dashed_line(plain):
current_verse["purport"].append(html)
continue
# --- Section heading within purport (p31) ---
if state == "PURPORT" and current_verse and cls == SECTION_HEADING_CLASS:
html = decode_html_content(p)
if html:
current_verse["purport"].append(html)
continue
# --- Body text classes that might appear in purport (p6) ---
if state == "PURPORT" and current_verse and cls in BODY_CLASSES:
html = decode_html_content(p)
if html:
plain = re.sub(r"<[^>]+>", "", html).strip()
if not is_dashed_line(plain):
current_verse["purport"].append(html)
continue
# Finalize last verse
if current_verse and state != "IDLE":
_finalize_verse(current_verse)
verses.append(current_verse)
return verses
def _finalize_verse(verse: dict):
"""Build transliteration and Devanagari from pādas. Remove internal fields."""
padas = verse.get("_padas", [])
# Transliteration: one pāda per line
verse["transliteration"] = "\n".join(padas)
# Devanagari: pair pādas into 2-line verse structure, then convert
if padas:
paired = pair_padas(padas)
verse["devanagari"] = iast_to_devanagari(paired)
# Clean up internal field
if "_padas" in verse:
del verse["_padas"]
# ---------------------------------------------------------------------------
# JSON writing
# ---------------------------------------------------------------------------
def write_json(data, filepath):
"""Write JSON with proper Unicode and formatting."""
os.makedirs(os.path.dirname(filepath), exist_ok=True)
with open(filepath, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False, indent=2)
print(f" Written: {filepath}")
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
def main():
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
html_path = os.path.join(base_dir, "public", "html", "other.html")
output_base = os.path.join(base_dir, "lib", "jsons", "books", "iso")
if not os.path.exists(html_path):
print(f"ERROR: {html_path} not found!")
sys.exit(1)
print(f"Reading {html_path}...")
with open(html_path, "r", encoding="utf-8") as f:
soup = BeautifulSoup(f, "html.parser")
all_paragraphs = soup.find_all("p")
print(f"Total paragraphs in file: {len(all_paragraphs)}")
# --- Find Iso boundaries ---
iso_start, iso_end = find_iso_boundaries(all_paragraphs)
print(f"Iso content: paragraphs {iso_start} to {iso_end}")
# --- Find section bookmarks (p13) ---
section_bookmarks = find_section_bookmarks(all_paragraphs, iso_start, iso_end)
print(f"\nSection bookmarks found:")
for bm_text, bm_idx in sorted(section_bookmarks.items(), key=lambda x: x[1]):
print(f" [{bm_idx}] {bm_text}")
# Key bookmark indices
front_pages_idx = section_bookmarks.get("Iso back cover & front pages-1969")
introduction_idx = section_bookmarks.get("Iso Introduction-1969")
texts_idx = section_bookmarks.get("Iso Texts-1969")
about_idx = section_bookmarks.get("Iso About the author-1969")
glossary_idx = section_bookmarks.get("Iso Glossary-1969")
if texts_idx is None:
print("ERROR: Could not find 'Iso Texts-1969' bookmark!")
sys.exit(1)
# Use iso_end as fallback for glossary boundary
end_boundary = glossary_idx if glossary_idx is not None else iso_end
# --- Extract intro segments ---
print("\n--- Extracting Intro Segments ---")
intro_segments = []
# 1. Front pages segment (up to the Introduction bookmark, or texts if no intro)
front_end = introduction_idx if introduction_idx is not None else texts_idx
if front_pages_idx is not None and front_end is not None:
front_paras = extract_intro_segment(all_paragraphs, front_pages_idx, front_end)
front_segment = {
"slug": "front-pages",
"title": "Back Cover & Front Pages",
"type": "intro-segment",
"book": "iso",
"order": 1,
"paragraphs": front_paras,
}
intro_segments.append(front_segment)
print(f" Front Pages: {len(front_paras)} paragraphs")
# 2. Introduction (Teachings of the Vedas)
if introduction_idx is not None and texts_idx is not None:
intro_paras = extract_intro_segment(all_paragraphs, introduction_idx, texts_idx)
intro_segment = {
"slug": "introduction",
"title": "Introduction (Teachings of the Vedas)",
"type": "intro-segment",
"book": "iso",
"order": 2,
"paragraphs": intro_paras,
}
intro_segments.append(intro_segment)
print(f" Introduction: {len(intro_paras)} paragraphs")
# 3. About the Author segment
if about_idx is not None:
about_paras = extract_intro_segment(all_paragraphs, about_idx, end_boundary)
about_segment = {
"slug": "about-the-author",
"title": "About the Author",
"type": "intro-segment",
"book": "iso",
"order": 3,
"paragraphs": about_paras,
}
intro_segments.append(about_segment)
print(f" About the Author: {len(about_paras)} paragraphs")
# --- Extract verses ---
print("\n--- Extracting Verses ---")
# Verse section runs from texts_idx to about_idx (or end_boundary)
verse_end = about_idx if about_idx is not None else end_boundary
verses = extract_verses(all_paragraphs, texts_idx, verse_end)
print(f"Extracted {len(verses)} verses (expected 19: Invocation + Mantras 1-18)")
# --- Write intro.json ---
mantras_listing = []
for num in range(0, 19):
title = MANTRA_TITLES[num]
if num == 0:
slug = "invocation"
else:
slug = str(num).zfill(2)
mantras_listing.append({"number": num, "title": title, "slug": slug})
intro_meta = {
"book": "iso",
"title": "Śrī Īśopaniṣad",
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
"year": 1969,
"description": "The foremost of the 108 Upaniṣads, containing the essence of all Vedic knowledge.",
"totalMantras": 18,
"mantras": mantras_listing,
}
write_json(intro_meta, os.path.join(output_base, "intro.json"))
# --- Write intro segments ---
print("\n--- Writing Intro Segments ---")
for seg in intro_segments:
filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
write_json(seg, filepath)
# --- Write verse JSON files ---
print("\n--- Writing Verse Files ---")
for verse in verses:
mantra_num = verse["mantra"]
if mantra_num == 0:
filename = "invocation.json"
else:
filename = f"{str(mantra_num).zfill(2)}.json"
filepath = os.path.join(output_base, "verses", filename)
write_json(verse, filepath)
# --- Summary ---
print(f"\n{'=' * 60}")
print(f" EXTRACTION SUMMARY")
print(f"{'=' * 60}")
print(f" Book: Śrī Īśopaniṣad (1969 Edition)")
print(f" Intro segments: {len(intro_segments)}")
for seg in intro_segments:
print(
f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs"
)
print(f" Verses extracted: {len(verses)}")
print(
f" Invocation: {'Yes' if any(v['mantra'] == 0 for v in verses) else 'No'}"
)
mantras_found = sorted([v["mantra"] for v in verses if v["mantra"] > 0])
print(
f" Mantras: {mantras_found[0]}-{mantras_found[-1]} ({len(mantras_found)} total)"
if mantras_found
else " Mantras: None"
)
total_purport_paras = sum(len(v["purport"]) for v in verses)
total_synonyms = sum(len(v["synonyms"]) for v in verses)
print(f" Total purport ¶s: {total_purport_paras}")
print(f" Total synonyms: {total_synonyms}")
print(f" Output directory: {output_base}/")
# --- Quality checks ---
print(f"\n--- Quality Checks ---")
# Check all verses have expected fields
issues = []
for verse in verses:
ref = verse["reference"]
if not verse["transliteration"]:
issues.append(f" {ref}: missing transliteration")
if not verse["devanagari"]:
issues.append(f" {ref}: missing devanagari")
if not verse["synonyms"]:
issues.append(f" {ref}: missing synonyms")
if not verse["translation"]:
issues.append(f" {ref}: missing translation")
if not verse["purport"]:
issues.append(f" {ref}: missing purport")
if issues:
print(" ISSUES FOUND:")
for issue in issues:
print(issue)
else:
print(
" All verses have transliteration, devanagari, synonyms, translation, and purport ✓"
)
# --- Samples ---
print(f"\n--- Sample: Invocation ---")
inv = [v for v in verses if v["mantra"] == 0]
if inv:
v = inv[0]
print(f" Reference: {v['reference']}")
print(f" Transliteration: {v['transliteration'][:80]}...")
print(f" Devanagari: {v['devanagari'][:80]}...")
print(f" Synonyms: {len(v['synonyms'])} entries")
if v["synonyms"]:
print(f" First: {v['synonyms'][0]}")
trans_preview = re.sub(r"<[^>]+>", "", v["translation"])[:100]
print(f" Translation: {trans_preview}...")
print(f" Purport ¶s: {len(v['purport'])}")
print(f"\n--- Sample: Mantra One ---")
m1 = [v for v in verses if v["mantra"] == 1]
if m1:
v = m1[0]
print(f" Reference: {v['reference']}")
print(f" Transliteration: {v['transliteration'][:80]}...")
print(f" Devanagari: {v['devanagari'][:80]}...")
print(f" Synonyms: {len(v['synonyms'])} entries")
if v["synonyms"]:
print(f" First: {v['synonyms'][0]}")
print(f" Last: {v['synonyms'][-1]}")
trans_preview = re.sub(r"<[^>]+>", "", v["translation"])[:100]
print(f" Translation: {trans_preview}...")
print(f" Purport ¶s: {len(v['purport'])}")
print(f"\n--- Sample: Mantra Eighteen (last) ---")
m18 = [v for v in verses if v["mantra"] == 18]
if m18:
v = m18[0]
print(f" Reference: {v['reference']}")
print(f" Transliteration: {v['transliteration'][:80]}...")
print(f" Devanagari: {v['devanagari'][:80]}...")
print(f" Synonyms: {len(v['synonyms'])} entries")
trans_preview = re.sub(r"<[^>]+>", "", v["translation"])[:100]
print(f" Translation: {trans_preview}...")
print(f" Purport ¶s: {len(v['purport'])}")
if __name__ == "__main__":
main()
scripts/mg_extract.py
+644
View File
@@ -0,0 +1,644 @@
#!/usr/bin/env python3
"""Extract Kṛṣṇa Consciousness: The Matchless Gift (1974) from other.html.
The book has 1 intro segment (front-pages) and 7 chapters.
Output:
lib/jsons/books/mg/intro.json — Book metadata
lib/jsons/books/mg/intro/*.json — Intro segments
lib/jsons/books/mg/chapters/*.json — Chapters
Usage:
python3 scripts/mg_extract.py
"""
import json
import os
import re
import sys
from bs4 import BeautifulSoup, Tag
# ---------------------------------------------------------------------------
# ScaGoudy → IAST Unicode mapping
# ---------------------------------------------------------------------------
SCAGOUDY_TO_IAST = {
"ä": "ā",
"Ä": "Ā",
"å": "ṛ",
"Å": "Ṛ",
"ñ": "ṣ",
"Ñ": "Ṣ",
"ö": "ṭ",
"Ö": "Ṭ",
"ë": "ṇ",
"Ë": "Ṇ",
"ò": "ḍ",
"Ò": "Ḍ",
"ç": "ś",
"Ç": "Ś",
"ì": "ṅ",
"ï": "ñ",
"ü": "ū",
"Ü": "Ū",
"é": "ī",
"É": "Ī",
"à": "ṁ",
"ù": "ḥ",
"è": "ṝ",
"û": "ṁl",
"ÿ": "ḷ",
}
def decode_scagoudy(text: str) -> str:
"""Replace ScaGoudy-encoded characters with proper IAST Unicode."""
for src, dst in SCAGOUDY_TO_IAST.items():
text = text.replace(src, dst)
return text
def decode_html_content(tag: Tag) -> str:
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
inner = "".join(str(child) for child in tag.children)
inner = decode_scagoudy(inner)
# Remove span tags with CSS classes (RTF conversion artifacts)
inner = re.sub(
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
)
# Clean up Apple-tab-span leftovers (tab characters)
inner = inner.replace("\t", " ")
# Normalize whitespace (but preserve <br> tags)
inner = re.sub(r"\s+", " ", inner).strip()
return inner
def decode_text(tag: Tag) -> str:
"""Get plain text from a tag with ScaGoudy decoding."""
return decode_scagoudy(tag.get_text().strip())
def get_css_class(tag: Tag) -> str:
"""Get the first CSS class of a tag, or empty string."""
classes = tag.get("class", [])
return classes[0] if classes else ""
def is_empty(tag: Tag) -> bool:
"""Check if a tag is empty or whitespace-only."""
text = tag.get_text().strip()
return not text or text == "\xa0"
def is_dash_line(text: str) -> bool:
"""Return True if text is only dashes, spaces, and/or whitespace."""
cleaned = text.strip()
if not cleaned:
return False
return all(c in "-—–\u2014\u2013 \t" for c in cleaned)
# ---------------------------------------------------------------------------
# CSS class categorization for MG
# ---------------------------------------------------------------------------
# Book title header class
BOOK_HEADER_CLASS = "p12"
# Chapter bookmark class
CHAPTER_BOOKMARK_CLASS = "p13"
# Body paragraph classes (primary content)
BODY_CLASSES = frozenset(["p6", "p78", "p30"])
# Sanskrit verse pāda classes (italic, centered) — BOTH p53 AND p24
VERSE_PADA_CLASSES = frozenset(["p53", "p24"])
# Verse translations / body text
TRANSLATION_CLASS = "p44"
# Chapter heading (bold)
CHAPTER_HEADING_CLASS = "p27"
# Title page text
TITLE_PAGE_CLASS = "p3"
# Book list header/footer
BOOK_LIST_CLASSES = frozenset(["p76", "p77"])
# Classes to skip entirely (empty/decorative)
SKIP_CLASSES = frozenset(["p9", "p70", "p79"])
# ---------------------------------------------------------------------------
# Extraction helpers
# ---------------------------------------------------------------------------
def find_mg_boundaries(all_paragraphs: list) -> tuple:
"""Find the start and end paragraph indices of the MG content.
Start: <p class="p12"> containing "Matchless Gift" and "1974"
End: next <p class="p12"> after start, or <p class="p27"> containing "APPENDIX"
"""
start_idx = None
end_idx = len(all_paragraphs)
for i, p in enumerate(all_paragraphs):
cls = get_css_class(p)
if cls != BOOK_HEADER_CLASS:
continue
text = decode_text(p)
if "Matchless Gift" in text and "1974" in text:
start_idx = i
elif start_idx is not None:
# Next p12 after MG start = end of MG
end_idx = i
break
if start_idx is None:
print(
"ERROR: Could not find 'Kṛṣṇa Consciousness: The Matchless Gift (1974)' header in other.html!"
)
sys.exit(1)
# Also check for APPENDIXES/APPENDIX as end boundary (p27 with bold text)
for i in range(start_idx + 1, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
if cls == CHAPTER_HEADING_CLASS:
text = p.get_text().strip().upper()
if text.startswith("APPENDIX") or text.startswith("APPENDIXES"):
end_idx = i
break
return start_idx, end_idx
def find_chapter_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
"""Find all MG chapter bookmark tags (p13) in the MG range.
Returns list of (paragraph_index, chapter_number, chapter_title).
"""
bookmarks = []
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
if cls == CHAPTER_BOOKMARK_CLASS:
text = decode_text(p)
# Use re.DOTALL to handle titles that span multiple lines
m = re.match(r"MG\s+(\d+)\s*:\s*(.+)", text, re.DOTALL)
if m:
ch_num = int(m.group(1))
# Clean the title: replace newlines with spaces, normalize whitespace
ch_title = re.sub(r"\s+", " ", m.group(2)).strip()
# Capitalize "love" in "Learning to love" -> "Learning to Love"
if ch_title == "Learning to love":
ch_title = "Learning to Love"
bookmarks.append((i, ch_num, ch_title))
return bookmarks
def extract_paragraphs(
all_paragraphs: list,
start_idx: int,
end_idx: int,
is_front_pages: bool = False,
) -> list:
"""Extract paragraphs for a section (chapter or intro segment).
Handles all CSS classes within the MG content.
Groups consecutive p53/p24 verse pādas into centered italic divs.
"""
result = []
verse_accumulator = []
def flush_verse():
if verse_accumulator:
joined = "<br>".join(verse_accumulator)
result.append(f'<div style="text-align:center"><i>{joined}</i></div>')
verse_accumulator.clear()
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
# Skip empty paragraphs
if is_empty(p):
continue
# Skip noise classes
if cls in SKIP_CLASSES:
continue
# Skip bookmark tags (p13)
if cls == CHAPTER_BOOKMARK_CLASS:
continue
# Skip book title header (p12)
if cls == BOOK_HEADER_CLASS:
continue
# Check for dash lines — skip them
text = decode_text(p)
if is_dash_line(text):
continue
# Sanskrit verse pādas (p53 and p24) — accumulate into centered block
if cls in VERSE_PADA_CLASSES:
html = decode_html_content(p)
if html:
# Remove inner <i> tags before re-wrapping
stripped = re.sub(r"</?i>", "", html)
verse_accumulator.append(stripped)
continue
# Non-verse paragraph — flush any accumulated verse first
flush_verse()
# Chapter heading (p27) — include as paragraph
if cls == CHAPTER_HEADING_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Title page text (p3) — include
if cls == TITLE_PAGE_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Book list classes (p76, p77) — include in front-pages
if cls in BOOK_LIST_CLASSES:
if is_front_pages:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Verse translation / body text (p44)
if cls == TRANSLATION_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Body paragraphs (p6, p78, p30)
if cls in BODY_CLASSES:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Any other class — try to extract content (be permissive)
html = decode_html_content(p)
if html and not is_dash_line(text):
result.append(html)
# Flush any trailing verse
flush_verse()
return result
# ---------------------------------------------------------------------------
# Main extraction logic
# ---------------------------------------------------------------------------
def extract_mg(soup: BeautifulSoup) -> dict:
"""Extract the entire Kṛṣṇa Consciousness: The Matchless Gift (1974) from the parsed HTML.
Returns a dict with:
"intro_segments": list of intro segment dicts
"chapters": list of chapter dicts
"chapter_listing": list of {number, title} for intro.json
"""
all_paragraphs = list(soup.find_all("p"))
# Step 1: Find MG boundaries
mg_start, mg_end = find_mg_boundaries(all_paragraphs)
print(f" MG content: paragraphs {mg_start} to {mg_end}")
print(f" Total paragraphs in range: {mg_end - mg_start}")
# Step 2: Find all chapter bookmarks
chapter_bookmarks = find_chapter_bookmarks(all_paragraphs, mg_start, mg_end)
print(f" Found {len(chapter_bookmarks)} MG chapter bookmarks")
for pos, ch_num, ch_title in chapter_bookmarks:
print(f" [chapter {ch_num}] paragraph {pos}: {ch_title}")
# Step 3: Extract front pages
# From book start (mg_start) to first chapter bookmark (MG 1)
intro_segments = []
if chapter_bookmarks:
front_pages_end = chapter_bookmarks[0][0]
else:
front_pages_end = mg_end
front_pages_paragraphs = extract_paragraphs(
all_paragraphs, mg_start, front_pages_end, is_front_pages=True
)
intro_segments.append(
{
"slug": "front-pages",
"title": "Front Pages",
"type": "intro-segment",
"book": "mg",
"order": 1,
"paragraphs": front_pages_paragraphs,
}
)
# Step 4: Extract chapters
chapters = []
for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_bookmarks):
content_start = ch_pos + 1
# Content ends at next chapter bookmark or mg_end
if idx + 1 < len(chapter_bookmarks):
content_end = chapter_bookmarks[idx + 1][0]
else:
content_end = mg_end
paragraphs = extract_paragraphs(all_paragraphs, content_start, content_end)
ch = {
"book": "mg",
"chapter": ch_num,
"title": ch_title,
"type": "chapter",
"paragraphs": paragraphs,
}
chapters.append(ch)
# Step 5: Build chapter listing for intro.json
chapter_listing = [
{"number": ch["chapter"], "title": ch["title"]} for ch in chapters
]
return {
"intro_segments": intro_segments,
"chapters": chapters,
"chapter_listing": chapter_listing,
}
# ---------------------------------------------------------------------------
# Output
# ---------------------------------------------------------------------------
def write_json(data, filepath):
"""Write data as formatted JSON to filepath, creating directories as needed."""
os.makedirs(os.path.dirname(filepath), exist_ok=True)
with open(filepath, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False, indent=2)
def main():
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
html_path = os.path.join(base_dir, "public", "html", "other.html")
output_base = os.path.join(base_dir, "lib", "jsons", "books", "mg")
if not os.path.exists(html_path):
print(f"ERROR: {html_path} not found!")
sys.exit(1)
print(f"Reading {html_path}...")
with open(html_path, "r", encoding="utf-8") as f:
soup = BeautifulSoup(f, "html.parser")
print("Extracting Kṛṣṇa Consciousness: The Matchless Gift (1974)...\n")
result = extract_mg(soup)
intro_segments = result["intro_segments"]
chapters = result["chapters"]
chapter_listing = result["chapter_listing"]
# --- Write intro.json ---
intro_meta = {
"book": "mg",
"title": "Kṛṣṇa Consciousness: The Matchless Gift",
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
"year": 1974,
"description": "A profound introduction to the science of Kṛṣṇa consciousness and the matchless gift of love of God.",
"totalChapters": 7,
"chapters": chapter_listing,
}
intro_path = os.path.join(output_base, "intro.json")
write_json(intro_meta, intro_path)
print(f"\n Written: {intro_path}")
# --- Write intro segments ---
print(f"\n--- Intro Segments ---")
for seg in intro_segments:
filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
write_json(seg, filepath)
print(
f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}"
)
# --- Write chapters ---
print(f"\n--- Chapters ---")
for ch in chapters:
ch_num_str = str(ch["chapter"]).zfill(2)
filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
write_json(ch, filepath)
print(
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
f"{ch['title']}"
)
# --- Summary ---
total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments)
total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
total_paras = total_intro_paras + total_chapter_paras
print(f"\n{'=' * 60}")
print(f" EXTRACTION SUMMARY")
print(f"{'=' * 60}")
print(f" Intro segments: {len(intro_segments)}")
print(f" Chapters: {len(chapters)}")
print(f" Total paragraphs: {total_paras}")
print(f" Intro: {total_intro_paras}")
print(f" Chapters: {total_chapter_paras}")
if chapters:
print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
print(f" Output directory: {output_base}/")
# --- Samples ---
print(f"\n--- Sample: Front Pages ---")
if intro_segments:
front = intro_segments[0]
for i, p in enumerate(front["paragraphs"][:5]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
if chapters:
print(f"\n--- Sample: Chapter 1 ({chapters[0]['title']}) ---")
ch1 = chapters[0]
for i, p in enumerate(ch1["paragraphs"][:5]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
if chapters and len(chapters) > 1:
print(
f"\n--- Sample: Chapter {chapters[-1]['chapter']} ({chapters[-1]['title']}) ---"
)
last = chapters[-1]
for i, p in enumerate(last["paragraphs"][:5]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
# --- Quality Checks ---
print(f"\n--- Quality Checks ---")
# Check all 7 chapters present
expected_chapters = set(range(1, 8))
actual_chapters = set(ch["chapter"] for ch in chapters)
missing = expected_chapters - actual_chapters
extra = actual_chapters - expected_chapters
if missing:
print(f" ⚠ Missing chapters: {sorted(missing)}")
if extra:
print(f" ⚠ Extra chapters: {sorted(extra)}")
if not missing and not extra:
print(f" ✅ All 7 chapters present")
# Check all 1 intro segment present
expected_intros = {"front-pages"}
actual_intros = {s["slug"] for s in intro_segments}
missing_intros = expected_intros - actual_intros
extra_intros = actual_intros - expected_intros
if missing_intros:
print(f" ⚠ Missing intro segments: {missing_intros}")
if extra_intros:
print(f" ⚠ Extra intro segments: {extra_intros}")
if not missing_intros and not extra_intros:
print(f" ✅ All 1 intro segments present")
# Check empty chapters
empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
if empty_chapters:
print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
else:
print(f" ✅ No empty chapters")
# Check empty intro segments
empty_intros = [s["slug"] for s in intro_segments if len(s["paragraphs"]) == 0]
if empty_intros:
print(f" ⚠ Empty intro segments: {empty_intros}")
else:
print(f" ✅ No empty intro segments")
# Check ScaGoudy decoding
print(f"\n--- ScaGoudy Decoding Check ---")
all_text = ""
for ch in chapters:
all_text += " ".join(ch["paragraphs"][:20])
for seg in intro_segments:
all_text += " ".join(seg["paragraphs"][:10])
# Check for properly decoded words
if "Kṛṣṇa" in all_text:
print(f" ✅ 'Kṛṣṇa' found (ScaGoudy properly decoded)")
else:
print(f" ⚠ 'Kṛṣṇa' not found — ScaGoudy decoding may have failed")
if "Bhagavad-gītā" in all_text:
print(f" ✅ 'Bhagavad-gītā' found (ScaGoudy properly decoded)")
elif "gītā" in all_text or "Gītā" in all_text:
print(f" ✅ 'gītā' found (ScaGoudy properly decoded)")
else:
print(f" ⚠ 'Gītā' not found — ScaGoudy decoding may have failed")
if "Prabhupāda" in all_text:
print(f" ✅ 'Prabhupāda' found (ScaGoudy properly decoded)")
# Check for un-decoded ScaGoudy remnants (common encoded chars)
scagoudy_remnants = 0
for ch in chapters:
for p in ch["paragraphs"]:
if "Kåñëa" in p or "Prabhupäda" in p:
scagoudy_remnants += 1
for seg in intro_segments:
for p in seg["paragraphs"]:
if "Kåñëa" in p or "Prabhupäda" in p:
scagoudy_remnants += 1
if scagoudy_remnants == 0:
print(f" ✅ No ScaGoudy remnants detected")
else:
print(f" ⚠ {scagoudy_remnants} paragraphs with ScaGoudy remnants")
# Check for centered verse blocks
verse_block_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
if '<div style="text-align:center"><i>' in p:
verse_block_count += 1
if verse_block_count > 0:
print(f" ✅ {verse_block_count} centered Sanskrit verse blocks found")
else:
print(f" ⚠ No centered verse blocks found — verse grouping may have failed")
# Check for residual span tags
span_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
if 'class="' in p and "<span" in p:
span_count += 1
for seg in intro_segments:
for p in seg["paragraphs"]:
if 'class="' in p and "<span" in p:
span_count += 1
if span_count == 0:
print(f" ✅ No residual span class artifacts found")
else:
print(f" ⚠ {span_count} paragraphs still contain span class artifacts")
# Check for dash-line paragraphs that shouldn't be there
dash_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
plain = re.sub(r"<[^>]+>", "", p)
if is_dash_line(plain):
dash_count += 1
for seg in intro_segments:
for p in seg["paragraphs"]:
plain = re.sub(r"<[^>]+>", "", p)
if is_dash_line(plain):
dash_count += 1
if dash_count == 0:
print(f" ✅ No dashed-line paragraphs in output")
else:
print(f" ⚠ {dash_count} dashed-line paragraphs found in output")
# Show a verse block sample
print(f"\n--- Sample: Verse Block ---")
for ch in chapters:
for p in ch["paragraphs"]:
if '<div style="text-align:center"><i>' in p:
preview = re.sub(r"<[^>]+>", "", p)[:120]
print(f" {preview}...")
break
else:
continue
break
# Chapter title decoding check
print(f"\n--- Chapter Title Decoding Check ---")
for ch in chapters:
title = ch["title"]
if "å" in title or "ë" in title or "ñ" in title or "ä" in title:
print(f" ⚠ Chapter {ch['chapter']} title has un-decoded ScaGoudy: {title}")
else:
print(f" ✅ Ch {ch['chapter']}: {title}")
print(f"\nDone! ✨")
if __name__ == "__main__":
main()
scripts/nod_extract.py
+803
View File
@@ -0,0 +1,803 @@
#!/usr/bin/env python3
"""Extract The Nectar of Devotion (1970 Edition) from other.html.
The book has 3 intro segments (Front Flap, Preface, Introduction),
51 chapters organized in 4 parts, and a Concluding Words section.
Output:
lib/jsons/books/nod/intro.json — Book metadata
lib/jsons/books/nod/intro/*.json — Intro segments
lib/jsons/books/nod/chapters/*.json — Chapters + concluding words
Usage:
python3 scripts/nod_extract.py
"""
import json
import os
import re
import sys
from bs4 import BeautifulSoup, Tag
# ---------------------------------------------------------------------------
# ScaGoudy → IAST Unicode mapping
# ---------------------------------------------------------------------------
SCAGOUDY_TO_IAST = {
"ä": "ā",
"Ä": "Ā",
"å": "ṛ",
"Å": "Ṛ",
"ñ": "ṣ",
"Ñ": "Ṣ",
"ö": "ṭ",
"Ö": "Ṭ",
"ë": "ṇ",
"Ë": "Ṇ",
"ò": "ḍ",
"Ò": "Ḍ",
"ç": "ś",
"Ç": "Ś",
"ì": "ṅ",
"ï": "ñ",
"ü": "ū",
"Ü": "Ū",
"é": "ī",
"É": "Ī",
"à": "ṁ",
"ù": "ḥ",
"è": "ṝ",
"û": "ṁl",
"ÿ": "ḷ",
}
def decode_scagoudy(text: str) -> str:
"""Replace ScaGoudy-encoded characters with proper IAST Unicode."""
for src, dst in SCAGOUDY_TO_IAST.items():
text = text.replace(src, dst)
return text
def decode_html_content(tag: Tag) -> str:
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
inner = "".join(str(child) for child in tag.children)
inner = decode_scagoudy(inner)
# Remove span tags with CSS classes (RTF conversion artifacts)
inner = re.sub(
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
)
# Clean up Apple-tab-span leftovers (tab characters)
inner = inner.replace("\t", " ")
# Normalize whitespace
inner = re.sub(r"\s+", " ", inner).strip()
return inner
def decode_text(tag: Tag) -> str:
"""Get plain text from a tag with ScaGoudy decoding."""
return decode_scagoudy(tag.get_text().strip())
def get_css_class(tag: Tag) -> str:
"""Get the first CSS class of a tag, or empty string."""
classes = tag.get("class", [])
return classes[0] if classes else ""
def is_empty(tag: Tag) -> bool:
"""Check if a tag is empty or whitespace-only."""
text = tag.get_text().strip()
return not text or text == "\xa0"
# ---------------------------------------------------------------------------
# CSS class categorization for NoD
# ---------------------------------------------------------------------------
# Bookmark class — used for section boundary detection
BOOKMARK_CLASS = "p13"
# Book title header class
BOOK_HEADER_CLASS = "p12"
# Body paragraph classes
BODY_CLASSES = frozenset(["p6", "p35", "p30"])
# Section heading (bold) — "Chapter One", "Preface", etc.
SECTION_HEADING_CLASS = "p31"
# Chapter title (bold) — "Characteristics of Pure Devotional Service", etc.
CHAPTER_TITLE_CLASS = "p27"
# Part header class — "PART ONE", "PART TWO", etc.
PART_HEADER_CLASS = "p34"
# Sub-section heading (bold) within chapters
SUB_HEADING_CLASS = "p36"
# Italic sub-headings
ITALIC_SUB_HEADING_CLASSES = frozenset(["p37", "p38"])
# Title page / copyright text classes (in intro area)
TITLE_PAGE_CLASS = "p3"
COPYRIGHT_CLASS = "p32"
# Sanskrit verse (italic with br tags)
SANSKRIT_VERSE_CLASS = "p33"
# Classes to skip entirely (empty/decorative)
SKIP_CLASSES = frozenset(["p9", "p15"])
# ---------------------------------------------------------------------------
# Intro segment definitions
# ---------------------------------------------------------------------------
INTRO_SEGMENTS = [
{
"slug": "front-flap",
"title": "Front Flap",
"order": 1,
"bookmark_pattern": "NoD-1970: Front Flap",
},
{
"slug": "preface",
"title": "Preface",
"order": 2,
"bookmark_pattern": "NoD-1970: Preface",
},
{
"slug": "introduction",
"title": "Introduction",
"order": 3,
"bookmark_pattern": "NoD-1970: Introduction",
},
]
# ---------------------------------------------------------------------------
# Extraction helpers
# ---------------------------------------------------------------------------
def find_nod_boundaries(all_paragraphs: list) -> tuple[int, int]:
"""Find the start and end paragraph indices of the NoD content.
Start: <p class="p12"> containing "Nectar of Devotion"
End: next <p class="p12"> after start, or end of file
"""
start_idx = None
end_idx = len(all_paragraphs)
for i, p in enumerate(all_paragraphs):
cls = get_css_class(p)
if cls != BOOK_HEADER_CLASS:
continue
text = decode_text(p)
if "Nectar of Devotion" in text:
start_idx = i
elif start_idx is not None:
# Next p12 after NoD start = end of NoD
end_idx = i
break
if start_idx is None:
print("ERROR: Could not find 'The Nectar of Devotion' header in other.html!")
sys.exit(1)
return start_idx, end_idx
def find_all_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
"""Find all p13 bookmark tags in the NoD range.
Returns list of (paragraph_index, bookmark_text).
"""
bookmarks = []
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
if cls == BOOKMARK_CLASS:
text = decode_text(p)
if "NoD" in text:
bookmarks.append((i, text))
return bookmarks
def parse_chapter_bookmark(text: str) -> tuple[int, str] | None:
"""Parse a chapter bookmark like 'NoD 1-1970: Characteristics of Pure Devotional Service'.
Returns (chapter_number, title) or None if not a chapter bookmark.
"""
m = re.match(r"NoD\s+(\d+)-1970:\s*(.+)", text.strip())
if m:
return int(m.group(1)), m.group(2).strip()
return None
def parse_intro_bookmark(text: str) -> str | None:
"""Parse an intro bookmark like 'NoD-1970: Preface'.
Returns the segment name or None if not an intro bookmark.
"""
m = re.match(r"NoD-1970:\s*(.+)", text.strip())
if m:
return m.group(1).strip()
return None
def extract_paragraphs(
all_paragraphs: list,
start_idx: int,
end_idx: int,
is_intro: bool = False,
) -> list[str]:
"""Extract paragraphs for a section (chapter or intro segment).
Handles all CSS classes within the NoD content.
"""
result: list[str] = []
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
# Skip empty paragraphs
if is_empty(p):
continue
# Skip noise classes
if cls in SKIP_CLASSES:
continue
# Skip bookmark tags
if cls == BOOKMARK_CLASS:
continue
# Skip book title header
if cls == BOOK_HEADER_CLASS:
continue
# Part headers — include as bold paragraph
if cls == PART_HEADER_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Section heading (p31) — include as paragraph
if cls == SECTION_HEADING_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Chapter title (p27) — include as bold paragraph
if cls == CHAPTER_TITLE_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Sub-section headings within chapters (p36, bold)
if cls == SUB_HEADING_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Italic sub-headings (p37, p38)
if cls in ITALIC_SUB_HEADING_CLASSES:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Title page / copyright text (in intro area)
if cls == TITLE_PAGE_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Copyright text
if cls == COPYRIGHT_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Sanskrit verse (italic with br tags) — format as centered italic div
# Strip all inner <i>/<\/i> tags before re-wrapping to avoid nesting
if cls == SANSKRIT_VERSE_CLASS:
html = decode_html_content(p)
if html:
stripped = re.sub(r"</?i>", "", html)
result.append(f'<div style="text-align:center"><i>{stripped}</i></div>')
continue
# Body paragraphs (p6, p35, p30)
if cls in BODY_CLASSES:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Any other class — try to extract content (be permissive)
html = decode_html_content(p)
if html:
result.append(html)
return result
# ---------------------------------------------------------------------------
# Main extraction logic
# ---------------------------------------------------------------------------
def extract_nod(soup: BeautifulSoup) -> dict:
"""Extract the entire Nectar of Devotion from the parsed HTML.
Returns a dict with:
"intro_segments": list of intro segment dicts
"chapters": list of chapter dicts
"concluding_words": concluding words dict
"parts": list of part assignment dicts
"chapter_listing": list of {number, title} for intro.json
"""
all_paragraphs = list(soup.find_all("p"))
# Step 1: Find NoD boundaries
nod_start, nod_end = find_nod_boundaries(all_paragraphs)
print(f" NoD content: paragraphs {nod_start} to {nod_end}")
print(f" Total paragraphs in range: {nod_end - nod_start}")
# Step 2: Find all bookmarks
bookmarks = find_all_bookmarks(all_paragraphs, nod_start, nod_end)
print(f" Found {len(bookmarks)} NoD bookmarks")
# Step 3: Categorize bookmarks into intro vs chapter
intro_bookmarks = [] # (para_idx, segment_name)
chapter_bookmarks = [] # (para_idx, chapter_num, title)
for pos, text in bookmarks:
ch = parse_chapter_bookmark(text)
if ch:
chapter_bookmarks.append((pos, ch[0], ch[1]))
else:
intro_name = parse_intro_bookmark(text)
if intro_name:
intro_bookmarks.append((pos, intro_name))
print(f" Intro bookmarks: {len(intro_bookmarks)}")
print(f" Chapter bookmarks: {len(chapter_bookmarks)}")
# Step 4: Track PART assignments by scanning for p34 tags
# We need to know which chapters belong to which part.
part_assignments = {} # chapter_num -> part_num
parts_data = [] # list of {number, title, chapters}
# Walk through all paragraphs in NoD range and track parts
# We process this before extracting chapters so we know part assignments
part_positions = [] # (para_idx, part_title)
for i in range(nod_start, nod_end):
p = all_paragraphs[i]
cls = get_css_class(p)
if cls == PART_HEADER_CLASS:
text = decode_text(p)
if text:
part_positions.append((i, text))
print(f" Part headers found: {len(part_positions)}")
for pos, title in part_positions:
print(f" Line ~{pos}: {title}")
# Assign parts to chapters based on position
# A chapter belongs to the most recent PART header before its bookmark
part_number_map = {}
for pos, title in part_positions:
# Extract part number from title like "PART ONE", "PART TWO", etc.
part_num = None
title_upper = title.upper()
if "ONE" in title_upper or "I" == title_upper.replace("PART", "").strip():
part_num = 1
elif "TWO" in title_upper or "II" == title_upper.replace("PART", "").strip():
part_num = 2
elif "THREE" in title_upper or "III" == title_upper.replace("PART", "").strip():
part_num = 3
elif "FOUR" in title_upper or "IV" == title_upper.replace("PART", "").strip():
part_num = 4
if part_num:
part_number_map[pos] = part_num
# Now assign each chapter to a part
sorted_part_positions = sorted(part_number_map.keys())
for ch_pos, ch_num, ch_title in chapter_bookmarks:
# Find the most recent PART header before this chapter's bookmark
assigned_part = None
for pp in sorted_part_positions:
if pp < ch_pos:
assigned_part = part_number_map[pp]
else:
break
if assigned_part:
part_assignments[ch_num] = assigned_part
# Build parts_data
for part_num in sorted(set(part_assignments.values())):
chs = sorted([ch for ch, pn in part_assignments.items() if pn == part_num])
part_title = f"Part {'One' if part_num == 1 else 'Two' if part_num == 2 else 'Three' if part_num == 3 else 'Four'}"
parts_data.append(
{
"number": part_num,
"title": part_title,
"chapters": chs,
}
)
# Step 5: Extract intro segments
intro_segments = []
for seg_def in INTRO_SEGMENTS:
# Find matching bookmark
bookmark_pos = None
for pos, name in intro_bookmarks:
if (
seg_def["bookmark_pattern"].endswith(name)
or name == seg_def["bookmark_pattern"].split(": ", 1)[-1]
):
bookmark_pos = pos
break
if bookmark_pos is None:
print(f" WARNING: Could not find bookmark for '{seg_def['slug']}'")
continue
# Content starts after the bookmark
content_start = bookmark_pos + 1
# Content ends at the next bookmark (intro or chapter) or PART header
content_end = nod_end
all_section_starts = sorted(
[pos for pos, _ in intro_bookmarks]
+ [pos for pos, _, _ in chapter_bookmarks]
)
for s in all_section_starts:
if s > bookmark_pos:
content_end = s
break
paragraphs = extract_paragraphs(
all_paragraphs, content_start, content_end, is_intro=True
)
seg = {
"slug": seg_def["slug"],
"title": seg_def["title"],
"type": "intro-segment",
"book": "nod",
"order": seg_def["order"],
"paragraphs": paragraphs,
}
intro_segments.append(seg)
# Step 6: Extract chapters
chapters = []
for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_bookmarks):
content_start = ch_pos + 1
# Content ends at next chapter bookmark or "Concluding Words" p27 or nod_end
if idx + 1 < len(chapter_bookmarks):
content_end = chapter_bookmarks[idx + 1][0]
else:
# Last chapter (51) — content ends at "Concluding Words" p27 heading
# or at nod_end. We need to find where "Concluding Words" starts.
# Look for p27 containing "Concluding Words" after the last chapter bookmark.
concluding_pos = None
for i in range(ch_pos + 1, nod_end):
p = all_paragraphs[i]
cls = get_css_class(p)
if cls == CHAPTER_TITLE_CLASS:
text = decode_text(p)
if "Concluding Words" in text:
concluding_pos = i
break
if concluding_pos:
content_end = concluding_pos
else:
content_end = nod_end
# For the PART header: if a PART header appears right before this
# chapter's bookmark, include it in this chapter's content range.
# The PART header might be 1-3 paragraphs before the bookmark (with
# possible p15 empty lines in between).
actual_start = content_start
for pp in sorted_part_positions:
# Check if this PART header is between the previous section end and
# this chapter's bookmark
if pp < ch_pos and pp >= content_start - 5:
# Check that there's no other bookmark between pp and ch_pos
has_other_bookmark = False
for bpos, _ in bookmarks:
if bpos > pp and bpos < ch_pos:
has_other_bookmark = True
break
if not has_other_bookmark:
actual_start = pp
paragraphs = extract_paragraphs(all_paragraphs, actual_start, content_end)
ch = {
"book": "nod",
"chapter": ch_num,
"title": ch_title,
"type": "chapter",
"paragraphs": paragraphs,
}
chapters.append(ch)
# Step 7: Extract "Concluding Words"
concluding_words = None
# Find the "Concluding Words" p27 heading after chapter 51
last_ch_pos = chapter_bookmarks[-1][0] if chapter_bookmarks else nod_start
for i in range(last_ch_pos, nod_end):
p = all_paragraphs[i]
cls = get_css_class(p)
if cls == CHAPTER_TITLE_CLASS:
text = decode_text(p)
if "Concluding Words" in text:
# Extract from this heading to nod_end
paragraphs = extract_paragraphs(all_paragraphs, i, nod_end)
concluding_words = {
"book": "nod",
"chapter": None,
"title": "Concluding Words",
"type": "conclusion",
"slug": "concluding-words",
"paragraphs": paragraphs,
}
break
# Step 8: Build chapter listing for intro.json
chapter_listing = [
{"number": ch["chapter"], "title": ch["title"]} for ch in chapters
]
return {
"intro_segments": intro_segments,
"chapters": chapters,
"concluding_words": concluding_words,
"parts": parts_data,
"chapter_listing": chapter_listing,
}
# ---------------------------------------------------------------------------
# Output
# ---------------------------------------------------------------------------
def write_json(data, filepath):
"""Write data as formatted JSON to filepath, creating directories as needed."""
os.makedirs(os.path.dirname(filepath), exist_ok=True)
with open(filepath, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False, indent=2)
def main():
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
html_path = os.path.join(base_dir, "public", "html", "other.html")
output_base = os.path.join(base_dir, "lib", "jsons", "books", "nod")
if not os.path.exists(html_path):
print(f"ERROR: {html_path} not found!")
sys.exit(1)
print(f"Reading {html_path}...")
with open(html_path, "r", encoding="utf-8") as f:
soup = BeautifulSoup(f, "html.parser")
print("Extracting The Nectar of Devotion (1970 Edition)...\n")
result = extract_nod(soup)
intro_segments = result["intro_segments"]
chapters = result["chapters"]
concluding_words = result["concluding_words"]
parts = result["parts"]
chapter_listing = result["chapter_listing"]
# --- Write intro.json ---
intro_meta = {
"book": "nod",
"title": "The Nectar of Devotion",
"subtitle": "A Summary Study of Śrīla Rūpa Gosvāmī's Bhakti-rasāmṛta-sindhu",
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
"year": 1970,
"description": "The complete science of bhakti-yoga, or devotional service to Lord Kṛṣṇa.",
"totalChapters": 51,
"parts": parts,
"chapters": chapter_listing,
}
intro_path = os.path.join(output_base, "intro.json")
write_json(intro_meta, intro_path)
print(f"\n Written: {intro_path}")
# --- Write intro segments ---
print(f"\n--- Intro Segments ---")
for seg in intro_segments:
filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
write_json(seg, filepath)
print(
f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}"
)
# --- Write chapters ---
print(f"\n--- Chapters ---")
for ch in chapters:
ch_num_str = str(ch["chapter"]).zfill(2)
filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
write_json(ch, filepath)
part_info = ""
for p in parts:
if ch["chapter"] in p["chapters"]:
part_info = f" [Part {p['number']}]"
break
print(
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
f"{ch['title']}{part_info}"
)
# --- Write concluding words ---
if concluding_words:
filepath = os.path.join(output_base, "chapters", "concluding-words.json")
write_json(concluding_words, filepath)
print(
f"\n Concluding Words: {len(concluding_words['paragraphs'])} paragraphs → {filepath}"
)
# --- Summary ---
total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments)
total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
total_concluding_paras = (
len(concluding_words["paragraphs"]) if concluding_words else 0
)
total_paras = total_intro_paras + total_chapter_paras + total_concluding_paras
print(f"\n{'=' * 60}")
print(f" EXTRACTION SUMMARY")
print(f"{'=' * 60}")
print(f" Intro segments: {len(intro_segments)}")
print(f" Chapters: {len(chapters)}")
print(f" Concluding Words: {'Yes' if concluding_words else 'No'}")
print(f" Parts: {len(parts)}")
for p in parts:
print(
f" Part {p['number']} ({p['title']}): chapters {p['chapters'][0]}-{p['chapters'][-1]} ({len(p['chapters'])} chapters)"
)
print(f" Total paragraphs: {total_paras}")
print(f" Intro: {total_intro_paras}")
print(f" Chapters: {total_chapter_paras}")
print(f" Concluding: {total_concluding_paras}")
print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
print(f" Output directory: {output_base}/")
# --- Samples ---
print(f"\n--- Sample: Chapter 1 ---")
if chapters:
ch1 = chapters[0]
for i, p in enumerate(ch1["paragraphs"][:4]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
print(f"\n--- Sample: Chapter {chapters[-1]['chapter']} (last) ---")
if chapters:
last = chapters[-1]
for i, p in enumerate(last["paragraphs"][:3]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
if concluding_words:
print(f"\n--- Sample: Concluding Words ---")
for i, p in enumerate(concluding_words["paragraphs"][:3]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
# --- Quality Checks ---
print(f"\n--- Quality Checks ---")
# Check all 51 chapters present
expected_chapters = set(range(1, 52))
actual_chapters = set(ch["chapter"] for ch in chapters)
missing = expected_chapters - actual_chapters
extra = actual_chapters - expected_chapters
if missing:
print(f" ⚠ Missing chapters: {sorted(missing)}")
if extra:
print(f" ⚠ Extra chapters: {sorted(extra)}")
if not missing and not extra:
print(f" ✅ All 51 chapters present")
# Check all 3 intro segments present
expected_intros = {"front-flap", "preface", "introduction"}
actual_intros = {s["slug"] for s in intro_segments}
missing_intros = expected_intros - actual_intros
if missing_intros:
print(f" ⚠ Missing intro segments: {missing_intros}")
else:
print(f" ✅ All 3 intro segments present")
# Check "Thus ends" markers in chapters
chapters_with_end = 0
for ch in chapters:
has_end = any("Thus end" in p for p in ch["paragraphs"])
if has_end:
chapters_with_end += 1
print(f" ℹ {chapters_with_end}/{len(chapters)} chapters have 'Thus ends' markers")
# Check empty chapters
empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
if empty_chapters:
print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
else:
print(f" ✅ No empty chapters")
# Check ScaGoudy decoding — look for common encoded characters that should have been decoded
# Note: ñ is excluded because it's a valid IAST output character (ï→ñ mapping)
encoded_chars = set("äåöëòçüéàùè")
suspect_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
# Only flag if the encoded chars appear outside HTML tags
plain = re.sub(r"<[^>]+>", "", p)
for c in encoded_chars:
if c in plain:
# Many of these are legitimate uses (e.g., German loanwords)
# but in this context they almost always indicate ScaGoudy encoding
suspect_count += 1
break
if suspect_count == 0:
print(f" ✅ No suspected undecoded ScaGoudy characters found")
else:
print(
f" ℹ {suspect_count} paragraphs contain characters that may be undecoded ScaGoudy"
)
# Check part assignments
all_part_chapters = set()
for p in parts:
all_part_chapters.update(p["chapters"])
unassigned = [
ch["chapter"] for ch in chapters if ch["chapter"] not in all_part_chapters
]
if unassigned:
print(f" ⚠ Chapters not assigned to any part: {unassigned}")
else:
print(f" ✅ All chapters assigned to parts")
# Verify ScaGoudy decoding with a spot check on known text
print(f"\n--- ScaGoudy Decoding Spot Check ---")
# Chapter 1 should contain "Kṛṣṇa" (encoded as "Kåñëa")
if chapters:
ch1_text = " ".join(chapters[0]["paragraphs"][:5])
if "Kṛṣṇa" in ch1_text:
print(f" ✅ 'Kṛṣṇa' correctly decoded in Chapter 1")
elif "Kåñëa" in ch1_text:
print(f" ⚠ 'Kåñëa' still encoded — ScaGoudy decoding may have failed!")
else:
print(
f" ℹ 'Kṛṣṇa' not found in Chapter 1 first 5 paragraphs (may be normal)"
)
# Check for "Śrīmad-Bhāgavatam" (encoded as "Çrémad-Bhägavatam")
if "Śrīmad-Bhāgavatam" in ch1_text:
print(f" ✅ 'Śrīmad-Bhāgavatam' correctly decoded")
elif "Çrémad-Bhägavatam" in ch1_text:
print(f" ⚠ ScaGoudy decoding incomplete for 'Çrémad-Bhägavatam'")
print(f"\nDone! ✨")
if __name__ == "__main__":
main()
scripts/noi_extract.py
+859
View File
@@ -0,0 +1,859 @@
#!/usr/bin/env python3
"""Extract The Nectar of Instruction (1976 Edition) verses from other.html into structured JSON files."""
import json
import os
import re
import sys
from bs4 import BeautifulSoup, Tag
from indic_transliteration import sanscript
from indic_transliteration.sanscript import transliterate
# --- ScaGoudy → IAST Unicode mapping ---
SCAGOUDY_TO_IAST = {
"ä": "ā",
"Ä": "Ā",
"å": "ṛ",
"Å": "Ṛ",
"ñ": "ṣ",
"Ñ": "Ṣ",
"ö": "ṭ",
"Ö": "Ṭ",
"ë": "ṇ",
"Ë": "Ṇ",
"ò": "ḍ",
"Ò": "Ḍ",
"ç": "ś",
"Ç": "Ś",
"ì": "ṅ",
"ï": "ñ",
"ü": "ū",
"Ü": "Ū",
"é": "ī",
"É": "Ī",
"à": "ṁ",
"ù": "ḥ",
"è": "ṝ", # long vocalic r (U+1E5D)
"û": "ṁl", # anusvara + l (ScaGoudy ligature glyph)
"ÿ": "ḷ", # vocalic l (U+1E37)
}
# --- Text number to title mapping ---
TEXT_TITLES = {
1: "Text One",
2: "Text Two",
3: "Text Three",
4: "Text Four",
5: "Text Five",
6: "Text Six",
7: "Text Seven",
8: "Text Eight",
9: "Text Nine",
10: "Text Ten",
11: "Text Eleven",
}
# --- CSS class constants ---
BOOK_HEADER_CLASS = "p12"
SECTION_BOOKMARK_CLASS = "p64" # NoI section bookmarks (Front pages, Preface)
VERSE_BOOKMARK_CLASS = "p49" # NoI verse bookmarks (NoI: verse N) AND appendixes
VERSE_HEADING_CLASS = "p50"
DEVANAGARI_CLASS = "p51" # RM Devanagari — SKIP
TEXT_LABEL_CLASS = "p52"
PADA_TRANSLIT_CLASS = "p5"
SYNONYM_CLASS = "p54"
TRANSLATION_LABEL_CLASS = "p55"
TRANSLATION_TEXT_CLASS = "p56"
PURPORT_LABEL_CLASS = "p57"
PURPORT_BODY_CLASS = "p48"
QUOTED_VERSE_CLASSES = (
"p33",
"p53",
"p66",
) # Quoted Sanskrit/Bengali verses in purport
BODY_TEXT_CLASS = "p44" # Verse translations within purport
MISC_TEXT_CLASS = "p67" # Miscellaneous text (1 occurrence in NoI)
SECTION_HEADING_CLASS = "p31"
BODY_CLASSES = ("p6", "p45")
SKIP_CLASSES = ("p9", "p15", "p19", "p25", "p32", "p37", "p65")
def decode_scagoudy(text: str) -> str:
"""Convert ScaGoudy-encoded text to proper IAST Unicode."""
for src, dst in SCAGOUDY_TO_IAST.items():
text = text.replace(src, dst)
return text
def decode_html_content(tag: Tag) -> str:
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
inner = "".join(str(child) for child in tag.children)
inner = decode_scagoudy(inner)
# Remove span tags with CSS classes (artifacts from RTF conversion)
inner = re.sub(
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
)
# Clean up Apple-tab-span leftovers (tab characters)
inner = inner.replace("\t", " ")
# Normalize whitespace
inner = re.sub(r"\s+", " ", inner).strip()
return inner
def decode_text(tag: Tag) -> str:
"""Extract plain text from a tag and decode ScaGoudy."""
text = tag.get_text()
return decode_scagoudy(text).strip()
def normalize_devanagari_anusvara(text: str) -> str:
"""Replace explicit class nasals before same-class consonants with anusvāra.
Only normalize ङ् and ञ् — these are almost always written with anusvāra.
Keep ण्ड, न्त, म्ब etc. as explicit conjuncts (standard spelling).
"""
replacements = [
# ङ् before velars (क, ख, ग, घ)
(r"ङ्([कखगघ])", r"ं\1"),
# ञ् before palatals (च, छ, ज, झ)
(r"ञ्([चछजझ])", r"ं\1"),
]
for pattern, repl in replacements:
text = re.sub(pattern, repl, text)
return text
def join_virama_words(text: str) -> str:
"""Join Devanagari words where the previous word ends with virāma (्).
Handles both virāma+consonant and virāma+vowel cases.
Uses [ \\t]+ instead of \\s+ to avoid joining across newlines.
"""
# Join virāma + space + consonant (क U+0915 through ह U+0939)
text = re.sub(r"्[ \t]+([क-ह])", r"्\1", text)
# Join virāma + space + independent vowel
_VOWEL_TO_MATRA = {
"अ": "", # inherent vowel — just drop virāma and the अ
"आ": "ा",
"इ": "ि",
"ई": "ी",
"उ": "ु",
"ऊ": "ू",
"ऋ": "ृ",
"ॠ": "ॄ",
"ऌ": "ॢ",
"ॡ": "ॣ",
"ए": "े",
"ऐ": "ै",
"ओ": "ो",
"औ": "ौ",
}
def _virama_vowel_replace(m: re.Match[str]) -> str:
vowel = m.group(1)
return _VOWEL_TO_MATRA.get(vowel, vowel)
text = re.sub(r"्[ \t]+([अ-औॠॡ])", _virama_vowel_replace, text)
return text
def iast_to_devanagari(iast_text: str) -> str:
"""Convert IAST transliteration to Devanagari script."""
lines = iast_text.split("\n")
result_lines = []
for line in lines:
# Remove hyphens between words for Devanagari
clean = line.replace("-", "")
# Normalize ṁ (U+1E41) to ṃ (U+1E43) for proper anusvāra
clean = clean.replace("ṁ", "ṃ")
deva = transliterate(clean, sanscript.IAST, sanscript.DEVANAGARI)
deva = normalize_devanagari_anusvara(deva)
deva = join_virama_words(deva)
result_lines.append(deva)
return "\n".join(result_lines)
def parse_synonyms(text: str) -> list:
"""Parse synonym text into structured array.
Input format: "word—meaning; word—meaning; word—meaning"
"""
result = []
entries = text.split(";")
for entry in entries:
entry = entry.strip()
if not entry:
continue
if "—" in entry:
parts = entry.split("—", 1)
elif "--" in entry:
parts = entry.split("--", 1)
else:
continue
if len(parts) == 2:
word = parts[0].strip()
meaning = parts[1].strip()
# Clean word: remove trailing/leading punctuation
word = word.strip("., ")
if word and meaning:
result.append({"word": word, "meanings": [meaning]})
return result
def pair_padas(padas: list) -> str:
"""Pair pādas into verse lines (2 pādas per line).
A śloka has 4 pādas displayed as 2 lines. If there's an odd number,
the first pāda stands alone (usually a speaker attribution).
"""
if not padas:
return ""
lines = []
i = 0
# If odd number of pādas, first one is standalone
if len(padas) % 2 == 1:
lines.append(padas[0])
i = 1
# Pair remaining pādas
while i < len(padas) - 1:
lines.append(padas[i] + " " + padas[i + 1])
i += 2
# Handle any trailing single pāda
if i < len(padas):
lines.append(padas[i])
return "\n".join(lines)
def get_css_class(tag: Tag) -> str:
"""Get the primary CSS class of a tag."""
classes = tag.get("class", [])
return classes[0] if classes else ""
def is_empty(tag: Tag) -> bool:
"""Check if a tag is empty or whitespace-only."""
text = tag.get_text().strip()
return not text or text == "\xa0"
def is_dashed_line(text: str) -> bool:
"""Check if text is a dashed-line separator."""
clean = text.strip().replace("-", "").replace("—", "").strip()
return clean == ""
def parse_verse_bookmark(text: str) -> int | None:
"""Parse a verse bookmark like 'NoI: verse 1' through 'NoI: verse 11'.
Returns text number (1-11), or None if not a verse bookmark.
"""
text = text.strip()
m = re.match(r"NoI:\s*verse\s+(\d+)", text)
if m:
return int(m.group(1))
return None
# ---------------------------------------------------------------------------
# Boundary detection
# ---------------------------------------------------------------------------
def find_noi_boundaries(all_paragraphs: list) -> tuple[int, int]:
"""Find the start and end paragraph indices of the NoI content.
Start: <p class="p12"> containing "Nectar of Instruction" and "1976"
End: next <p class="p12"> after start (start of next book), or end of file
"""
start_idx = None
end_idx = len(all_paragraphs)
for i, p in enumerate(all_paragraphs):
cls = get_css_class(p)
if cls != BOOK_HEADER_CLASS:
continue
text = p.get_text().strip()
if "Nectar of Instruction" in text and "1976" in text:
start_idx = i
elif start_idx is not None:
# Next p12 after NoI start = end of NoI
end_idx = i
break
if start_idx is None:
print(
"ERROR: Could not find 'The Nectar of Instruction - 1976 Edition' header in other.html!"
)
sys.exit(1)
return start_idx, end_idx
def find_section_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> dict:
"""Find the key p64 bookmarks for NoI section boundaries.
Returns dict with bookmark text -> paragraph index.
"""
bookmarks = {}
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
if cls == SECTION_BOOKMARK_CLASS:
text = decode_text(p)
if "NoI" in text:
bookmarks[text] = i
return bookmarks
def find_verse_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
"""Find all p49 verse bookmark tags in the NoI range.
Returns list of (paragraph_index, text_number) for actual verse bookmarks.
Also returns appendixes bookmark index separately.
"""
verse_bookmarks = []
appendixes_idx = None
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
if cls == VERSE_BOOKMARK_CLASS:
text = decode_text(p)
text_num = parse_verse_bookmark(text)
if text_num is not None:
verse_bookmarks.append((i, text_num))
elif text.strip() == "NoI Appendixes":
appendixes_idx = i
return verse_bookmarks, appendixes_idx
# ---------------------------------------------------------------------------
# Intro segment extraction
# ---------------------------------------------------------------------------
def extract_intro_segment(
all_paragraphs: list, start_idx: int, end_idx: int
) -> list[str]:
"""Extract paragraphs for an intro segment (front pages, preface, or appendixes).
Decodes ScaGoudy, preserves HTML formatting, skips dashed lines and empty paragraphs.
"""
result = []
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
if is_empty(p):
continue
# Skip bookmark and header classes
if cls in (SECTION_BOOKMARK_CLASS, BOOK_HEADER_CLASS, VERSE_BOOKMARK_CLASS):
continue
# Skip noise classes
if cls in SKIP_CLASSES:
continue
html = decode_html_content(p)
if not html:
continue
# Skip dashed-line separators
plain = re.sub(r"<[^>]+>", "", html).strip()
if is_dashed_line(plain):
continue
result.append(html)
return result
# ---------------------------------------------------------------------------
# Verse extraction state machine
# ---------------------------------------------------------------------------
def extract_verses(all_paragraphs: list, verses_start: int, verses_end: int) -> list:
"""Extract verses from the NoI section using a state machine.
Returns list of verse dicts.
"""
verses = []
current_verse = None
state = "IDLE"
for i in range(verses_start, verses_end):
p = all_paragraphs[i]
cls = get_css_class(p)
# --- Verse bookmark (p49) ---
if cls == VERSE_BOOKMARK_CLASS:
# Finalize previous verse if any
if current_verse and state != "IDLE":
_finalize_verse(current_verse)
verses.append(current_verse)
text = decode_text(p)
text_num = parse_verse_bookmark(text)
if text_num is not None:
title = TEXT_TITLES.get(text_num, f"Text {text_num}")
reference = f"NoI {text_num}"
current_verse = {
"book": "noi",
"book_title": "The Nectar of Instruction",
"text": text_num,
"title": title,
"reference": reference,
"devanagari": "",
"transliteration": "",
"_padas": [],
"synonyms": [],
"translation": "",
"purport": [],
}
state = "VERSE_START"
else:
# Non-verse bookmark (e.g., "NoI Appendixes") — stop processing
current_verse = None
state = "IDLE"
continue
if not current_verse:
continue
# --- Verse heading (p50) ---
if cls == VERSE_HEADING_CLASS:
# Just confirms the text title; skip
continue
# --- RM Devanagari (p51) — SKIP ---
if cls == DEVANAGARI_CLASS:
continue
# --- TEXT label (p52) — skip ---
if cls == TEXT_LABEL_CLASS:
state = "TRANSLITERATION"
continue
# --- Pāda transliteration lines (p5) ---
if cls == PADA_TRANSLIT_CLASS and state in (
"VERSE_START",
"TRANSLITERATION",
):
state = "TRANSLITERATION"
line = decode_text(p)
# Remove trailing period
line = line.rstrip(".")
# Strip leading/trailing whitespace
line = line.strip()
if line:
current_verse["_padas"].append(line)
continue
# --- Synonyms (p54) ---
if cls == SYNONYM_CLASS and current_verse:
state = "SYNONYMS"
syn_text = decode_text(p)
syn_text = syn_text.strip()
parsed = parse_synonyms(syn_text)
current_verse["synonyms"].extend(parsed)
continue
# --- TRANSLATION / PURPORT label (p55) ---
if cls == TRANSLATION_LABEL_CLASS and current_verse:
label_text = decode_text(p).strip().upper()
if "TRANSLATION" in label_text:
state = "TRANSLATION"
elif "PURPORT" in label_text:
state = "PURPORT"
else:
# Other labels within purport (e.g., "The Author", "Glossary")
if state == "PURPORT":
html = decode_html_content(p)
if html:
current_verse["purport"].append(html)
continue
# --- Translation text (p56) ---
if cls == TRANSLATION_TEXT_CLASS and current_verse and state == "TRANSLATION":
html = decode_html_content(p)
# Remove bold wrapper if present (the whole translation is bold)
html = re.sub(r"^<b>(.*)</b>$", r"\1", html, flags=re.DOTALL)
html = html.strip()
current_verse["translation"] = html
state = "POST_TRANSLATION"
continue
# --- PURPORT label (p57) ---
if cls == PURPORT_LABEL_CLASS and current_verse:
state = "PURPORT"
continue
# --- Purport body paragraphs (p48) ---
if state == "PURPORT" and current_verse and cls == PURPORT_BODY_CLASS:
html = decode_html_content(p)
if html:
plain = re.sub(r"<[^>]+>", "", html).strip()
if not is_dashed_line(plain):
current_verse["purport"].append(html)
continue
# --- Quoted Sanskrit/Bengali verses within purport (p33, p53, p66) ---
if state == "PURPORT" and current_verse and cls in QUOTED_VERSE_CLASSES:
html = decode_html_content(p)
if html:
# Strip inner <i>/<\/i> tags to avoid double wrapping
stripped = re.sub(r"</?i>", "", html)
current_verse["purport"].append(
f'<div style="text-align:center"><i>{stripped}</i></div>'
)
continue
# --- Body text / verse translations within purport (p44) ---
if state == "PURPORT" and current_verse and cls == BODY_TEXT_CLASS:
html = decode_html_content(p)
if html:
plain = re.sub(r"<[^>]+>", "", html).strip()
if not is_dashed_line(plain):
current_verse["purport"].append(html)
continue
# --- Miscellaneous text (p67) — treat as body text within purport ---
if state == "PURPORT" and current_verse and cls == MISC_TEXT_CLASS:
html = decode_html_content(p)
if html:
current_verse["purport"].append(html)
continue
# --- Section heading within purport (p31) ---
if state == "PURPORT" and current_verse and cls == SECTION_HEADING_CLASS:
html = decode_html_content(p)
if html:
current_verse["purport"].append(html)
continue
# --- Body text classes that might appear in purport (p6, p45) ---
if state == "PURPORT" and current_verse and cls in BODY_CLASSES:
html = decode_html_content(p)
if html:
plain = re.sub(r"<[^>]+>", "", html).strip()
if not is_dashed_line(plain):
current_verse["purport"].append(html)
continue
# Finalize last verse
if current_verse and state != "IDLE":
_finalize_verse(current_verse)
verses.append(current_verse)
return verses
def _finalize_verse(verse: dict):
"""Build transliteration and Devanagari from pādas. Remove internal fields."""
padas = verse.get("_padas", [])
# Transliteration: one pāda per line
verse["transliteration"] = "\n".join(padas)
# Devanagari: pair pādas into 2-line verse structure, then convert
if padas:
paired = pair_padas(padas)
verse["devanagari"] = iast_to_devanagari(paired)
# Clean up internal field
if "_padas" in verse:
del verse["_padas"]
# ---------------------------------------------------------------------------
# JSON writing
# ---------------------------------------------------------------------------
def write_json(data, filepath):
"""Write JSON with proper Unicode and formatting."""
os.makedirs(os.path.dirname(filepath), exist_ok=True)
with open(filepath, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False, indent=2)
print(f" Written: {filepath}")
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
def main():
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
html_path = os.path.join(base_dir, "public", "html", "other.html")
output_base = os.path.join(base_dir, "lib", "jsons", "books", "noi")
if not os.path.exists(html_path):
print(f"ERROR: {html_path} not found!")
sys.exit(1)
print(f"Reading {html_path}...")
with open(html_path, "r", encoding="utf-8") as f:
soup = BeautifulSoup(f, "html.parser")
all_paragraphs = soup.find_all("p")
print(f"Total paragraphs in file: {len(all_paragraphs)}")
# --- Find NoI boundaries ---
noi_start, noi_end = find_noi_boundaries(all_paragraphs)
print(f"NoI content: paragraphs {noi_start} to {noi_end}")
# --- Find section bookmarks (p64) ---
section_bookmarks = find_section_bookmarks(all_paragraphs, noi_start, noi_end)
print(f"\nSection bookmarks (p64) found:")
for bm_text, bm_idx in sorted(section_bookmarks.items(), key=lambda x: x[1]):
print(f" [{bm_idx}] {bm_text}")
# --- Find verse bookmarks (p49) ---
verse_bookmarks, appendixes_idx = find_verse_bookmarks(
all_paragraphs, noi_start, noi_end
)
print(f"\nVerse bookmarks (p49) found: {len(verse_bookmarks)}")
for bm_idx, text_num in verse_bookmarks:
print(f" [{bm_idx}] NoI: verse {text_num}")
if appendixes_idx is not None:
print(f" [{appendixes_idx}] NoI Appendixes")
# Key bookmark indices
front_pages_idx = section_bookmarks.get("NoI Front pages")
preface_idx = section_bookmarks.get("NoI Preface")
# First verse bookmark
first_verse_idx = verse_bookmarks[0][0] if verse_bookmarks else None
if first_verse_idx is None:
print("ERROR: Could not find any verse bookmarks!")
sys.exit(1)
# End boundary: the Glossary <p class="p55"><b>Glossary</b></p> or noi_end
end_boundary = noi_end
# Look for the Glossary marker just before the next book
for i in range(noi_start, noi_end):
p = all_paragraphs[i]
cls = get_css_class(p)
if cls == TRANSLATION_LABEL_CLASS:
text = decode_text(p).strip()
if text.upper() == "GLOSSARY":
end_boundary = i
break
# --- Extract intro segments ---
print("\n--- Extracting Intro Segments ---")
intro_segments = []
# 1. Front pages segment — from NoI Front pages (p64) to NoI Preface (p64)
front_end = preface_idx if preface_idx is not None else first_verse_idx
if front_pages_idx is not None and front_end is not None:
front_paras = extract_intro_segment(all_paragraphs, front_pages_idx, front_end)
front_segment = {
"slug": "front-pages",
"title": "Front Pages",
"type": "intro-segment",
"book": "noi",
"order": 1,
"paragraphs": front_paras,
}
intro_segments.append(front_segment)
print(f" Front Pages: {len(front_paras)} paragraphs")
# 2. Preface — from NoI Preface (p64) to first verse bookmark
if preface_idx is not None and first_verse_idx is not None:
preface_paras = extract_intro_segment(
all_paragraphs, preface_idx, first_verse_idx
)
preface_segment = {
"slug": "preface",
"title": "Preface",
"type": "intro-segment",
"book": "noi",
"order": 2,
"paragraphs": preface_paras,
}
intro_segments.append(preface_segment)
print(f" Preface: {len(preface_paras)} paragraphs")
# 3. Appendixes — from NoI Appendixes (p49) to end boundary
if appendixes_idx is not None:
appendixes_paras = extract_intro_segment(
all_paragraphs, appendixes_idx, end_boundary
)
appendixes_segment = {
"slug": "appendixes",
"title": "Appendixes",
"type": "intro-segment",
"book": "noi",
"order": 3,
"paragraphs": appendixes_paras,
}
intro_segments.append(appendixes_segment)
print(f" Appendixes: {len(appendixes_paras)} paragraphs")
# --- Extract verses ---
print("\n--- Extracting Verses ---")
# Verse section runs from first verse bookmark to appendixes (or end boundary)
verse_end = appendixes_idx if appendixes_idx is not None else end_boundary
verses = extract_verses(all_paragraphs, first_verse_idx, verse_end)
print(f"Extracted {len(verses)} texts (expected 11: Texts 1-11)")
# --- Write intro.json ---
texts_listing = []
for num in range(1, 12):
title = TEXT_TITLES[num]
slug = str(num).zfill(2)
texts_listing.append({"number": num, "title": title, "slug": slug})
intro_meta = {
"book": "noi",
"title": "The Nectar of Instruction",
"subtitle": "Śrī Upadeśāmṛta",
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
"year": 1976,
"description": "Eleven essential instructions on the science of bhakti-yoga by Śrīla Rūpa Gosvāmī.",
"totalTexts": 11,
"texts": texts_listing,
}
write_json(intro_meta, os.path.join(output_base, "intro.json"))
# --- Write intro segments ---
print("\n--- Writing Intro Segments ---")
for seg in intro_segments:
filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
write_json(seg, filepath)
# --- Write verse JSON files ---
print("\n--- Writing Verse Files ---")
for verse in verses:
text_num = verse["text"]
filename = f"{str(text_num).zfill(2)}.json"
filepath = os.path.join(output_base, "verses", filename)
write_json(verse, filepath)
# --- Summary ---
print(f"\n{'=' * 60}")
print(f" EXTRACTION SUMMARY")
print(f"{'=' * 60}")
print(f" Book: The Nectar of Instruction (1976 Edition)")
print(f" Intro segments: {len(intro_segments)}")
for seg in intro_segments:
print(
f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs"
)
print(f" Texts extracted: {len(verses)}")
texts_found = sorted([v["text"] for v in verses])
if texts_found:
print(
f" Texts: {texts_found[0]}-{texts_found[-1]} ({len(texts_found)} total)"
)
else:
print(" Texts: None")
total_purport_paras = sum(len(v["purport"]) for v in verses)
total_synonyms = sum(len(v["synonyms"]) for v in verses)
print(f" Total purport ¶s: {total_purport_paras}")
print(f" Total synonyms: {total_synonyms}")
print(f" Output directory: {output_base}/")
# --- Quality checks ---
print(f"\n--- Quality Checks ---")
# Check all verses have expected fields
issues = []
for verse in verses:
ref = verse["reference"]
if not verse["transliteration"]:
issues.append(f" {ref}: missing transliteration")
if not verse["devanagari"]:
issues.append(f" {ref}: missing devanagari")
if not verse["synonyms"]:
issues.append(f" {ref}: missing synonyms")
if not verse["translation"]:
issues.append(f" {ref}: missing translation")
if not verse["purport"]:
issues.append(f" {ref}: missing purport")
if issues:
print(" ISSUES FOUND:")
for issue in issues:
print(issue)
else:
print(
" All texts have transliteration, devanagari, synonyms, translation, and purport ✓"
)
# Check expected text numbers are present
expected = set(range(1, 12))
found = set(texts_found)
missing = expected - found
extra = found - expected
if missing:
print(f" MISSING texts: {sorted(missing)}")
if extra:
print(f" UNEXPECTED texts: {sorted(extra)}")
if not missing and not extra:
print(f" All 11 texts present ✓")
# --- Samples ---
print(f"\n--- Sample: Text One ---")
t1 = [v for v in verses if v["text"] == 1]
if t1:
v = t1[0]
print(f" Reference: {v['reference']}")
print(f" Title: {v['title']}")
print(f" Transliteration: {v['transliteration'][:80]}...")
print(f" Devanagari: {v['devanagari'][:80]}...")
print(f" Synonyms: {len(v['synonyms'])} entries")
if v["synonyms"]:
print(f" First: {v['synonyms'][0]}")
print(f" Last: {v['synonyms'][-1]}")
trans_preview = re.sub(r"<[^>]+>", "", v["translation"])[:100]
print(f" Translation: {trans_preview}...")
print(f" Purport ¶s: {len(v['purport'])}")
print(f"\n--- Sample: Text Six (middle) ---")
t6 = [v for v in verses if v["text"] == 6]
if t6:
v = t6[0]
print(f" Reference: {v['reference']}")
print(f" Title: {v['title']}")
print(f" Transliteration: {v['transliteration'][:80]}...")
print(f" Devanagari: {v['devanagari'][:80]}...")
print(f" Synonyms: {len(v['synonyms'])} entries")
trans_preview = re.sub(r"<[^>]+>", "", v["translation"])[:100]
print(f" Translation: {trans_preview}...")
print(f" Purport ¶s: {len(v['purport'])}")
print(f"\n--- Sample: Text Eleven (last) ---")
t11 = [v for v in verses if v["text"] == 11]
if t11:
v = t11[0]
print(f" Reference: {v['reference']}")
print(f" Title: {v['title']}")
print(f" Transliteration: {v['transliteration'][:80]}...")
print(f" Devanagari: {v['devanagari'][:80]}...")
print(f" Synonyms: {len(v['synonyms'])} entries")
trans_preview = re.sub(r"<[^>]+>", "", v["translation"])[:100]
print(f" Translation: {trans_preview}...")
print(f" Purport ¶s: {len(v['purport'])}")
if __name__ == "__main__":
main()
scripts/ochkm_extract.py
+593
View File
@@ -0,0 +1,593 @@
#!/usr/bin/env python3
"""Extract On Chanting the Hare Kṛṣṇa Mantra (OCHKM) from other.html.
The book has 1 intro segment (front-pages) and 4 chapters/sections.
There are NO p13 bookmarks — chapters are detected via p27 headings.
Output:
lib/jsons/books/ochkm/intro.json — Book metadata
lib/jsons/books/ochkm/intro/*.json — Intro segments
lib/jsons/books/ochkm/chapters/*.json — Chapters
Usage:
python3 scripts/ochkm_extract.py
"""
import json
import os
import re
import sys
from bs4 import BeautifulSoup, Tag
# ---------------------------------------------------------------------------
# ScaGoudy → IAST Unicode mapping
# ---------------------------------------------------------------------------
SCAGOUDY_TO_IAST = {
"ä": "ā",
"Ä": "Ā",
"å": "ṛ",
"Å": "Ṛ",
"ñ": "ṣ",
"Ñ": "Ṣ",
"ö": "ṭ",
"Ö": "Ṭ",
"ë": "ṇ",
"Ë": "Ṇ",
"ò": "ḍ",
"Ò": "Ḍ",
"ç": "ś",
"Ç": "Ś",
"ì": "ṅ",
"ï": "ñ",
"ü": "ū",
"Ü": "Ū",
"é": "ī",
"É": "Ī",
"à": "ṁ",
"ù": "ḥ",
"è": "ṝ",
"û": "ṁl",
"ÿ": "ḷ",
}
def decode_scagoudy(text: str) -> str:
"""Replace ScaGoudy-encoded characters with proper IAST Unicode."""
for src, dst in SCAGOUDY_TO_IAST.items():
text = text.replace(src, dst)
return text
def decode_html_content(tag: Tag) -> str:
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
inner = "".join(str(child) for child in tag.children)
inner = decode_scagoudy(inner)
# Remove span tags with CSS classes (RTF conversion artifacts)
inner = re.sub(
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
)
# Clean up Apple-tab-span leftovers (tab characters)
inner = inner.replace("\t", " ")
# Normalize whitespace (but preserve <br> tags)
inner = re.sub(r"\s+", " ", inner).strip()
return inner
def decode_text(tag: Tag) -> str:
"""Get plain text from a tag with ScaGoudy decoding."""
return decode_scagoudy(tag.get_text().strip())
def get_css_class(tag: Tag) -> str:
"""Get the first CSS class of a tag, or empty string."""
classes = tag.get("class", [])
return classes[0] if classes else ""
def is_empty(tag: Tag) -> bool:
"""Check if a tag is empty or whitespace-only."""
text = tag.get_text().strip()
return not text or text == "\xa0"
def is_dash_line(text: str) -> bool:
"""Return True if text is only dashes, spaces, and/or whitespace."""
cleaned = text.strip()
if not cleaned:
return False
return all(c in "-—–\u2014\u2013 \t" for c in cleaned)
# ---------------------------------------------------------------------------
# CSS class categorization for OCHKM
# ---------------------------------------------------------------------------
# Book title header class
BOOK_HEADER_CLASS = "p12"
# Chapter heading class (used as chapter boundary since no p13 bookmarks)
CHAPTER_HEADING_CLASS = "p27"
# Body paragraph classes (all treated as paragraphs)
BODY_CLASSES = frozenset(["p6", "p19", "p76", "p25", "p21", "p100"])
# Classes to skip entirely
SKIP_CLASSES = frozenset(["p9", "p15", "p70"])
# Chapter titles mapping (proper IAST decoded)
CHAPTER_TITLES = {
1: "Brief Biography of the Spiritual Master",
2: "On Chanting the Hare Kṛṣṇa Mantra",
3: "What is ISKCON?",
4: "Śrī Śrī Śikṣāṣṭakam",
}
# ---------------------------------------------------------------------------
# Extraction helpers
# ---------------------------------------------------------------------------
def find_ochkm_boundaries(all_paragraphs: list) -> tuple:
"""Find the start and end paragraph indices of the OCHKM content.
Start: <p class="p12"> containing "ON CHANTING THE HARE"
End: next <p class="p12"> after start
"""
start_idx = None
end_idx = len(all_paragraphs)
for i, p in enumerate(all_paragraphs):
cls = get_css_class(p)
if cls != BOOK_HEADER_CLASS:
continue
text = p.get_text().strip().upper()
if "ON CHANTING THE HARE" in text:
start_idx = i
elif start_idx is not None:
# Next p12 after OCHKM start = end of OCHKM
end_idx = i
break
if start_idx is None:
print(
"ERROR: Could not find 'ON CHANTING THE HARE KṚṢṆA MANTRA' header in other.html!"
)
sys.exit(1)
return start_idx, end_idx
def find_chapter_headings(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
"""Find all chapter headings (p27) in the OCHKM range.
Returns list of (paragraph_index, sequential_chapter_number, chapter_title).
"""
headings = []
seq = 0
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
if cls != CHAPTER_HEADING_CLASS:
continue
text = decode_text(p)
# Remove bold/italic tags from text content
text = re.sub(r"</?[bi]>", "", text).strip()
if not text:
continue
seq += 1
# Use pre-defined proper IAST titles
ch_title = CHAPTER_TITLES.get(seq, text)
headings.append((i, seq, ch_title))
return headings
def extract_paragraphs(
all_paragraphs: list,
start_idx: int,
end_idx: int,
is_front_pages: bool = False,
) -> list:
"""Extract paragraphs for a section (chapter or intro segment).
Handles all CSS classes within the OCHKM content.
Detects italic+br verses in any body class → centered divs.
"""
result = []
verse_accumulator = []
def flush_verse():
if verse_accumulator:
joined = "<br>".join(verse_accumulator)
result.append(f'<div style="text-align:center"><i>{joined}</i></div>')
verse_accumulator.clear()
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
# Skip empty paragraphs
if is_empty(p):
continue
# Skip noise classes
if cls in SKIP_CLASSES:
continue
# Skip book title header (p12)
if cls == BOOK_HEADER_CLASS:
continue
# Skip chapter headings (p27) — they are boundaries, not content
if cls == CHAPTER_HEADING_CLASS:
continue
# Check for dash lines — skip them
text = decode_text(p)
if is_dash_line(text):
continue
# Non-verse paragraph — flush any accumulated verse first
flush_verse()
# Body paragraphs (p6, p19, p76, p25, p21, p100)
if cls in BODY_CLASSES:
html = decode_html_content(p)
if html:
# Detect Sanskrit verses disguised as body paragraphs:
# they start with <i> and contain <br> (multi-line italic = verse)
stripped_html = html.strip()
if stripped_html.startswith("<i>") and "<br" in stripped_html:
# This is a verse — wrap in centered italic div
inner = re.sub(r"</?i>", "", stripped_html)
result.append(
f'<div style="text-align:center"><i>{inner}</i></div>'
)
else:
result.append(html)
continue
# Any other class — try to extract content (be permissive)
html = decode_html_content(p)
if html and not is_dash_line(text):
result.append(html)
# Flush any trailing verse
flush_verse()
return result
# ---------------------------------------------------------------------------
# Main extraction logic
# ---------------------------------------------------------------------------
def extract_ochkm(soup: BeautifulSoup) -> dict:
"""Extract the entire OCHKM book from the parsed HTML.
Returns a dict with:
"intro_segments": list of intro segment dicts
"chapters": list of chapter dicts
"chapter_listing": list of {number, title} for intro.json
"""
all_paragraphs = list(soup.find_all("p"))
# Step 1: Find OCHKM boundaries
ochkm_start, ochkm_end = find_ochkm_boundaries(all_paragraphs)
print(f" OCHKM content: paragraphs {ochkm_start} to {ochkm_end}")
print(f" Total paragraphs in range: {ochkm_end - ochkm_start}")
# Step 2: Find all chapter headings (p27)
chapter_headings = find_chapter_headings(all_paragraphs, ochkm_start, ochkm_end)
print(f" Found {len(chapter_headings)} chapter headings")
for pos, ch_num, ch_title in chapter_headings:
print(f" [chapter {ch_num}] paragraph {pos}: {ch_title}")
# Step 3: Extract front pages
# From book start (ochkm_start) to first chapter heading
intro_segments = []
if chapter_headings:
front_pages_end = chapter_headings[0][0]
else:
front_pages_end = ochkm_end
front_pages_paragraphs = extract_paragraphs(
all_paragraphs, ochkm_start, front_pages_end, is_front_pages=True
)
intro_segments.append(
{
"slug": "front-pages",
"title": "Front Pages",
"type": "intro-segment",
"book": "ochkm",
"order": 1,
"paragraphs": front_pages_paragraphs,
}
)
# Step 4: Extract chapters
chapters = []
for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_headings):
# Content starts after the heading
content_start = ch_pos + 1
# Content ends at next chapter heading or ochkm_end
if idx + 1 < len(chapter_headings):
content_end = chapter_headings[idx + 1][0]
else:
content_end = ochkm_end
paragraphs = extract_paragraphs(all_paragraphs, content_start, content_end)
ch = {
"book": "ochkm",
"chapter": ch_num,
"title": ch_title,
"type": "chapter",
"paragraphs": paragraphs,
}
chapters.append(ch)
# Step 5: Build chapter listing for intro.json
chapter_listing = [
{"number": ch["chapter"], "title": ch["title"]} for ch in chapters
]
return {
"intro_segments": intro_segments,
"chapters": chapters,
"chapter_listing": chapter_listing,
}
# ---------------------------------------------------------------------------
# Output
# ---------------------------------------------------------------------------
def write_json(data, filepath):
"""Write data as formatted JSON to filepath, creating directories as needed."""
os.makedirs(os.path.dirname(filepath), exist_ok=True)
with open(filepath, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False, indent=2)
def main():
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
html_path = os.path.join(base_dir, "public", "html", "other.html")
output_base = os.path.join(base_dir, "lib", "jsons", "books", "ochkm")
if not os.path.exists(html_path):
print(f"ERROR: {html_path} not found!")
sys.exit(1)
print(f"Reading {html_path}...")
with open(html_path, "r", encoding="utf-8") as f:
soup = BeautifulSoup(f, "html.parser")
print("Extracting On Chanting the Hare Kṛṣṇa Mantra...\n")
result = extract_ochkm(soup)
intro_segments = result["intro_segments"]
chapters = result["chapters"]
chapter_listing = result["chapter_listing"]
# --- Write intro.json ---
intro_meta = {
"book": "ochkm",
"title": "On Chanting the Hare Kṛṣṇa Mantra",
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
"year": 1967,
"description": "The sublime method for reviving our transcendental consciousness through the chanting of the holy names.",
"totalChapters": len(chapters),
"chapters": chapter_listing,
}
intro_path = os.path.join(output_base, "intro.json")
write_json(intro_meta, intro_path)
print(f"\n Written: {intro_path}")
# --- Write intro segments ---
print(f"\n--- Intro Segments ---")
for seg in intro_segments:
filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
write_json(seg, filepath)
print(
f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}"
)
# --- Write chapters ---
print(f"\n--- Chapters ---")
for ch in chapters:
ch_num_str = str(ch["chapter"]).zfill(2)
filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
write_json(ch, filepath)
print(
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
f"{ch['title']}"
)
# --- Summary ---
total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments)
total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
total_paras = total_intro_paras + total_chapter_paras
print(f"\n{'=' * 60}")
print(f" EXTRACTION SUMMARY")
print(f"{'=' * 60}")
print(f" Intro segments: {len(intro_segments)}")
print(f" Chapters: {len(chapters)}")
print(f" Total paragraphs: {total_paras}")
print(f" Intro: {total_intro_paras}")
print(f" Chapters: {total_chapter_paras}")
if chapters:
print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
print(f" Output directory: {output_base}/")
# --- Samples ---
print(f"\n--- Sample: Front Pages ---")
if intro_segments:
front = intro_segments[0]
for i, p in enumerate(front["paragraphs"][:5]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
if chapters:
print(f"\n--- Sample: Chapter 1 ({chapters[0]['title']}) ---")
ch1 = chapters[0]
for i, p in enumerate(ch1["paragraphs"][:5]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
if chapters and len(chapters) > 1:
print(
f"\n--- Sample: Chapter {chapters[-1]['chapter']} ({chapters[-1]['title']}) ---"
)
last = chapters[-1]
for i, p in enumerate(last["paragraphs"][:5]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
# --- Quality Checks ---
print(f"\n--- Quality Checks ---")
# Check all 4 chapters present
expected_chapters = set(range(1, 5))
actual_chapters = set(ch["chapter"] for ch in chapters)
missing = expected_chapters - actual_chapters
extra = actual_chapters - expected_chapters
if missing:
print(f" ⚠ Missing chapters: {sorted(missing)}")
if extra:
print(f" ⚠ Extra chapters: {sorted(extra)}")
if not missing and not extra:
print(f" ✅ All 4 chapters present")
# Check all 1 intro segment present
expected_intros = {"front-pages"}
actual_intros = {s["slug"] for s in intro_segments}
missing_intros = expected_intros - actual_intros
extra_intros = actual_intros - expected_intros
if missing_intros:
print(f" ⚠ Missing intro segments: {missing_intros}")
if extra_intros:
print(f" ⚠ Extra intro segments: {extra_intros}")
if not missing_intros and not extra_intros:
print(f" ✅ All 1 intro segments present")
# Check empty chapters
empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
if empty_chapters:
print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
else:
print(f" ✅ No empty chapters")
# Check empty intro segments
empty_intros = [s["slug"] for s in intro_segments if len(s["paragraphs"]) == 0]
if empty_intros:
print(f" ⚠ Empty intro segments: {empty_intros}")
else:
print(f" ✅ No empty intro segments")
# Check ScaGoudy decoding
print(f"\n--- ScaGoudy Decoding Check ---")
all_text = ""
for ch in chapters:
all_text += " ".join(ch["paragraphs"][:20])
for seg in intro_segments:
all_text += " ".join(seg["paragraphs"][:10])
if "Kṛṣṇa" in all_text:
print(f" ✅ 'Kṛṣṇa' found (ScaGoudy properly decoded)")
else:
print(f" ⚠ 'Kṛṣṇa' not found — ScaGoudy decoding may have failed")
if "Prabhupāda" in all_text:
print(f" ✅ 'Prabhupāda' found (ScaGoudy properly decoded)")
if "saṅkīrtana" in all_text or "saṅkértana" in all_text or "saṁkīrtana" in all_text:
print(f" ✅ 'saṅkīrtana' variant found (ScaGoudy properly decoded)")
# Check for un-decoded ScaGoudy remnants
scagoudy_remnants = 0
for ch in chapters:
for p in ch["paragraphs"]:
if "Kåñëa" in p or "Prabhupäda" in p:
scagoudy_remnants += 1
for seg in intro_segments:
for p in seg["paragraphs"]:
if "Kåñëa" in p or "Prabhupäda" in p:
scagoudy_remnants += 1
if scagoudy_remnants == 0:
print(f" ✅ No ScaGoudy remnants detected")
else:
print(f" ⚠ {scagoudy_remnants} paragraphs with ScaGoudy remnants")
# Check for centered verse blocks
verse_block_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
if '<div style="text-align:center"><i>' in p:
verse_block_count += 1
for seg in intro_segments:
for p in seg["paragraphs"]:
if '<div style="text-align:center"><i>' in p:
verse_block_count += 1
if verse_block_count > 0:
print(f" ✅ {verse_block_count} centered Sanskrit verse blocks found")
else:
print(f" ℹ No centered verse blocks found (may be expected for this book)")
# Check for residual span tags
span_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
if 'class="' in p and "<span" in p:
span_count += 1
for seg in intro_segments:
for p in seg["paragraphs"]:
if 'class="' in p and "<span" in p:
span_count += 1
if span_count == 0:
print(f" ✅ No residual span class artifacts found")
else:
print(f" ⚠ {span_count} paragraphs still contain span class artifacts")
# Check for dash-line paragraphs that shouldn't be there
dash_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
plain = re.sub(r"<[^>]+>", "", p)
if is_dash_line(plain):
dash_count += 1
for seg in intro_segments:
for p in seg["paragraphs"]:
plain = re.sub(r"<[^>]+>", "", p)
if is_dash_line(plain):
dash_count += 1
if dash_count == 0:
print(f" ✅ No dashed-line paragraphs in output")
else:
print(f" ⚠ {dash_count} dashed-line paragraphs found in output")
# Chapter title decoding check
print(f"\n--- Chapter Title Decoding Check ---")
for ch in chapters:
title = ch["title"]
if "å" in title or "ë" in title or "ñ" in title or "ä" in title:
print(f" ⚠ Chapter {ch['chapter']} title has un-decoded ScaGoudy: {title}")
else:
print(f" ✅ Ch {ch['chapter']}: {title}")
print(f"\nDone! ✨")
if __name__ == "__main__":
main()
scripts/owk_extract.py
+668
View File
@@ -0,0 +1,668 @@
#!/usr/bin/env python3
"""Extract On the Way to Kṛṣṇa (1973) from other.html.
The book has 1 intro segment (front-pages) and 5 chapters.
Output:
lib/jsons/books/owk/intro.json — Book metadata
lib/jsons/books/owk/intro/*.json — Intro segments
lib/jsons/books/owk/chapters/*.json — Chapters
Usage:
python3 scripts/owk_extract.py
"""
import json
import os
import re
import sys
from bs4 import BeautifulSoup, Tag
# ---------------------------------------------------------------------------
# ScaGoudy → IAST Unicode mapping
# ---------------------------------------------------------------------------
SCAGOUDY_TO_IAST = {
"ä": "ā",
"Ä": "Ā",
"å": "ṛ",
"Å": "Ṛ",
"ñ": "ṣ",
"Ñ": "Ṣ",
"ö": "ṭ",
"Ö": "Ṭ",
"ë": "ṇ",
"Ë": "Ṇ",
"ò": "ḍ",
"Ò": "Ḍ",
"ç": "ś",
"Ç": "Ś",
"ì": "ṅ",
"ï": "ñ",
"ü": "ū",
"Ü": "Ū",
"é": "ī",
"É": "Ī",
"à": "ṁ",
"ù": "ḥ",
"è": "ṝ",
"û": "ṁl",
"ÿ": "ḷ",
}
def decode_scagoudy(text: str) -> str:
"""Replace ScaGoudy-encoded characters with proper IAST Unicode."""
for src, dst in SCAGOUDY_TO_IAST.items():
text = text.replace(src, dst)
return text
def decode_html_content(tag: Tag) -> str:
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
inner = "".join(str(child) for child in tag.children)
inner = decode_scagoudy(inner)
# Remove span tags with CSS classes (RTF conversion artifacts)
inner = re.sub(
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
)
# Clean up Apple-tab-span leftovers (tab characters)
inner = inner.replace("\t", " ")
# Normalize whitespace (but preserve <br> tags)
inner = re.sub(r"\s+", " ", inner).strip()
return inner
def decode_text(tag: Tag) -> str:
"""Get plain text from a tag with ScaGoudy decoding."""
return decode_scagoudy(tag.get_text().strip())
def get_css_class(tag: Tag) -> str:
"""Get the first CSS class of a tag, or empty string."""
classes = tag.get("class", [])
return classes[0] if classes else ""
def is_empty(tag: Tag) -> bool:
"""Check if a tag is empty or whitespace-only."""
text = tag.get_text().strip()
return not text or text == "\xa0"
def is_dash_line(text: str) -> bool:
"""Return True if text is only dashes, spaces, and/or whitespace."""
cleaned = text.strip()
if not cleaned:
return False
return all(c in "-—–\u2014\u2013 \t" for c in cleaned)
# ---------------------------------------------------------------------------
# CSS class categorization for OWK
# ---------------------------------------------------------------------------
# Book title header class (OWK uses p85, NOT p12)
BOOK_HEADER_CLASS = "p85"
# Chapter bookmark class
CHAPTER_BOOKMARK_CLASS = "p13"
# Body paragraph classes (primary content)
BODY_CLASSES = frozenset(["p6"])
# Sanskrit verse pāda classes (italic, centered)
VERSE_PADA_CLASSES = frozenset(["p53"])
# Bold quote paragraphs — include as body
BOLD_QUOTE_CLASSES = frozenset(["p56", "p30"])
# Verse translations / body text
TRANSLATION_CLASS = "p44"
# Chapter heading (bold)
CHAPTER_HEADING_CLASS = "p27"
# Title page text
TITLE_PAGE_CLASS = "p3"
# Other books listing text
OTHER_BOOKS_CLASS = "p18"
# Contents header
CONTENTS_CLASS = "p69"
# Misc text (p86 — bold quote paragraph)
MISC_CLASS = "p86"
# Classes to skip entirely (empty/decorative)
SKIP_CLASSES = frozenset(["p9", "p70"])
# ---------------------------------------------------------------------------
# Extraction helpers
# ---------------------------------------------------------------------------
def find_owk_boundaries(all_paragraphs: list) -> tuple:
"""Find the start and end paragraph indices of the OWK content.
Start: <p class="p85"> containing "On the Way" and "1973"
End: next <p class="p85"> after start (RĀJA-VIDYĀ)
"""
start_idx = None
end_idx = len(all_paragraphs)
for i, p in enumerate(all_paragraphs):
cls = get_css_class(p)
if cls != BOOK_HEADER_CLASS:
continue
text = decode_text(p)
if "On the Way" in text and "1973" in text:
start_idx = i
elif start_idx is not None:
# Next p85 after OWK start = end of OWK
end_idx = i
break
if start_idx is None:
print(
"ERROR: Could not find 'On the Way to Kṛṣṇa (1973)' header in other.html!"
)
sys.exit(1)
return start_idx, end_idx
def find_chapter_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
"""Find all OWK chapter bookmark tags (p13) in the OWK range.
Returns list of (paragraph_index, chapter_number, chapter_title).
"""
bookmarks = []
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
if cls == CHAPTER_BOOKMARK_CLASS:
text = decode_text(p)
m = re.match(r"OWK\s+(\d+):\s+(.+)", text.strip())
if m:
ch_num = int(m.group(1))
ch_title = m.group(2).strip()
bookmarks.append((i, ch_num, ch_title))
return bookmarks
def extract_paragraphs(
all_paragraphs: list,
start_idx: int,
end_idx: int,
is_front_pages: bool = False,
) -> list:
"""Extract paragraphs for a section (chapter or intro segment).
Handles all CSS classes within the OWK content.
Groups consecutive p53 verse pādas into centered italic divs.
"""
result = []
verse_accumulator = []
def flush_verse():
if verse_accumulator:
joined = "<br>".join(verse_accumulator)
result.append(f'<div style="text-align:center"><i>{joined}</i></div>')
verse_accumulator.clear()
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
# Skip empty paragraphs
if is_empty(p):
continue
# Skip noise classes
if cls in SKIP_CLASSES:
continue
# Skip bookmark tags (p13)
if cls == CHAPTER_BOOKMARK_CLASS:
continue
# Skip book title header (p85)
if cls == BOOK_HEADER_CLASS:
continue
# Check for dash lines — skip them
text = decode_text(p)
if is_dash_line(text):
continue
# Sanskrit verse pādas (p53) — accumulate into centered block
if cls in VERSE_PADA_CLASSES:
html = decode_html_content(p)
if html:
# Remove inner <i> tags before re-wrapping
stripped = re.sub(r"</?i>", "", html)
verse_accumulator.append(stripped)
continue
# Non-verse paragraph — flush any accumulated verse first
flush_verse()
# Chapter heading (p27) — include as paragraph
if cls == CHAPTER_HEADING_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Title page text (p3) — include
if cls == TITLE_PAGE_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Contents header (p69) — include in front-pages
if cls == CONTENTS_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Other books listing (p18) — include in front-pages
if cls == OTHER_BOOKS_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Bold quote paragraphs (p56, p30) — include as body
if cls in BOLD_QUOTE_CLASSES:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Verse translation / body text (p44)
if cls == TRANSLATION_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Misc text (p86) — include as body
if cls == MISC_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Body paragraphs (p6)
if cls in BODY_CLASSES:
html = decode_html_content(p)
if html:
# Detect Sanskrit verses disguised as body paragraphs:
# they start with <i> and contain <br> (multi-line italic = verse)
stripped_html = html.strip()
if stripped_html.startswith("<i>") and "<br" in stripped_html:
# This is a verse — wrap in centered italic div
inner = re.sub(r"</?i>", "", stripped_html)
result.append(
f'<div style="text-align:center"><i>{inner}</i></div>'
)
else:
result.append(html)
continue
# Any other class — try to extract content (be permissive)
html = decode_html_content(p)
if html and not is_dash_line(text):
result.append(html)
# Flush any trailing verse
flush_verse()
return result
# ---------------------------------------------------------------------------
# Main extraction logic
# ---------------------------------------------------------------------------
def extract_owk(soup: BeautifulSoup) -> dict:
"""Extract the entire On the Way to Kṛṣṇa (1973) from the parsed HTML.
Returns a dict with:
"intro_segments": list of intro segment dicts
"chapters": list of chapter dicts
"chapter_listing": list of {number, title} for intro.json
"""
all_paragraphs = list(soup.find_all("p"))
# Step 1: Find OWK boundaries
owk_start, owk_end = find_owk_boundaries(all_paragraphs)
print(f" OWK content: paragraphs {owk_start} to {owk_end}")
print(f" Total paragraphs in range: {owk_end - owk_start}")
# Step 2: Find all chapter bookmarks
chapter_bookmarks = find_chapter_bookmarks(all_paragraphs, owk_start, owk_end)
print(f" Found {len(chapter_bookmarks)} OWK chapter bookmarks")
for pos, ch_num, ch_title in chapter_bookmarks:
print(f" [chapter {ch_num}] paragraph {pos}: {ch_title}")
# Step 3: Extract front pages
# From book start (owk_start) to first chapter bookmark (OWK 1)
intro_segments = []
if chapter_bookmarks:
front_pages_end = chapter_bookmarks[0][0]
else:
front_pages_end = owk_end
front_pages_paragraphs = extract_paragraphs(
all_paragraphs, owk_start, front_pages_end, is_front_pages=True
)
intro_segments.append(
{
"slug": "front-pages",
"title": "Front Pages",
"type": "intro-segment",
"book": "owk",
"order": 1,
"paragraphs": front_pages_paragraphs,
}
)
# Step 4: Extract chapters
chapters = []
for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_bookmarks):
content_start = ch_pos + 1
# Content ends at next chapter bookmark or owk_end
if idx + 1 < len(chapter_bookmarks):
content_end = chapter_bookmarks[idx + 1][0]
else:
content_end = owk_end
paragraphs = extract_paragraphs(all_paragraphs, content_start, content_end)
ch = {
"book": "owk",
"chapter": ch_num,
"title": ch_title,
"type": "chapter",
"paragraphs": paragraphs,
}
chapters.append(ch)
# Step 5: Build chapter listing for intro.json
chapter_listing = [
{"number": ch["chapter"], "title": ch["title"]} for ch in chapters
]
return {
"intro_segments": intro_segments,
"chapters": chapters,
"chapter_listing": chapter_listing,
}
# ---------------------------------------------------------------------------
# Output
# ---------------------------------------------------------------------------
def write_json(data, filepath):
"""Write data as formatted JSON to filepath, creating directories as needed."""
os.makedirs(os.path.dirname(filepath), exist_ok=True)
with open(filepath, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False, indent=2)
def main():
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
html_path = os.path.join(base_dir, "public", "html", "other.html")
output_base = os.path.join(base_dir, "lib", "jsons", "books", "owk")
if not os.path.exists(html_path):
print(f"ERROR: {html_path} not found!")
sys.exit(1)
print(f"Reading {html_path}...")
with open(html_path, "r", encoding="utf-8") as f:
soup = BeautifulSoup(f, "html.parser")
print("Extracting On the Way to Kṛṣṇa (1973)...\n")
result = extract_owk(soup)
intro_segments = result["intro_segments"]
chapters = result["chapters"]
chapter_listing = result["chapter_listing"]
# --- Write intro.json ---
intro_meta = {
"book": "owk",
"title": "On the Way to Kṛṣṇa",
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
"year": 1973,
"description": "A guide to achieving happiness and perfection through Kṛṣṇa consciousness.",
"totalChapters": 5,
"chapters": chapter_listing,
}
intro_path = os.path.join(output_base, "intro.json")
write_json(intro_meta, intro_path)
print(f"\n Written: {intro_path}")
# --- Write intro segments ---
print(f"\n--- Intro Segments ---")
for seg in intro_segments:
filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
write_json(seg, filepath)
print(
f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}"
)
# --- Write chapters ---
print(f"\n--- Chapters ---")
for ch in chapters:
ch_num_str = str(ch["chapter"]).zfill(2)
filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
write_json(ch, filepath)
print(
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
f"{ch['title']}"
)
# --- Summary ---
total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments)
total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
total_paras = total_intro_paras + total_chapter_paras
print(f"\n{'=' * 60}")
print(f" EXTRACTION SUMMARY")
print(f"{'=' * 60}")
print(f" Intro segments: {len(intro_segments)}")
print(f" Chapters: {len(chapters)}")
print(f" Total paragraphs: {total_paras}")
print(f" Intro: {total_intro_paras}")
print(f" Chapters: {total_chapter_paras}")
if chapters:
print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
print(f" Output directory: {output_base}/")
# --- Samples ---
print(f"\n--- Sample: Front Pages ---")
if intro_segments:
front = intro_segments[0]
for i, p in enumerate(front["paragraphs"][:5]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
if chapters:
print(f"\n--- Sample: Chapter 1 ({chapters[0]['title']}) ---")
ch1 = chapters[0]
for i, p in enumerate(ch1["paragraphs"][:5]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
if chapters and len(chapters) > 1:
print(
f"\n--- Sample: Chapter {chapters[-1]['chapter']} ({chapters[-1]['title']}) ---"
)
last = chapters[-1]
for i, p in enumerate(last["paragraphs"][:5]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
# --- Quality Checks ---
print(f"\n--- Quality Checks ---")
# Check all 5 chapters present
expected_chapters = set(range(1, 6))
actual_chapters = set(ch["chapter"] for ch in chapters)
missing = expected_chapters - actual_chapters
extra = actual_chapters - expected_chapters
if missing:
print(f" ⚠ Missing chapters: {sorted(missing)}")
if extra:
print(f" ⚠ Extra chapters: {sorted(extra)}")
if not missing and not extra:
print(f" ✅ All 5 chapters present")
# Check all 1 intro segment present
expected_intros = {"front-pages"}
actual_intros = {s["slug"] for s in intro_segments}
missing_intros = expected_intros - actual_intros
extra_intros = actual_intros - expected_intros
if missing_intros:
print(f" ⚠ Missing intro segments: {missing_intros}")
if extra_intros:
print(f" ⚠ Extra intro segments: {extra_intros}")
if not missing_intros and not extra_intros:
print(f" ✅ All 1 intro segments present")
# Check empty chapters
empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
if empty_chapters:
print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
else:
print(f" ✅ No empty chapters")
# Check empty intro segments
empty_intros = [s["slug"] for s in intro_segments if len(s["paragraphs"]) == 0]
if empty_intros:
print(f" ⚠ Empty intro segments: {empty_intros}")
else:
print(f" ✅ No empty intro segments")
# Check ScaGoudy decoding
print(f"\n--- ScaGoudy Decoding Check ---")
all_text = ""
for ch in chapters:
all_text += " ".join(ch["paragraphs"][:20])
for seg in intro_segments:
all_text += " ".join(seg["paragraphs"][:10])
# Check for properly decoded words
if "Kṛṣṇa" in all_text:
print(f" ✅ 'Kṛṣṇa' found (ScaGoudy properly decoded)")
else:
print(f" ⚠ 'Kṛṣṇa' not found — ScaGoudy decoding may have failed")
if "Bhagavad-gītā" in all_text:
print(f" ✅ 'Bhagavad-gītā' found (ScaGoudy properly decoded)")
elif "gītā" in all_text or "Gītā" in all_text:
print(f" ✅ 'gītā' found (ScaGoudy properly decoded)")
else:
print(f" ⚠ 'Gītā' not found — ScaGoudy decoding may have failed")
if "Prabhupāda" in all_text:
print(f" ✅ 'Prabhupāda' found (ScaGoudy properly decoded)")
# Check for un-decoded ScaGoudy remnants (common encoded chars)
scagoudy_remnants = 0
for ch in chapters:
for p in ch["paragraphs"]:
if "Kåñëa" in p or "Prabhupäda" in p:
scagoudy_remnants += 1
for seg in intro_segments:
for p in seg["paragraphs"]:
if "Kåñëa" in p or "Prabhupäda" in p:
scagoudy_remnants += 1
if scagoudy_remnants == 0:
print(f" ✅ No ScaGoudy remnants detected")
else:
print(f" ⚠ {scagoudy_remnants} paragraphs with ScaGoudy remnants")
# Check for centered verse blocks
verse_block_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
if '<div style="text-align:center"><i>' in p:
verse_block_count += 1
if verse_block_count > 0:
print(f" ✅ {verse_block_count} centered Sanskrit verse blocks found")
else:
print(f" ⚠ No centered verse blocks found — verse grouping may have failed")
# Check for residual span tags
span_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
if 'class="' in p and "<span" in p:
span_count += 1
for seg in intro_segments:
for p in seg["paragraphs"]:
if 'class="' in p and "<span" in p:
span_count += 1
if span_count == 0:
print(f" ✅ No residual span class artifacts found")
else:
print(f" ⚠ {span_count} paragraphs still contain span class artifacts")
# Check for dash-line paragraphs that shouldn't be there
dash_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
plain = re.sub(r"<[^>]+>", "", p)
if is_dash_line(plain):
dash_count += 1
for seg in intro_segments:
for p in seg["paragraphs"]:
plain = re.sub(r"<[^>]+>", "", p)
if is_dash_line(plain):
dash_count += 1
if dash_count == 0:
print(f" ✅ No dashed-line paragraphs in output")
else:
print(f" ⚠ {dash_count} dashed-line paragraphs found in output")
# Show a verse block sample
print(f"\n--- Sample: Verse Block ---")
for ch in chapters:
for p in ch["paragraphs"]:
if '<div style="text-align:center"><i>' in p:
preview = re.sub(r"<[^>]+>", "", p)[:120]
print(f" {preview}...")
break
else:
continue
break
# Chapter title decoding check
print(f"\n--- Chapter Title Decoding Check ---")
for ch in chapters:
title = ch["title"]
if "å" in title or "ë" in title or "ñ" in title or "ä" in title:
print(f" ⚠ Chapter {ch['chapter']} title has un-decoded ScaGoudy: {title}")
else:
print(f" ✅ Ch {ch['chapter']}: {title}")
print(f"\nDone! ✨")
if __name__ == "__main__":
main()
scripts/poy_extract.py
+714
View File
@@ -0,0 +1,714 @@
#!/usr/bin/env python3
"""Extract The Perfection of Yoga (1972) from other.html.
The book has 1 intro segment (front-pages) and 11 chapters.
Chapters come from TWO bookmark prefixes:
- PoY 1–8: "The Perfection of Yoga" chapters
- TYS 4–6: "The Topmost Yoga System" bonus chapters (renumbered 9–11)
Output:
lib/jsons/books/poy/intro.json — Book metadata
lib/jsons/books/poy/intro/*.json — Intro segments
lib/jsons/books/poy/chapters/*.json — Chapters
Usage:
python3 scripts/poy_extract.py
"""
import json
import os
import re
import sys
from bs4 import BeautifulSoup, Tag
# ---------------------------------------------------------------------------
# ScaGoudy → IAST Unicode mapping
# ---------------------------------------------------------------------------
SCAGOUDY_TO_IAST = {
"ä": "ā",
"Ä": "Ā",
"å": "ṛ",
"Å": "Ṛ",
"ñ": "ṣ",
"Ñ": "Ṣ",
"ö": "ṭ",
"Ö": "Ṭ",
"ë": "ṇ",
"Ë": "Ṇ",
"ò": "ḍ",
"Ò": "Ḍ",
"ç": "ś",
"Ç": "Ś",
"ì": "ṅ",
"ï": "ñ",
"ü": "ū",
"Ü": "Ū",
"é": "ī",
"É": "Ī",
"à": "ṁ",
"ù": "ḥ",
"è": "ṝ",
"û": "ṁl",
"ÿ": "ḷ",
}
def decode_scagoudy(text: str) -> str:
"""Replace ScaGoudy-encoded characters with proper IAST Unicode."""
for src, dst in SCAGOUDY_TO_IAST.items():
text = text.replace(src, dst)
return text
def decode_html_content(tag: Tag) -> str:
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
inner = "".join(str(child) for child in tag.children)
inner = decode_scagoudy(inner)
# Remove span tags with CSS classes (RTF conversion artifacts)
inner = re.sub(
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
)
# Clean up Apple-tab-span leftovers (tab characters)
inner = inner.replace("\t", " ")
# Normalize whitespace (but preserve <br> tags)
inner = re.sub(r"\s+", " ", inner).strip()
return inner
def decode_text(tag: Tag) -> str:
"""Get plain text from a tag with ScaGoudy decoding."""
return decode_scagoudy(tag.get_text().strip())
def get_css_class(tag: Tag) -> str:
"""Get the first CSS class of a tag, or empty string."""
classes = tag.get("class", [])
return classes[0] if classes else ""
def is_empty(tag: Tag) -> bool:
"""Check if a tag is empty or whitespace-only."""
text = tag.get_text().strip()
return not text or text == "\xa0"
def is_dash_line(text: str) -> bool:
"""Return True if text is only dashes, spaces, and/or whitespace."""
cleaned = text.strip()
if not cleaned:
return False
return all(c in "-—–\u2014\u2013 \t" for c in cleaned)
# ---------------------------------------------------------------------------
# CSS class categorization for POY
# ---------------------------------------------------------------------------
# Book title header class (p85 for POY)
BOOK_HEADER_CLASS = "p85"
# Chapter bookmark class
CHAPTER_BOOKMARK_CLASS = "p13"
# Body paragraph classes (primary content)
BODY_CLASSES = frozenset(["p6"])
# Sanskrit verse pāda classes (italic, centered)
VERSE_PADA_CLASSES = frozenset(["p53"])
# Bold quote paragraphs — include as body
BOLD_QUOTE_CLASSES = frozenset(["p56"])
# Verse translations / body text
TRANSLATION_CLASS = "p44"
# Chapter heading (bold)
CHAPTER_HEADING_CLASS = "p27"
# Title page text
TITLE_PAGE_CLASS = "p3"
# Contents heading class
CONTENTS_CLASS = "p69"
# Misc classes to include as body
MISC_BODY_CLASSES = frozenset(["p90", "p91"])
# Classes to skip entirely (empty/decorative)
SKIP_CLASSES = frozenset(["p9", "p15", "p25", "p70"])
# ---------------------------------------------------------------------------
# Chapter title mapping (ScaGoudy → proper IAST)
# ---------------------------------------------------------------------------
CHAPTER_TITLES = {
1: "Yoga as Reestablishing Relations with Kṛṣṇa",
2: "Yoga as Work in Devotion",
3: "Yoga as Meditation on Kṛṣṇa",
4: "Yoga as Rejected by Arjuna",
5: "Yoga as Body and Mind Control",
6: "Yoga as Freedom from Duality and Designation",
7: "The Fate of the Unsuccessful Yogī",
8: "The Perfection of Yoga",
9: "The Hare Kṛṣṇa Mantra",
10: "How Bhakti-yoga Works",
11: "Sources of Absolute Knowledge",
}
# ---------------------------------------------------------------------------
# Extraction helpers
# ---------------------------------------------------------------------------
def find_poy_boundaries(all_paragraphs: list) -> tuple:
"""Find the start and end paragraph indices of the POY content.
Start: <p class="p85"> containing "Perfection of Yoga" and "1972"
End: next <p class="p85"> after start, or end of document
"""
start_idx = None
end_idx = len(all_paragraphs)
for i, p in enumerate(all_paragraphs):
cls = get_css_class(p)
if cls != BOOK_HEADER_CLASS:
continue
text = decode_text(p)
if "Perfection of Yoga" in text and "1972" in text:
start_idx = i
elif start_idx is not None:
# Next p85 after POY start = end of POY
end_idx = i
break
if start_idx is None:
print(
"ERROR: Could not find 'The Perfection of Yoga (1972)' header in other.html!"
)
sys.exit(1)
return start_idx, end_idx
def find_glossary_boundary(all_paragraphs: list, start_idx: int, end_idx: int) -> int:
"""Find the Glossary heading within POY range.
Returns the paragraph index of the Glossary heading, or end_idx if not found.
"""
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
if cls == CHAPTER_HEADING_CLASS:
text = p.get_text().strip()
if "Glossary" in text:
return i
return end_idx
def find_chapter_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
"""Find all POY + TYS chapter bookmark tags (p13) in the POY range.
Parses both "PoY N:" and "TYS N:" patterns.
TYS 4→ch9, TYS 5→ch10, TYS 6→ch11.
Returns list of (paragraph_index, sequential_chapter_number, chapter_title).
"""
bookmarks = []
seq_counter = 0
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
if cls != CHAPTER_BOOKMARK_CLASS:
continue
text = decode_text(p)
m = re.match(r"(?:PoY|TYS)\s+(\d+):\s+(.+)", text.strip())
if m:
seq_counter += 1
ch_title_raw = m.group(2).strip()
# Use our pre-defined proper IAST titles
ch_title = CHAPTER_TITLES.get(seq_counter, ch_title_raw)
bookmarks.append((i, seq_counter, ch_title))
return bookmarks
def extract_paragraphs(
all_paragraphs: list,
start_idx: int,
end_idx: int,
is_front_pages: bool = False,
) -> list:
"""Extract paragraphs for a section (chapter or intro segment).
Handles all CSS classes within the POY content.
Groups consecutive p53 verse pādas into centered italic divs.
"""
result = []
verse_accumulator = []
def flush_verse():
if verse_accumulator:
joined = "<br>".join(verse_accumulator)
result.append(f'<div style="text-align:center"><i>{joined}</i></div>')
verse_accumulator.clear()
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
# Skip empty paragraphs
if is_empty(p):
continue
# Skip noise classes
if cls in SKIP_CLASSES:
continue
# Skip bookmark tags (p13)
if cls == CHAPTER_BOOKMARK_CLASS:
continue
# Skip book title header (p85)
if cls == BOOK_HEADER_CLASS:
continue
# Check for dash lines — skip them
text = decode_text(p)
if is_dash_line(text):
continue
# Sanskrit verse pādas (p53) — accumulate into centered block
if cls in VERSE_PADA_CLASSES:
html = decode_html_content(p)
if html:
# Remove inner <i> tags before re-wrapping
stripped = re.sub(r"</?i>", "", html)
verse_accumulator.append(stripped)
continue
# Non-verse paragraph — flush any accumulated verse first
flush_verse()
# Chapter heading (p27) — include as paragraph
if cls == CHAPTER_HEADING_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Title page text (p3) — include
if cls == TITLE_PAGE_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Contents heading (p69) — include in front-pages
if cls == CONTENTS_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Bold quote paragraphs (p56) — include as body
if cls in BOLD_QUOTE_CLASSES:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Verse translation / body text (p44)
if cls == TRANSLATION_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Body paragraphs (p6)
if cls in BODY_CLASSES:
html = decode_html_content(p)
if html:
# Detect Sanskrit verses disguised as body paragraphs:
# they start with <i> and contain <br> (multi-line italic = verse)
stripped_html = html.strip()
if stripped_html.startswith("<i>") and "<br" in stripped_html:
# This is a verse — wrap in centered italic div
inner = re.sub(r"</?i>", "", stripped_html)
result.append(
f'<div style="text-align:center"><i>{inner}</i></div>'
)
else:
result.append(html)
continue
# Misc body classes (p90, p91)
if cls in MISC_BODY_CLASSES:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Any other class — try to extract content (be permissive)
html = decode_html_content(p)
if html and not is_dash_line(text):
result.append(html)
# Flush any trailing verse
flush_verse()
return result
# ---------------------------------------------------------------------------
# Main extraction logic
# ---------------------------------------------------------------------------
def extract_poy(soup: BeautifulSoup) -> dict:
"""Extract the entire Perfection of Yoga (1972) from the parsed HTML.
Returns a dict with:
"intro_segments": list of intro segment dicts
"chapters": list of chapter dicts
"chapter_listing": list of {number, title} for intro.json
"""
all_paragraphs = list(soup.find_all("p"))
# Step 1: Find POY boundaries
poy_start, poy_end = find_poy_boundaries(all_paragraphs)
print(f" POY content: paragraphs {poy_start} to {poy_end}")
print(f" Total paragraphs in range: {poy_end - poy_start}")
# Step 2: Find Glossary boundary — everything from Glossary onward is excluded
glossary_idx = find_glossary_boundary(all_paragraphs, poy_start, poy_end)
if glossary_idx < poy_end:
print(f" Glossary found at paragraph {glossary_idx} — trimming end boundary")
poy_end = glossary_idx
else:
print(f" No Glossary heading found — using full range")
print(f" Effective content range: paragraphs {poy_start} to {poy_end}")
# Step 3: Find all chapter bookmarks (PoY 1-8, TYS 4-6)
chapter_bookmarks = find_chapter_bookmarks(all_paragraphs, poy_start, poy_end)
print(f" Found {len(chapter_bookmarks)} chapter bookmarks")
for pos, ch_num, ch_title in chapter_bookmarks:
print(f" [chapter {ch_num}] paragraph {pos}: {ch_title}")
# Step 4: Extract front pages
# From book start (poy_start) to first chapter bookmark
intro_segments = []
if chapter_bookmarks:
front_pages_end = chapter_bookmarks[0][0]
else:
front_pages_end = poy_end
front_pages_paragraphs = extract_paragraphs(
all_paragraphs, poy_start, front_pages_end, is_front_pages=True
)
intro_segments.append(
{
"slug": "front-pages",
"title": "Front Pages",
"type": "intro-segment",
"book": "poy",
"order": 1,
"paragraphs": front_pages_paragraphs,
}
)
# Step 5: Extract chapters
chapters = []
for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_bookmarks):
content_start = ch_pos + 1
# Content ends at next chapter bookmark or poy_end
if idx + 1 < len(chapter_bookmarks):
content_end = chapter_bookmarks[idx + 1][0]
else:
content_end = poy_end
paragraphs = extract_paragraphs(all_paragraphs, content_start, content_end)
ch = {
"book": "poy",
"chapter": ch_num,
"title": ch_title,
"type": "chapter",
"paragraphs": paragraphs,
}
chapters.append(ch)
# Step 6: Build chapter listing for intro.json
chapter_listing = [
{"number": ch["chapter"], "title": ch["title"]} for ch in chapters
]
return {
"intro_segments": intro_segments,
"chapters": chapters,
"chapter_listing": chapter_listing,
}
# ---------------------------------------------------------------------------
# Output
# ---------------------------------------------------------------------------
def write_json(data, filepath):
"""Write data as formatted JSON to filepath, creating directories as needed."""
os.makedirs(os.path.dirname(filepath), exist_ok=True)
with open(filepath, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False, indent=2)
def main():
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
html_path = os.path.join(base_dir, "public", "html", "other.html")
output_base = os.path.join(base_dir, "lib", "jsons", "books", "poy")
if not os.path.exists(html_path):
print(f"ERROR: {html_path} not found!")
sys.exit(1)
print(f"Reading {html_path}...")
with open(html_path, "r", encoding="utf-8") as f:
soup = BeautifulSoup(f, "html.parser")
print("Extracting The Perfection of Yoga (1972)...\n")
result = extract_poy(soup)
intro_segments = result["intro_segments"]
chapters = result["chapters"]
chapter_listing = result["chapter_listing"]
# --- Write intro.json ---
intro_meta = {
"book": "poy",
"title": "The Perfection of Yoga",
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
"year": 1972,
"description": "The complete science of yoga as taught by Lord Kṛṣṇa in the Bhagavad-gītā.",
"totalChapters": 11,
"chapters": chapter_listing,
}
intro_path = os.path.join(output_base, "intro.json")
write_json(intro_meta, intro_path)
print(f"\n Written: {intro_path}")
# --- Write intro segments ---
print(f"\n--- Intro Segments ---")
for seg in intro_segments:
filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
write_json(seg, filepath)
print(
f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}"
)
# --- Write chapters ---
print(f"\n--- Chapters ---")
for ch in chapters:
ch_num_str = str(ch["chapter"]).zfill(2)
filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
write_json(ch, filepath)
print(
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
f"{ch['title']}"
)
# --- Summary ---
total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments)
total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
total_paras = total_intro_paras + total_chapter_paras
print(f"\n{'=' * 60}")
print(f" EXTRACTION SUMMARY")
print(f"{'=' * 60}")
print(f" Intro segments: {len(intro_segments)}")
print(f" Chapters: {len(chapters)}")
print(f" Total paragraphs: {total_paras}")
print(f" Intro: {total_intro_paras}")
print(f" Chapters: {total_chapter_paras}")
if chapters:
print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
print(f" Output directory: {output_base}/")
# --- Samples ---
print(f"\n--- Sample: Front Pages ---")
if intro_segments:
front = intro_segments[0]
for i, p in enumerate(front["paragraphs"][:5]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
if chapters:
print(f"\n--- Sample: Chapter 1 ({chapters[0]['title']}) ---")
ch1 = chapters[0]
for i, p in enumerate(ch1["paragraphs"][:5]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
if chapters and len(chapters) > 1:
print(
f"\n--- Sample: Chapter {chapters[-1]['chapter']} ({chapters[-1]['title']}) ---"
)
last = chapters[-1]
for i, p in enumerate(last["paragraphs"][:5]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
# --- Quality Checks ---
print(f"\n--- Quality Checks ---")
# Check all 11 chapters present
expected_chapters = set(range(1, 12))
actual_chapters = set(ch["chapter"] for ch in chapters)
missing = expected_chapters - actual_chapters
extra = actual_chapters - expected_chapters
if missing:
print(f" ⚠ Missing chapters: {sorted(missing)}")
if extra:
print(f" ⚠ Extra chapters: {sorted(extra)}")
if not missing and not extra:
print(f" ✅ All 11 chapters present")
# Check all 1 intro segment present
expected_intros = {"front-pages"}
actual_intros = {s["slug"] for s in intro_segments}
missing_intros = expected_intros - actual_intros
extra_intros = actual_intros - expected_intros
if missing_intros:
print(f" ⚠ Missing intro segments: {missing_intros}")
if extra_intros:
print(f" ⚠ Extra intro segments: {extra_intros}")
if not missing_intros and not extra_intros:
print(f" ✅ All 1 intro segments present")
# Check empty chapters
empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
if empty_chapters:
print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
else:
print(f" ✅ No empty chapters")
# Check empty intro segments
empty_intros = [s["slug"] for s in intro_segments if len(s["paragraphs"]) == 0]
if empty_intros:
print(f" ⚠ Empty intro segments: {empty_intros}")
else:
print(f" ✅ No empty intro segments")
# Check ScaGoudy decoding
print(f"\n--- ScaGoudy Decoding Check ---")
all_text = ""
for ch in chapters:
all_text += " ".join(ch["paragraphs"][:20])
for seg in intro_segments:
all_text += " ".join(seg["paragraphs"][:10])
# Check for properly decoded words
if "Kṛṣṇa" in all_text:
print(f" ✅ 'Kṛṣṇa' found (ScaGoudy properly decoded)")
else:
print(f" ⚠ 'Kṛṣṇa' not found — ScaGoudy decoding may have failed")
if "Bhagavad-gītā" in all_text:
print(f" ✅ 'Bhagavad-gītā' found (ScaGoudy properly decoded)")
elif "gītā" in all_text or "Gītā" in all_text:
print(f" ✅ 'gītā' found (ScaGoudy properly decoded)")
else:
print(f" ⚠ 'Gītā' not found — ScaGoudy decoding may have failed")
if "Prabhupāda" in all_text:
print(f" ✅ 'Prabhupāda' found (ScaGoudy properly decoded)")
# Check for un-decoded ScaGoudy remnants (common encoded chars)
scagoudy_remnants = 0
for ch in chapters:
for p in ch["paragraphs"]:
if "Kåñëa" in p or "Prabhupäda" in p:
scagoudy_remnants += 1
for seg in intro_segments:
for p in seg["paragraphs"]:
if "Kåñëa" in p or "Prabhupäda" in p:
scagoudy_remnants += 1
if scagoudy_remnants == 0:
print(f" ✅ No ScaGoudy remnants detected")
else:
print(f" ⚠ {scagoudy_remnants} paragraphs with ScaGoudy remnants")
# Check for centered verse blocks
verse_block_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
if '<div style="text-align:center"><i>' in p:
verse_block_count += 1
if verse_block_count > 0:
print(f" ✅ {verse_block_count} centered Sanskrit verse blocks found")
else:
print(f" ⚠ No centered verse blocks found — verse grouping may have failed")
# Check for residual span tags
span_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
if 'class="' in p and "<span" in p:
span_count += 1
for seg in intro_segments:
for p in seg["paragraphs"]:
if 'class="' in p and "<span" in p:
span_count += 1
if span_count == 0:
print(f" ✅ No residual span class artifacts found")
else:
print(f" ⚠ {span_count} paragraphs still contain span class artifacts")
# Check for dash-line paragraphs that shouldn't be there
dash_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
plain = re.sub(r"<[^>]+>", "", p)
if is_dash_line(plain):
dash_count += 1
for seg in intro_segments:
for p in seg["paragraphs"]:
plain = re.sub(r"<[^>]+>", "", p)
if is_dash_line(plain):
dash_count += 1
if dash_count == 0:
print(f" ✅ No dashed-line paragraphs in output")
else:
print(f" ⚠ {dash_count} dashed-line paragraphs found in output")
# Show a verse block sample
print(f"\n--- Sample: Verse Block ---")
for ch in chapters:
for p in ch["paragraphs"]:
if '<div style="text-align:center"><i>' in p:
preview = re.sub(r"<[^>]+>", "", p)[:120]
print(f" {preview}...")
break
else:
continue
break
# Chapter title decoding check
print(f"\n--- Chapter Title Decoding Check ---")
for ch in chapters:
title = ch["title"]
if "å" in title or "ë" in title or "ñ" in title or "ä" in title:
print(f" ⚠ Chapter {ch['chapter']} title has un-decoded ScaGoudy: {title}")
else:
print(f" ✅ Ch {ch['chapter']}: {title}")
print(f"\nDone! ✨")
if __name__ == "__main__":
main()
scripts/pqpa_extract.py
+848
View File
@@ -0,0 +1,848 @@
#!/usr/bin/env python3
"""Extract Perfect Questions Perfect Answers (1977) from other.html.
The book has 3 intro segments (front-pages, introduction, about-the-author),
9 chapters, and a Concluding Words section.
Output:
lib/jsons/books/pqpa/intro.json — Book metadata
lib/jsons/books/pqpa/intro/*.json — Intro segments
lib/jsons/books/pqpa/chapters/*.json — Chapters + concluding words
Usage:
python3 scripts/pqpa_extract.py
"""
import json
import os
import re
import sys
from bs4 import BeautifulSoup, Tag
# ---------------------------------------------------------------------------
# ScaGoudy → IAST Unicode mapping
# ---------------------------------------------------------------------------
SCAGOUDY_TO_IAST = {
"ä": "ā",
"Ä": "Ā",
"å": "ṛ",
"Å": "Ṛ",
"ñ": "ṣ",
"Ñ": "Ṣ",
"ö": "ṭ",
"Ö": "Ṭ",
"ë": "ṇ",
"Ë": "Ṇ",
"ò": "ḍ",
"Ò": "Ḍ",
"ç": "ś",
"Ç": "Ś",
"ì": "ṅ",
"ï": "ñ",
"ü": "ū",
"Ü": "Ū",
"é": "ī",
"É": "Ī",
"à": "ṁ",
"ù": "ḥ",
"è": "ṝ",
"û": "ṁl",
"ÿ": "ḷ",
}
def decode_scagoudy(text: str) -> str:
"""Replace ScaGoudy-encoded characters with proper IAST Unicode."""
for src, dst in SCAGOUDY_TO_IAST.items():
text = text.replace(src, dst)
return text
def decode_html_content(tag: Tag) -> str:
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
inner = "".join(str(child) for child in tag.children)
inner = decode_scagoudy(inner)
# Remove span tags with CSS classes (RTF conversion artifacts)
inner = re.sub(
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
)
# Clean up Apple-tab-span leftovers (tab characters)
inner = inner.replace("\t", " ")
# Normalize whitespace
inner = re.sub(r"\s+", " ", inner).strip()
return inner
def decode_text(tag: Tag) -> str:
"""Get plain text from a tag with ScaGoudy decoding."""
return decode_scagoudy(tag.get_text().strip())
def get_css_class(tag: Tag) -> str:
"""Get the first CSS class of a tag, or empty string."""
classes = tag.get("class", [])
return classes[0] if classes else ""
def is_empty(tag: Tag) -> bool:
"""Check if a tag is empty or whitespace-only."""
text = tag.get_text().strip()
return not text or text == "\xa0"
def is_dash_line(text: str) -> bool:
"""Return True if text is only dashes, spaces, and/or whitespace."""
cleaned = text.strip()
if not cleaned:
return False
return all(c in "-—–\u2014\u2013 \t" for c in cleaned)
# ---------------------------------------------------------------------------
# CSS class categorization for PQPA
# ---------------------------------------------------------------------------
# Book title header class
BOOK_HEADER_CLASS = "p12"
# Bookmark class — used for section boundary detection
BOOKMARK_CLASS = "p13"
# Body paragraph classes (primary content)
BODY_CLASSES = frozenset(["p6", "p48"])
# Dialogue line class — the main content class (714 occurrences)
DIALOGUE_CLASS = "p92"
# Sanskrit verse pāda classes (italic, centered)
VERSE_PADA_CLASSES = frozenset(["p53", "p66"])
# Verse translations / body text
TRANSLATION_CLASS = "p44"
# Chapter heading (bold)
CHAPTER_HEADING_CLASS = "p27"
# Section heading (bold)
SECTION_HEADING_CLASS = "p31"
# Front pages text
FRONT_PAGES_CLASS = "p18"
# Attribution text
ATTRIBUTION_CLASS = "p42"
# Numbered items / verse quotes
NUMBERED_CLASS = "p24"
# Mantra text
MANTRA_CLASS = "p19"
# Contents heading
CONTENTS_CLASS = "p69"
# Title page text
TITLE_PAGE_CLASS = "p3"
# Misc classes to include
MISC_CLASSES = frozenset(["p28", "p76", "p93", "p94", "p95", "p96"])
# Classes to skip entirely (empty/decorative)
SKIP_CLASSES = frozenset(["p9", "p15", "p70"])
# ---------------------------------------------------------------------------
# Extraction helpers
# ---------------------------------------------------------------------------
def find_pqpa_boundaries(all_paragraphs: list) -> tuple:
"""Find the start and end paragraph indices of the PQPA content.
Start: <p class="p12"> containing "Perfect Questions Perfect Answers"
End: next <p class="p12"> after start, or end of file
"""
start_idx = None
end_idx = len(all_paragraphs)
for i, p in enumerate(all_paragraphs):
cls = get_css_class(p)
if cls != BOOK_HEADER_CLASS:
continue
text = decode_text(p)
if "Perfect Questions Perfect Answers" in text:
start_idx = i
elif start_idx is not None:
# Next p12 after PQPA start = end of PQPA
end_idx = i
break
if start_idx is None:
print(
"ERROR: Could not find 'Perfect Questions Perfect Answers' header in other.html!"
)
sys.exit(1)
# Also check for Glossary marker as end boundary
for i in range(start_idx + 1, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
if cls == TITLE_PAGE_CLASS:
text = p.get_text().strip()
if text == "Glossary":
end_idx = i
break
return start_idx, end_idx
def find_all_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
"""Find all p13 bookmark tags in the PQPA range.
Returns list of (paragraph_index, bookmark_text).
The bookmark text is stripped of any span noise like <span class="s27"> </span>.
"""
bookmarks = []
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
if cls == BOOKMARK_CLASS:
# Get text, stripping span noise
text = decode_text(p).strip()
if text.startswith("PQPA"):
bookmarks.append((i, text))
return bookmarks
def parse_chapter_bookmark(text: str):
"""Parse a chapter bookmark like 'PQPA 1: Kṛṣṇa, the All-Attractive'.
Returns (chapter_number, title) or None if not a chapter bookmark.
"""
m = re.match(r"PQPA\s+(\d+):\s+(.+)", text.strip())
if m:
return int(m.group(1)), m.group(2).strip()
return None
def parse_intro_bookmark(text: str) -> bool:
"""Check if this is the Introduction bookmark."""
return text.strip() == "PQPA Introduction"
def parse_concluding_bookmark(text: str) -> bool:
"""Check if this is the Concluding Words bookmark."""
return text.strip() == "PQPA Concluding Words"
def extract_paragraphs(
all_paragraphs: list,
start_idx: int,
end_idx: int,
is_front_pages: bool = False,
) -> list:
"""Extract paragraphs for a section (chapter, intro segment, or conclusion).
Handles all CSS classes within the PQPA content.
Groups consecutive p53/p66 verse pādas into centered italic divs.
Detects verse patterns in p6/p92 (starts with <i>, has <br>).
"""
result = []
verse_accumulator = []
def flush_verse():
if verse_accumulator:
joined = "<br>".join(verse_accumulator)
result.append(f'<div style="text-align:center"><i>{joined}</i></div>')
verse_accumulator.clear()
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
# Skip empty paragraphs
if is_empty(p):
continue
# Skip noise classes
if cls in SKIP_CLASSES:
continue
# Skip bookmark tags (p13)
if cls == BOOKMARK_CLASS:
continue
# Skip book title header (p12)
if cls == BOOK_HEADER_CLASS:
continue
# Check for dash lines — skip them
text = decode_text(p)
if is_dash_line(text):
continue
# Sanskrit verse pādas (p53, p66) — accumulate into centered block
if cls in VERSE_PADA_CLASSES:
html = decode_html_content(p)
if html:
# Remove inner <i> tags before re-wrapping
stripped = re.sub(r"</?i>", "", html)
verse_accumulator.append(stripped)
continue
# Non-verse paragraph — flush any accumulated verse first
flush_verse()
# Dialogue lines (p92) — the main content class
if cls == DIALOGUE_CLASS:
html = decode_html_content(p)
if html:
# Detect verses disguised as dialogue: starts with <i> and has <br>
stripped_html = html.strip()
if stripped_html.startswith("<i>") and "<br" in stripped_html:
inner = re.sub(r"</?i>", "", stripped_html)
result.append(
f'<div style="text-align:center"><i>{inner}</i></div>'
)
else:
result.append(html)
continue
# Chapter heading (p27) — include as paragraph
if cls == CHAPTER_HEADING_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Section heading (p31) — include as paragraph
if cls == SECTION_HEADING_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Front pages text (p18) — include
if cls == FRONT_PAGES_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Contents heading (p69) — include
if cls == CONTENTS_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Attribution text (p42)
if cls == ATTRIBUTION_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Numbered items / verse quotes (p24)
if cls == NUMBERED_CLASS:
html = decode_html_content(p)
if html:
# Detect verses: starts with <i> and has <br>
stripped_html = html.strip()
if stripped_html.startswith("<i>") and "<br" in stripped_html:
inner = re.sub(r"</?i>", "", stripped_html)
result.append(
f'<div style="text-align:center"><i>{inner}</i></div>'
)
else:
result.append(html)
continue
# Mantra text (p19) — wrap in centered div
if cls == MANTRA_CLASS:
html = decode_html_content(p)
if html:
# Check if it's a multi-line verse
if "<br" in html:
inner = re.sub(r"</?i>", "", html)
result.append(
f'<div style="text-align:center"><i>{inner}</i></div>'
)
else:
result.append(html)
continue
# Verse translation / body text (p44)
if cls == TRANSLATION_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Title page text (p3) — include
if cls == TITLE_PAGE_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Body paragraphs (p6, p48)
if cls in BODY_CLASSES:
html = decode_html_content(p)
if html:
# Detect Sanskrit verses disguised as body paragraphs:
# they start with <i> and contain <br> (multi-line italic = verse)
stripped_html = html.strip()
if stripped_html.startswith("<i>") and "<br" in stripped_html:
inner = re.sub(r"</?i>", "", stripped_html)
result.append(
f'<div style="text-align:center"><i>{inner}</i></div>'
)
else:
result.append(html)
continue
# Misc classes — include as-is
if cls in MISC_CLASSES:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Any other class — try to extract content (be permissive)
html = decode_html_content(p)
if html and not is_dash_line(text):
result.append(html)
# Flush any trailing verse
flush_verse()
return result
# ---------------------------------------------------------------------------
# Main extraction logic
# ---------------------------------------------------------------------------
def extract_pqpa(soup: BeautifulSoup) -> dict:
"""Extract the entire Perfect Questions Perfect Answers from the parsed HTML.
Returns a dict with:
"intro_segments": list of intro segment dicts
"chapters": list of chapter dicts
"concluding_words": concluding words dict
"chapter_listing": list of {number, title} for intro.json
"""
all_paragraphs = list(soup.find_all("p"))
# Step 1: Find PQPA boundaries
pqpa_start, pqpa_end = find_pqpa_boundaries(all_paragraphs)
print(f" PQPA content: paragraphs {pqpa_start} to {pqpa_end}")
print(f" Total paragraphs in range: {pqpa_end - pqpa_start}")
# Step 2: Find all bookmarks
bookmarks = find_all_bookmarks(all_paragraphs, pqpa_start, pqpa_end)
print(f" Found {len(bookmarks)} PQPA bookmarks")
for pos, text in bookmarks:
print(f" Line ~{pos}: {text}")
# Step 3: Categorize bookmarks
intro_bookmark = None # (para_idx,)
chapter_bookmarks = [] # (para_idx, chapter_num, title)
concluding_bookmark = None # (para_idx,)
for pos, text in bookmarks:
ch = parse_chapter_bookmark(text)
if ch:
chapter_bookmarks.append((pos, ch[0], ch[1]))
elif parse_intro_bookmark(text):
intro_bookmark = pos
elif parse_concluding_bookmark(text):
concluding_bookmark = pos
print(
f" Introduction bookmark: {'found' if intro_bookmark is not None else 'NOT FOUND'}"
)
print(f" Chapter bookmarks: {len(chapter_bookmarks)}")
print(
f" Concluding Words bookmark: {'found' if concluding_bookmark is not None else 'NOT FOUND'}"
)
# Collect all section starts for boundary computation
all_section_starts = sorted(
([intro_bookmark] if intro_bookmark is not None else [])
+ [pos for pos, _, _ in chapter_bookmarks]
+ ([concluding_bookmark] if concluding_bookmark is not None else [])
)
# Step 4: Extract front-pages (from p12 header to Introduction bookmark)
front_pages_start = pqpa_start + 1 # after p12 header
front_pages_end = (
intro_bookmark
if intro_bookmark is not None
else (chapter_bookmarks[0][0] if chapter_bookmarks else pqpa_end)
)
front_pages_paras = extract_paragraphs(
all_paragraphs, front_pages_start, front_pages_end, is_front_pages=True
)
front_pages = {
"slug": "front-pages",
"title": "Front Pages",
"type": "intro-segment",
"book": "pqpa",
"order": 1,
"paragraphs": front_pages_paras,
}
# Step 5: Extract Introduction (from PQPA Introduction to PQPA 1)
introduction = None
if intro_bookmark is not None:
intro_content_start = intro_bookmark + 1
# End at the first chapter bookmark
intro_content_end = chapter_bookmarks[0][0] if chapter_bookmarks else pqpa_end
intro_paras = extract_paragraphs(
all_paragraphs, intro_content_start, intro_content_end
)
introduction = {
"slug": "introduction",
"title": "Introduction",
"type": "intro-segment",
"book": "pqpa",
"order": 2,
"paragraphs": intro_paras,
}
# Step 6: Extract chapters 1-9
chapters = []
for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_bookmarks):
content_start = ch_pos + 1
# Content ends at next chapter bookmark, or concluding words, or pqpa_end
if idx + 1 < len(chapter_bookmarks):
content_end = chapter_bookmarks[idx + 1][0]
elif concluding_bookmark is not None:
content_end = concluding_bookmark
else:
content_end = pqpa_end
paragraphs = extract_paragraphs(all_paragraphs, content_start, content_end)
ch = {
"book": "pqpa",
"chapter": ch_num,
"title": ch_title,
"type": "chapter",
"paragraphs": paragraphs,
}
chapters.append(ch)
# Step 7: Extract Concluding Words + About the Author
concluding_words = None
about_the_author = None
if concluding_bookmark is not None:
# Find the "About the Author" boundary:
# <p class="p27"><b>His Divine Grace A. C. Bhaktivedanta Swami Prabhupāda</b></p>
about_author_pos = None
for i in range(concluding_bookmark + 1, pqpa_end):
p = all_paragraphs[i]
cls = get_css_class(p)
if cls == CHAPTER_HEADING_CLASS:
text = decode_text(p)
if "His Divine Grace" in text:
about_author_pos = i
break
# Concluding Words: from bookmark to About the Author (or pqpa_end)
cw_content_start = concluding_bookmark + 1
cw_content_end = about_author_pos if about_author_pos is not None else pqpa_end
cw_paras = extract_paragraphs(all_paragraphs, cw_content_start, cw_content_end)
concluding_words = {
"book": "pqpa",
"chapter": None,
"title": "Concluding Words",
"type": "conclusion",
"slug": "concluding-words",
"paragraphs": cw_paras,
}
# About the Author: from p27 heading to pqpa_end
if about_author_pos is not None:
ata_paras = extract_paragraphs(all_paragraphs, about_author_pos, pqpa_end)
about_the_author = {
"slug": "about-the-author",
"title": "About the Author",
"type": "intro-segment",
"book": "pqpa",
"order": 99,
"paragraphs": ata_paras,
}
# Step 8: Build intro segments list
intro_segments = [front_pages]
if introduction:
intro_segments.append(introduction)
if about_the_author:
intro_segments.append(about_the_author)
# Step 9: Build chapter listing for intro.json
chapter_listing = [
{"number": ch["chapter"], "title": ch["title"]} for ch in chapters
]
return {
"intro_segments": intro_segments,
"chapters": chapters,
"concluding_words": concluding_words,
"chapter_listing": chapter_listing,
}
# ---------------------------------------------------------------------------
# Output
# ---------------------------------------------------------------------------
def write_json(data, filepath):
"""Write data as formatted JSON to filepath, creating directories as needed."""
os.makedirs(os.path.dirname(filepath), exist_ok=True)
with open(filepath, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False, indent=2)
def main():
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
html_path = os.path.join(base_dir, "public", "html", "other.html")
output_base = os.path.join(base_dir, "lib", "jsons", "books", "pqpa")
if not os.path.exists(html_path):
print(f"ERROR: {html_path} not found!")
sys.exit(1)
print(f"Reading {html_path}...")
with open(html_path, "r", encoding="utf-8") as f:
soup = BeautifulSoup(f, "html.parser")
print("Extracting Perfect Questions Perfect Answers (1977)...\n")
result = extract_pqpa(soup)
intro_segments = result["intro_segments"]
chapters = result["chapters"]
concluding_words = result["concluding_words"]
chapter_listing = result["chapter_listing"]
# --- Write intro.json ---
intro_meta = {
"book": "pqpa",
"title": "Perfect Questions Perfect Answers",
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
"year": 1977,
"description": "Conversations between Śrīla Prabhupāda and Bob Cohen, a Peace Corps worker in India.",
"totalChapters": 9,
"chapters": chapter_listing,
}
intro_path = os.path.join(output_base, "intro.json")
write_json(intro_meta, intro_path)
print(f"\n Written: {intro_path}")
# --- Write intro segments ---
print(f"\n--- Intro Segments ---")
for seg in intro_segments:
filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
write_json(seg, filepath)
print(
f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}"
)
# --- Write chapters ---
print(f"\n--- Chapters ---")
for ch in chapters:
ch_num_str = str(ch["chapter"]).zfill(2)
filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
write_json(ch, filepath)
print(
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
f"{ch['title']}"
)
# --- Write concluding words ---
if concluding_words:
filepath = os.path.join(output_base, "chapters", "concluding-words.json")
write_json(concluding_words, filepath)
print(
f"\n Concluding Words: {len(concluding_words['paragraphs'])} paragraphs → {filepath}"
)
# --- Summary ---
total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments)
total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
total_concluding_paras = (
len(concluding_words["paragraphs"]) if concluding_words else 0
)
total_paras = total_intro_paras + total_chapter_paras + total_concluding_paras
print(f"\n{'=' * 60}")
print(f" EXTRACTION SUMMARY")
print(f"{'=' * 60}")
print(f" Intro segments: {len(intro_segments)}")
for seg in intro_segments:
print(
f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs"
)
print(f" Chapters: {len(chapters)}")
print(f" Concluding Words: {'Yes' if concluding_words else 'No'}")
print(f" Total paragraphs: {total_paras}")
print(f" Intro: {total_intro_paras}")
print(f" Chapters: {total_chapter_paras}")
print(f" Concluding: {total_concluding_paras}")
if chapters:
print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
print(f" Output directory: {output_base}/")
# --- Samples ---
print(f"\n--- Sample: Chapter 1 ---")
if chapters:
ch1 = chapters[0]
for i, p in enumerate(ch1["paragraphs"][:4]):
preview = re.sub(r"<[^>]+>", "", p)[:120]
print(f" [{i}] {preview}...")
if len(chapters) >= 9:
print(f"\n--- Sample: Chapter 9 (last) ---")
last = chapters[-1]
for i, p in enumerate(last["paragraphs"][:3]):
preview = re.sub(r"<[^>]+>", "", p)[:120]
print(f" [{i}] {preview}...")
if concluding_words:
print(f"\n--- Sample: Concluding Words ---")
for i, p in enumerate(concluding_words["paragraphs"][:3]):
preview = re.sub(r"<[^>]+>", "", p)[:120]
print(f" [{i}] {preview}...")
# --- Quality Checks ---
print(f"\n--- Quality Checks ---")
# Check all 9 chapters present
expected_chapters = set(range(1, 10))
actual_chapters = set(ch["chapter"] for ch in chapters)
missing = expected_chapters - actual_chapters
extra = actual_chapters - expected_chapters
if missing:
print(f" ⚠ Missing chapters: {sorted(missing)}")
if extra:
print(f" ⚠ Extra chapters: {sorted(extra)}")
if not missing and not extra:
print(f" ✅ All 9 chapters present")
# Check intro segments
expected_intros = {"front-pages", "introduction", "about-the-author"}
actual_intros = {s["slug"] for s in intro_segments}
missing_intros = expected_intros - actual_intros
if missing_intros:
print(f" ⚠ Missing intro segments: {missing_intros}")
else:
print(f" ✅ All 3 intro segments present")
# Check concluding words
if concluding_words:
print(f" ✅ Concluding Words present")
else:
print(f" ⚠ Concluding Words NOT found")
# Check empty chapters
empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
if empty_chapters:
print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
else:
print(f" ✅ No empty chapters")
# Check ScaGoudy decoding — look for common encoded characters
encoded_chars = set("äåöëòçüéàùè")
suspect_count = 0
all_sections = chapters + intro_segments
if concluding_words:
all_sections = all_sections + [concluding_words]
for section in all_sections:
for p in section["paragraphs"]:
plain = re.sub(r"<[^>]+>", "", p)
for c in encoded_chars:
if c in plain:
suspect_count += 1
break
if suspect_count == 0:
print(f" ✅ No suspected undecoded ScaGoudy characters found")
else:
print(
f" ℹ {suspect_count} paragraphs contain characters that may be undecoded ScaGoudy"
)
# Check verse blocks
verse_block_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
if '<div style="text-align:center"><i>' in p:
verse_block_count += 1
print(f" ℹ {verse_block_count} verse blocks in chapters")
# ScaGoudy decoding spot check
print(f"\n--- ScaGoudy Decoding Spot Check ---")
if chapters:
ch1_text = " ".join(chapters[0]["paragraphs"][:10])
if "Kṛṣṇa" in ch1_text:
print(f" ✅ 'Kṛṣṇa' correctly decoded in Chapter 1")
elif "Kåñëa" in ch1_text:
print(f" ⚠ 'Kåñëa' still encoded — ScaGoudy decoding may have failed!")
else:
print(
f" ℹ 'Kṛṣṇa' not found in Chapter 1 first 10 paragraphs (may be normal)"
)
# Check for "Śrīla Prabhupāda"
if "Prabhupāda" in ch1_text:
print(f" ✅ 'Prabhupāda' correctly decoded")
elif "Prabhupäda" in ch1_text:
print(f" ⚠ ScaGoudy decoding incomplete for 'Prabhupäda'")
# Dialogue line check — look for BOB: and ŚRĪLA PRABHUPĀDA: speakers
print(f"\n--- Dialogue Speaker Check ---")
speaker_counts = {}
for ch in chapters:
for p in ch["paragraphs"]:
# Look for bold speaker patterns
m = re.match(r"<b>([^<]+):</b>", p)
if m:
speaker = m.group(1).strip()
speaker_counts[speaker] = speaker_counts.get(speaker, 0) + 1
if speaker_counts:
print(f" Speakers found:")
for speaker, count in sorted(speaker_counts.items(), key=lambda x: -x[1]):
print(f" {speaker}: {count} lines")
else:
print(f" ⚠ No dialogue speakers detected")
# Sample verse blocks
print(f"\n--- Sample Verse Blocks ---")
shown = 0
for ch in chapters:
for p in ch["paragraphs"]:
if '<div style="text-align:center"><i>' in p:
preview = re.sub(r"<[^>]+>", "", p)[:120]
print(f" {preview}...")
shown += 1
if shown >= 3:
break
if shown >= 3:
break
print(f"\nDone! ✨")
if __name__ == "__main__":
main()
scripts/rv_extract.py
+647
View File
@@ -0,0 +1,647 @@
#!/usr/bin/env python3
"""Extract Rāja-vidyā: The King of Knowledge (1973) from other.html.
The book has 1 intro segment (front-pages) and 8 chapters.
Output:
lib/jsons/books/rv/intro.json — Book metadata
lib/jsons/books/rv/intro/*.json — Intro segments
lib/jsons/books/rv/chapters/*.json — Chapters
Usage:
python3 scripts/rv_extract.py
"""
import json
import os
import re
import sys
from bs4 import BeautifulSoup, Tag
# ---------------------------------------------------------------------------
# ScaGoudy → IAST Unicode mapping
# ---------------------------------------------------------------------------
SCAGOUDY_TO_IAST = {
"ä": "ā",
"Ä": "Ā",
"å": "ṛ",
"Å": "Ṛ",
"ñ": "ṣ",
"Ñ": "Ṣ",
"ö": "ṭ",
"Ö": "Ṭ",
"ë": "ṇ",
"Ë": "Ṇ",
"ò": "ḍ",
"Ò": "Ḍ",
"ç": "ś",
"Ç": "Ś",
"ì": "ṅ",
"ï": "ñ",
"ü": "ū",
"Ü": "Ū",
"é": "ī",
"É": "Ī",
"à": "ṁ",
"ù": "ḥ",
"è": "ṝ",
"û": "ṁl",
"ÿ": "ḷ",
}
def decode_scagoudy(text: str) -> str:
"""Replace ScaGoudy-encoded characters with proper IAST Unicode."""
for src, dst in SCAGOUDY_TO_IAST.items():
text = text.replace(src, dst)
return text
def decode_html_content(tag: Tag) -> str:
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
inner = "".join(str(child) for child in tag.children)
inner = decode_scagoudy(inner)
# Remove span tags with CSS classes (RTF conversion artifacts)
inner = re.sub(
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
)
# Clean up Apple-tab-span leftovers (tab characters)
inner = inner.replace("\t", " ")
# Normalize whitespace (but preserve <br> tags)
inner = re.sub(r"\s+", " ", inner).strip()
return inner
def decode_text(tag: Tag) -> str:
"""Get plain text from a tag with ScaGoudy decoding."""
return decode_scagoudy(tag.get_text().strip())
def get_css_class(tag: Tag) -> str:
"""Get the first CSS class of a tag, or empty string."""
classes = tag.get("class", [])
return classes[0] if classes else ""
def is_empty(tag: Tag) -> bool:
"""Check if a tag is empty or whitespace-only."""
text = tag.get_text().strip()
return not text or text == "\xa0"
def is_dash_line(text: str) -> bool:
"""Return True if text is only dashes, spaces, and/or whitespace."""
cleaned = text.strip()
if not cleaned:
return False
return all(c in "-—–\u2014\u2013 \t" for c in cleaned)
# ---------------------------------------------------------------------------
# CSS class categorization for RV
# ---------------------------------------------------------------------------
# Book title header class (multi-line with <br>)
BOOK_HEADER_CLASS = "p85"
# Chapter bookmark class
CHAPTER_BOOKMARK_CLASS = "p13"
# Body paragraph classes (primary content)
BODY_CLASSES = frozenset(["p6"])
# Sanskrit verse pāda classes (italic, centered)
VERSE_PADA_CLASSES = frozenset(["p53"])
# Verse translations / body text
TRANSLATION_CLASS = "p44"
# Chapter heading (bold)
CHAPTER_HEADING_CLASS = "p27"
# Title page text
TITLE_PAGE_CLASS = "p3"
# Contents heading classes
CONTENTS_CLASSES = frozenset(["p87", "p88"])
# Back-cover blurb class
BACK_COVER_CLASS = "p89"
# Classes to skip entirely (empty/decorative)
SKIP_CLASSES = frozenset(["p9", "p70", "p15"])
# ---------------------------------------------------------------------------
# Extraction helpers
# ---------------------------------------------------------------------------
def find_rv_boundaries(all_paragraphs: list) -> tuple:
"""Find the start and end paragraph indices of the RV content.
Start: <p class="p85"> containing "King of Knowledge" and "1973"
End: next <p class="p85"> after start, or <p class="p89"> back-cover text
"""
start_idx = None
end_idx = len(all_paragraphs)
for i, p in enumerate(all_paragraphs):
cls = get_css_class(p)
if cls != BOOK_HEADER_CLASS:
continue
text = decode_text(p)
if "King of Knowledge" in text and "1973" in text:
start_idx = i
elif start_idx is not None:
# Next p85 after RV start = end of RV
end_idx = i
break
if start_idx is None:
print("ERROR: Could not find 'King of Knowledge (1973)' header in other.html!")
sys.exit(1)
# Also check for p89 back-cover text as potential end boundary
# The p89 tag appears right before the next p85 and is part of RV
# so we keep end_idx at the next p85 (or end of file)
return start_idx, end_idx
def find_chapter_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
"""Find all RV chapter bookmark tags (p13) in the RV range.
Returns list of (paragraph_index, chapter_number, chapter_title).
"""
bookmarks = []
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
if cls == CHAPTER_BOOKMARK_CLASS:
text = decode_text(p)
m = re.match(r"RV\s+(\d+):\s+(.+)", text.strip())
if m:
ch_num = int(m.group(1))
ch_title = m.group(2).strip()
bookmarks.append((i, ch_num, ch_title))
return bookmarks
def extract_paragraphs(
all_paragraphs: list,
start_idx: int,
end_idx: int,
is_front_pages: bool = False,
) -> list:
"""Extract paragraphs for a section (chapter or intro segment).
Handles all CSS classes within the RV content.
Groups consecutive p53 verse pādas into centered italic divs.
"""
result = []
verse_accumulator = []
def flush_verse():
if verse_accumulator:
joined = "<br>".join(verse_accumulator)
result.append(f'<div style="text-align:center"><i>{joined}</i></div>')
verse_accumulator.clear()
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
# Skip empty paragraphs
if is_empty(p):
continue
# Skip noise classes
if cls in SKIP_CLASSES:
continue
# Skip bookmark tags (p13)
if cls == CHAPTER_BOOKMARK_CLASS:
continue
# Skip book title header (p85)
if cls == BOOK_HEADER_CLASS:
continue
# Skip back-cover blurb (p89)
if cls == BACK_COVER_CLASS:
continue
# Check for dash lines — skip them
text = decode_text(p)
if is_dash_line(text):
continue
# Sanskrit verse pādas (p53) — accumulate into centered block
if cls in VERSE_PADA_CLASSES:
html = decode_html_content(p)
if html:
# Remove inner <i> tags before re-wrapping
stripped = re.sub(r"</?i>", "", html)
verse_accumulator.append(stripped)
continue
# Non-verse paragraph — flush any accumulated verse first
flush_verse()
# Chapter heading (p27) — include as paragraph
if cls == CHAPTER_HEADING_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Title page text (p3) — include
if cls == TITLE_PAGE_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Contents heading classes (p87, p88) — include in front-pages
if cls in CONTENTS_CLASSES:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Verse translation / body text (p44)
if cls == TRANSLATION_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Body paragraphs (p6)
if cls in BODY_CLASSES:
html = decode_html_content(p)
if html:
# Detect Sanskrit verses disguised as body paragraphs:
# they start with <i> and contain <br> (multi-line italic = verse)
stripped_html = html.strip()
if stripped_html.startswith("<i>") and "<br" in stripped_html:
# This is a verse — wrap in centered italic div
inner = re.sub(r"</?i>", "", stripped_html)
result.append(
f'<div style="text-align:center"><i>{inner}</i></div>'
)
else:
result.append(html)
continue
# Any other class — try to extract content (be permissive)
html = decode_html_content(p)
if html and not is_dash_line(text):
result.append(html)
# Flush any trailing verse
flush_verse()
return result
# ---------------------------------------------------------------------------
# Main extraction logic
# ---------------------------------------------------------------------------
def extract_rv(soup: BeautifulSoup) -> dict:
"""Extract the entire Rāja-vidyā: The King of Knowledge (1973) from the parsed HTML.
Returns a dict with:
"intro_segments": list of intro segment dicts
"chapters": list of chapter dicts
"chapter_listing": list of {number, title} for intro.json
"""
all_paragraphs = list(soup.find_all("p"))
# Step 1: Find RV boundaries
rv_start, rv_end = find_rv_boundaries(all_paragraphs)
print(f" RV content: paragraphs {rv_start} to {rv_end}")
print(f" Total paragraphs in range: {rv_end - rv_start}")
# Step 2: Find all chapter bookmarks
chapter_bookmarks = find_chapter_bookmarks(all_paragraphs, rv_start, rv_end)
print(f" Found {len(chapter_bookmarks)} RV chapter bookmarks")
for pos, ch_num, ch_title in chapter_bookmarks:
print(f" [chapter {ch_num}] paragraph {pos}: {ch_title}")
# Step 3: Extract front pages
# From book start (rv_start) to first chapter bookmark (RV 1)
intro_segments = []
if chapter_bookmarks:
front_pages_end = chapter_bookmarks[0][0]
else:
front_pages_end = rv_end
front_pages_paragraphs = extract_paragraphs(
all_paragraphs, rv_start, front_pages_end, is_front_pages=True
)
intro_segments.append(
{
"slug": "front-pages",
"title": "Front Pages",
"type": "intro-segment",
"book": "rv",
"order": 1,
"paragraphs": front_pages_paragraphs,
}
)
# Step 4: Extract chapters
chapters = []
for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_bookmarks):
content_start = ch_pos + 1
# Content ends at next chapter bookmark or rv_end
if idx + 1 < len(chapter_bookmarks):
content_end = chapter_bookmarks[idx + 1][0]
else:
content_end = rv_end
paragraphs = extract_paragraphs(all_paragraphs, content_start, content_end)
ch = {
"book": "rv",
"chapter": ch_num,
"title": ch_title,
"type": "chapter",
"paragraphs": paragraphs,
}
chapters.append(ch)
# Step 5: Build chapter listing for intro.json
chapter_listing = [
{"number": ch["chapter"], "title": ch["title"]} for ch in chapters
]
return {
"intro_segments": intro_segments,
"chapters": chapters,
"chapter_listing": chapter_listing,
}
# ---------------------------------------------------------------------------
# Output
# ---------------------------------------------------------------------------
def write_json(data, filepath):
"""Write data as formatted JSON to filepath, creating directories as needed."""
os.makedirs(os.path.dirname(filepath), exist_ok=True)
with open(filepath, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False, indent=2)
def main():
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
html_path = os.path.join(base_dir, "public", "html", "other.html")
output_base = os.path.join(base_dir, "lib", "jsons", "books", "rv")
if not os.path.exists(html_path):
print(f"ERROR: {html_path} not found!")
sys.exit(1)
print(f"Reading {html_path}...")
with open(html_path, "r", encoding="utf-8") as f:
soup = BeautifulSoup(f, "html.parser")
print("Extracting Rāja-vidyā: The King of Knowledge (1973)...\n")
result = extract_rv(soup)
intro_segments = result["intro_segments"]
chapters = result["chapters"]
chapter_listing = result["chapter_listing"]
# --- Write intro.json ---
intro_meta = {
"book": "rv",
"title": "Rāja-vidyā: The King of Knowledge",
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
"year": 1973,
"description": "The king of knowledge — the most confidential wisdom of the Bhagavad-gītā.",
"totalChapters": 8,
"chapters": chapter_listing,
}
intro_path = os.path.join(output_base, "intro.json")
write_json(intro_meta, intro_path)
print(f"\n Written: {intro_path}")
# --- Write intro segments ---
print(f"\n--- Intro Segments ---")
for seg in intro_segments:
filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
write_json(seg, filepath)
print(
f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}"
)
# --- Write chapters ---
print(f"\n--- Chapters ---")
for ch in chapters:
ch_num_str = str(ch["chapter"]).zfill(2)
filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
write_json(ch, filepath)
print(
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
f"{ch['title']}"
)
# --- Summary ---
total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments)
total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
total_paras = total_intro_paras + total_chapter_paras
print(f"\n{'=' * 60}")
print(f" EXTRACTION SUMMARY")
print(f"{'=' * 60}")
print(f" Intro segments: {len(intro_segments)}")
print(f" Chapters: {len(chapters)}")
print(f" Total paragraphs: {total_paras}")
print(f" Intro: {total_intro_paras}")
print(f" Chapters: {total_chapter_paras}")
if chapters:
print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
print(f" Output directory: {output_base}/")
# --- Samples ---
print(f"\n--- Sample: Front Pages ---")
if intro_segments:
front = intro_segments[0]
for i, p in enumerate(front["paragraphs"][:5]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
if chapters:
print(f"\n--- Sample: Chapter 1 ({chapters[0]['title']}) ---")
ch1 = chapters[0]
for i, p in enumerate(ch1["paragraphs"][:5]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
if chapters and len(chapters) > 1:
print(
f"\n--- Sample: Chapter {chapters[-1]['chapter']} ({chapters[-1]['title']}) ---"
)
last = chapters[-1]
for i, p in enumerate(last["paragraphs"][:5]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
# --- Quality Checks ---
print(f"\n--- Quality Checks ---")
# Check all 8 chapters present
expected_chapters = set(range(1, 9))
actual_chapters = set(ch["chapter"] for ch in chapters)
missing = expected_chapters - actual_chapters
extra = actual_chapters - expected_chapters
if missing:
print(f" ⚠ Missing chapters: {sorted(missing)}")
if extra:
print(f" ⚠ Extra chapters: {sorted(extra)}")
if not missing and not extra:
print(f" ✅ All 8 chapters present")
# Check all 1 intro segment present
expected_intros = {"front-pages"}
actual_intros = {s["slug"] for s in intro_segments}
missing_intros = expected_intros - actual_intros
extra_intros = actual_intros - expected_intros
if missing_intros:
print(f" ⚠ Missing intro segments: {missing_intros}")
if extra_intros:
print(f" ⚠ Extra intro segments: {extra_intros}")
if not missing_intros and not extra_intros:
print(f" ✅ All 1 intro segments present")
# Check empty chapters
empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
if empty_chapters:
print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
else:
print(f" ✅ No empty chapters")
# Check empty intro segments
empty_intros = [s["slug"] for s in intro_segments if len(s["paragraphs"]) == 0]
if empty_intros:
print(f" ⚠ Empty intro segments: {empty_intros}")
else:
print(f" ✅ No empty intro segments")
# Check ScaGoudy decoding
print(f"\n--- ScaGoudy Decoding Check ---")
all_text = ""
for ch in chapters:
all_text += " ".join(ch["paragraphs"][:20])
for seg in intro_segments:
all_text += " ".join(seg["paragraphs"][:10])
# Check for properly decoded words
if "Kṛṣṇa" in all_text:
print(f" ✅ 'Kṛṣṇa' found (ScaGoudy properly decoded)")
else:
print(f" ⚠ 'Kṛṣṇa' not found — ScaGoudy decoding may have failed")
if "Bhagavad-gītā" in all_text:
print(f" ✅ 'Bhagavad-gītā' found (ScaGoudy properly decoded)")
elif "gītā" in all_text or "Gītā" in all_text:
print(f" ✅ 'gītā' found (ScaGoudy properly decoded)")
else:
print(f" ⚠ 'Gītā' not found — ScaGoudy decoding may have failed")
if "Prabhupāda" in all_text:
print(f" ✅ 'Prabhupāda' found (ScaGoudy properly decoded)")
# Check for un-decoded ScaGoudy remnants (common encoded chars)
scagoudy_remnants = 0
for ch in chapters:
for p in ch["paragraphs"]:
if "Kåñëa" in p or "Prabhupäda" in p:
scagoudy_remnants += 1
for seg in intro_segments:
for p in seg["paragraphs"]:
if "Kåñëa" in p or "Prabhupäda" in p:
scagoudy_remnants += 1
if scagoudy_remnants == 0:
print(f" ✅ No ScaGoudy remnants detected")
else:
print(f" ⚠ {scagoudy_remnants} paragraphs with ScaGoudy remnants")
# Check for centered verse blocks
verse_block_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
if '<div style="text-align:center"><i>' in p:
verse_block_count += 1
if verse_block_count > 0:
print(f" ✅ {verse_block_count} centered Sanskrit verse blocks found")
else:
print(f" ⚠ No centered verse blocks found — verse grouping may have failed")
# Check for residual span tags
span_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
if 'class="' in p and "<span" in p:
span_count += 1
for seg in intro_segments:
for p in seg["paragraphs"]:
if 'class="' in p and "<span" in p:
span_count += 1
if span_count == 0:
print(f" ✅ No residual span class artifacts found")
else:
print(f" ⚠ {span_count} paragraphs still contain span class artifacts")
# Check for dash-line paragraphs that shouldn't be there
dash_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
plain = re.sub(r"<[^>]+>", "", p)
if is_dash_line(plain):
dash_count += 1
for seg in intro_segments:
for p in seg["paragraphs"]:
plain = re.sub(r"<[^>]+>", "", p)
if is_dash_line(plain):
dash_count += 1
if dash_count == 0:
print(f" ✅ No dashed-line paragraphs in output")
else:
print(f" ⚠ {dash_count} dashed-line paragraphs found in output")
# Show a verse block sample
print(f"\n--- Sample: Verse Block ---")
for ch in chapters:
for p in ch["paragraphs"]:
if '<div style="text-align:center"><i>' in p:
preview = re.sub(r"<[^>]+>", "", p)[:120]
print(f" {preview}...")
break
else:
continue
break
# Chapter title decoding check
print(f"\n--- Chapter Title Decoding Check ---")
for ch in chapters:
title = ch["title"]
if "å" in title or "ë" in title or "ñ" in title or "ä" in title:
print(f" ⚠ Chapter {ch['chapter']} title has un-decoded ScaGoudy: {title}")
else:
print(f" ✅ Ch {ch['chapter']}: {title}")
print(f"\nDone! ✨")
if __name__ == "__main__":
main()
scripts/seed-bbd.ts
+204
View File
@@ -0,0 +1,204 @@
/**
* seed-bbd.ts — Push Beyond Birth and Death JSON data into MongoDB.
*
* Uses the native MongoDB driver (same pattern as seed-ek.ts) to avoid
* Prisma replica-set write requirements.
*
* BBD is a prose book (like EK) with no volume structure.
* We use volume=0 as the sentinel for "no volumes".
*
* Collections written:
* - book_meta → one doc { book: "bbd", data: <intro.json> }
* - prose_chapters → 1 intro segment + 5 chapters = 6 docs
*
* Usage:
* npx tsx scripts/seed-bbd.ts
*/
import dotenv from "dotenv";
import fs from "fs";
import path from "path";
import { MongoClient } from "mongodb";
// Load env
dotenv.config({ path: ".env.local" });
dotenv.config({ path: ".env" });
const DATABASE_URL = process.env.DATABASE_URL;
if (!DATABASE_URL) {
console.error("❌ DATABASE_URL is not set in .env");
process.exit(1);
}
const BASE_DIR = path.resolve(__dirname, "..");
const BBD_DIR = path.join(BASE_DIR, "lib", "jsons", "books", "bbd");
/** Volume sentinel for books without volumes. */
const NO_VOLUME = 0;
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function readJson<T>(filePath: string): T {
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
}
/**
* Insert documents one at a time with a small delay every N docs.
* Uses insertOne to avoid BSON wire-protocol corruption.
*/
async function insertBatched(
col: ReturnType<ReturnType<MongoClient["db"]>["collection"]>,
docs: Record<string, unknown>[],
) {
for (let i = 0; i < docs.length; i++) {
await col.insertOne(docs[i]);
if ((i + 1) % 100 === 0) {
await new Promise((r) => setTimeout(r, 100));
}
}
}
// ---------------------------------------------------------------------------
// Types for the JSON files
// ---------------------------------------------------------------------------
type BbdIntroJson = {
book: string;
title: string;
author: string;
year: number;
description: string;
totalChapters: number;
chapters: { number: number; title: string }[];
};
type BbdIntroSegmentJson = {
slug: string;
title: string;
type: string;
book: string;
order: number;
paragraphs: string[];
};
type BbdChapterJson = {
book: string;
chapter: number;
title: string;
type: string;
paragraphs: string[];
};
// ---------------------------------------------------------------------------
// Seed function
// ---------------------------------------------------------------------------
async function seedBbd(db: ReturnType<MongoClient["db"]>) {
const introPath = path.join(BBD_DIR, "intro.json");
if (!fs.existsSync(introPath)) {
console.error("❌ BBD intro.json not found at", introPath);
process.exit(1);
}
const intro = readJson<BbdIntroJson>(introPath);
// ---- 1. Book meta ----
const metaCol = db.collection("book_meta");
await metaCol.deleteMany({ book: "bbd" });
await metaCol.insertOne({ book: "bbd", data: intro });
console.log(" 📖 BBD book meta inserted");
// ---- 2. Prose chapters collection ----
const proseCol = db.collection("prose_chapters");
await proseCol.deleteMany({ book: "bbd" });
const docs: Record<string, unknown>[] = [];
// Intro segments
const introSegmentFiles = ["front-pages.json"];
const introDir = path.join(BBD_DIR, "intro");
for (const file of introSegmentFiles) {
const filePath = path.join(introDir, file);
if (!fs.existsSync(filePath)) {
console.warn(` ⚠️ Intro segment not found, skipping: ${file}`);
continue;
}
const seg = readJson<BbdIntroSegmentJson>(filePath);
docs.push({
book: "bbd",
volume: NO_VOLUME,
chapter: null,
title: seg.title,
type: "intro-segment",
slug: `intro-${seg.slug}`,
order: seg.order,
paragraphs: seg.paragraphs,
});
}
console.log(` 📄 Intro segments: ${docs.length}`);
// Chapters (01.json through 05.json)
const chaptersDir = path.join(BBD_DIR, "chapters");
if (fs.existsSync(chaptersDir)) {
const chapterFiles = fs
.readdirSync(chaptersDir)
.filter((f) => f.endsWith(".json"))
.sort();
for (const file of chapterFiles) {
const ch = readJson<BbdChapterJson>(path.join(chaptersDir, file));
const chNum = ch.chapter ?? parseInt(file.replace(".json", ""), 10);
docs.push({
book: "bbd",
volume: NO_VOLUME,
chapter: chNum,
title: ch.title,
type: "chapter",
slug: String(chNum),
order: 100 + chNum,
paragraphs: ch.paragraphs,
});
}
console.log(` 📖 Chapters: ${chapterFiles.length}`);
}
// Insert all docs
console.log(`\n Inserting ${docs.length} documents...`);
await insertBatched(proseCol, docs);
const totalDocs = await proseCol.countDocuments({ book: "bbd" });
console.log(` ✅ BBD: ${totalDocs} prose entries in MongoDB`);
}
// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------
async function main() {
console.log("🌱 Seeding Beyond Birth and Death...\n");
const client = new MongoClient(DATABASE_URL!);
await client.connect();
console.log("📡 Connected to MongoDB\n");
const db = client.db();
const start = Date.now();
await seedBbd(db);
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
console.log(`\n🎉 Done in ${elapsed}s`);
await client.close();
}
main().catch((err) => {
console.error("❌ Seed failed:", err);
process.exit(1);
});
scripts/seed-ds.ts
+237
View File
@@ -0,0 +1,237 @@
/**
* seed-ds.ts — Push Dialectic Spiritualism JSON data into MongoDB.
*
* Uses the native MongoDB driver (same pattern as seed-ssr.ts) to avoid
* Prisma replica-set write requirements.
*
* DS is a large prose/dialogue book with no volume structure.
* We use volume=0 as the sentinel for "no volumes".
*
* Sub-chapters are stored with slugs like "1-1", "1-2", "2-1", etc.
* Each sub-chapter JSON has parentChapter and subChapter fields used to
* compute a stable ordering: parentChapter * 100 + subChapter.
*
* Collections written:
* - book_meta → one doc { book: "ds", data: <intro.json> }
* - prose_chapters → 3 intro segments + 38 sub-chapters
*
* Usage:
* npx tsx scripts/seed-ds.ts
*/
import dotenv from "dotenv";
import fs from "fs";
import path from "path";
import { MongoClient } from "mongodb";
// Load env
dotenv.config({ path: ".env.local" });
dotenv.config({ path: ".env" });
const DATABASE_URL = process.env.DATABASE_URL;
if (!DATABASE_URL) {
console.error("❌ DATABASE_URL is not set in .env");
process.exit(1);
}
const BASE_DIR = path.resolve(__dirname, "..");
const DS_DIR = path.join(BASE_DIR, "lib", "jsons", "books", "ds");
/** Volume sentinel for books without volumes. */
const NO_VOLUME = 0;
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function readJson<T>(filePath: string): T {
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
}
/**
* Insert documents one at a time with a small delay every N docs.
* Uses insertOne to avoid BSON wire-protocol corruption.
*/
async function insertBatched(
col: ReturnType<ReturnType<MongoClient["db"]>["collection"]>,
docs: Record<string, unknown>[],
) {
for (let i = 0; i < docs.length; i++) {
await col.insertOne(docs[i]);
if ((i + 1) % 100 === 0) {
await new Promise((r) => setTimeout(r, 100));
}
}
}
// ---------------------------------------------------------------------------
// Types for the JSON files
// ---------------------------------------------------------------------------
type DsIntroJson = {
book: string;
title: string;
subtitle: string;
author: string;
year: number;
description: string;
totalChapters: number;
chapters: {
number: number;
title: string;
subChapters: { slug: string; title: string }[];
}[];
};
type DsIntroSegmentJson = {
slug: string;
title: string;
type: string;
book: string;
order: number;
paragraphs: string[];
};
type DsChapterJson = {
book: string;
chapter: string; // "1-1", "2-3", etc.
parentChapter: number;
subChapter: number;
title: string;
parentTitle: string;
type: string;
paragraphs: string[];
};
// ---------------------------------------------------------------------------
// Seed function
// ---------------------------------------------------------------------------
async function seedDs(db: ReturnType<MongoClient["db"]>) {
const introPath = path.join(DS_DIR, "intro.json");
if (!fs.existsSync(introPath)) {
console.error("❌ DS intro.json not found at", introPath);
process.exit(1);
}
const intro = readJson<DsIntroJson>(introPath);
// ---- 1. Book meta ----
const metaCol = db.collection("book_meta");
await metaCol.deleteMany({ book: "ds" });
await metaCol.insertOne({ book: "ds", data: intro });
console.log(" 📖 DS book meta inserted");
// ---- 2. Prose chapters collection ----
const proseCol = db.collection("prose_chapters");
await proseCol.deleteMany({ book: "ds" });
const docs: Record<string, unknown>[] = [];
// Intro segments (3 total: front-pages, foreword, preface)
const introSegmentFiles = [
"front-pages.json",
"foreword.json",
"preface.json",
];
const introDir = path.join(DS_DIR, "intro");
for (const file of introSegmentFiles) {
const filePath = path.join(introDir, file);
if (!fs.existsSync(filePath)) {
console.warn(` ⚠️ Intro segment not found, skipping: ${file}`);
continue;
}
const seg = readJson<DsIntroSegmentJson>(filePath);
docs.push({
book: "ds",
volume: NO_VOLUME,
chapter: null,
title: seg.title,
type: "intro-segment",
slug: `intro-${seg.slug}`,
order: seg.order,
paragraphs: seg.paragraphs,
});
}
console.log(` 📄 Intro segments: ${docs.length}`);
// Sub-chapters (1-1.json, 1-2.json, 2-1.json, etc.)
const chaptersDir = path.join(DS_DIR, "chapters");
let chapterCount = 0;
if (fs.existsSync(chaptersDir)) {
const chapterFiles = fs
.readdirSync(chaptersDir)
.filter((f) => f.endsWith(".json"))
.sort((a, b) => {
// Sort by parentChapter then subChapter numerically
const parseSlug = (name: string) => {
const slug = name.replace(".json", "");
const parts = slug.split("-");
return {
parent: parseInt(parts[0], 10),
sub: parseInt(parts[1], 10),
};
};
const aa = parseSlug(a);
const bb = parseSlug(b);
if (aa.parent !== bb.parent) return aa.parent - bb.parent;
return aa.sub - bb.sub;
});
for (const file of chapterFiles) {
const ch = readJson<DsChapterJson>(path.join(chaptersDir, file));
const order = ch.parentChapter * 100 + ch.subChapter;
docs.push({
book: "ds",
volume: NO_VOLUME,
chapter: null, // null since chapter is a string "1-1" not an int
title: ch.title,
type: "chapter",
slug: ch.chapter, // "1-1", "1-2", "2-1", etc.
order,
paragraphs: ch.paragraphs,
});
chapterCount++;
}
console.log(` 📖 Sub-chapters: ${chapterCount}`);
}
// Insert all docs
console.log(`\n Inserting ${docs.length} documents...`);
await insertBatched(proseCol, docs);
const totalDocs = await proseCol.countDocuments({ book: "ds" });
console.log(` ✅ DS: ${totalDocs} prose entries in MongoDB`);
}
// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------
async function main() {
console.log("🌱 Seeding Dialectic Spiritualism...\n");
const client = new MongoClient(DATABASE_URL!);
await client.connect();
console.log("📡 Connected to MongoDB\n");
const db = client.db();
const start = Date.now();
await seedDs(db);
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
console.log(`\n🎉 Done in ${elapsed}s`);
await client.close();
}
main().catch((err) => {
console.error("❌ Seed failed:", err);
process.exit(1);
});
scripts/seed-ej.ts
+208
View File
@@ -0,0 +1,208 @@
/**
* seed-ej.ts — Push Easy Journey to Other Planets JSON data into MongoDB.
*
* Uses the native MongoDB driver (same pattern as seed-nod.ts) to avoid
* Prisma replica-set write requirements.
*
* EJ is a prose book (like NoD/TLC) with no volume structure.
* We use volume=0 as the sentinel for "no volumes".
*
* Collections written:
* - book_meta → one doc { book: "ej", data: <intro.json> }
* - prose_chapters → 2 intro segments + 2 chapters = 4 docs
*
* Usage:
* npx tsx scripts/seed-ej.ts
*/
import dotenv from "dotenv";
import fs from "fs";
import path from "path";
import { MongoClient } from "mongodb";
// Load env
dotenv.config({ path: ".env.local" });
dotenv.config({ path: ".env" });
const DATABASE_URL = process.env.DATABASE_URL;
if (!DATABASE_URL) {
console.error("❌ DATABASE_URL is not set in .env");
process.exit(1);
}
const BASE_DIR = path.resolve(__dirname, "..");
const EJ_DIR = path.join(BASE_DIR, "lib", "jsons", "books", "ej");
/** Volume sentinel for books without volumes. */
const NO_VOLUME = 0;
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function readJson<T>(filePath: string): T {
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
}
/**
* Insert documents one at a time with a small delay every N docs.
* Uses insertOne to avoid BSON wire-protocol corruption.
*/
async function insertBatched(
col: ReturnType<ReturnType<MongoClient["db"]>["collection"]>,
docs: Record<string, unknown>[],
) {
for (let i = 0; i < docs.length; i++) {
await col.insertOne(docs[i]);
if ((i + 1) % 100 === 0) {
await new Promise((r) => setTimeout(r, 100));
}
}
}
// ---------------------------------------------------------------------------
// Types for the JSON files
// ---------------------------------------------------------------------------
type EjIntroJson = {
book: string;
title: string;
subtitle: string;
author: string;
year: number;
description: string;
totalChapters: number;
chapters: { number: number; title: string }[];
};
type EjIntroSegmentJson = {
slug: string;
title: string;
type: string;
book: string;
order: number;
paragraphs: string[];
};
type EjChapterJson = {
book: string;
chapter: number;
title: string;
type: string;
paragraphs: string[];
};
// ---------------------------------------------------------------------------
// Seed function
// ---------------------------------------------------------------------------
async function seedEj(db: ReturnType<MongoClient["db"]>) {
const introPath = path.join(EJ_DIR, "intro.json");
if (!fs.existsSync(introPath)) {
console.error("❌ EJ intro.json not found at", introPath);
process.exit(1);
}
const intro = readJson<EjIntroJson>(introPath);
// ---- 1. Book meta ----
const metaCol = db.collection("book_meta");
await metaCol.deleteMany({ book: "ej" });
await metaCol.insertOne({ book: "ej", data: intro });
console.log(" 📖 EJ book meta inserted");
// ---- 2. Prose chapters collection ----
const proseCol = db.collection("prose_chapters");
await proseCol.deleteMany({ book: "ej" });
const docs: Record<string, unknown>[] = [];
// Intro segments
const introSegmentFiles = [
"front-pages.json",
"preface.json",
];
const introDir = path.join(EJ_DIR, "intro");
for (const file of introSegmentFiles) {
const filePath = path.join(introDir, file);
if (!fs.existsSync(filePath)) {
console.warn(` ⚠️ Intro segment not found, skipping: ${file}`);
continue;
}
const seg = readJson<EjIntroSegmentJson>(filePath);
docs.push({
book: "ej",
volume: NO_VOLUME,
chapter: null,
title: seg.title,
type: "intro-segment",
slug: `intro-${seg.slug}`,
order: seg.order,
paragraphs: seg.paragraphs,
});
}
console.log(` 📄 Intro segments: ${docs.length}`);
// Chapters (01.json, 02.json)
const chaptersDir = path.join(EJ_DIR, "chapters");
if (fs.existsSync(chaptersDir)) {
const chapterFiles = fs
.readdirSync(chaptersDir)
.filter((f) => f.endsWith(".json"))
.sort();
for (const file of chapterFiles) {
const ch = readJson<EjChapterJson>(path.join(chaptersDir, file));
const chNum = ch.chapter ?? parseInt(file.replace(".json", ""), 10);
docs.push({
book: "ej",
volume: NO_VOLUME,
chapter: chNum,
title: ch.title,
type: "chapter",
slug: String(chNum),
order: 100 + chNum,
paragraphs: ch.paragraphs,
});
}
console.log(` 📖 Chapters: ${chapterFiles.length}`);
}
// Insert all docs
console.log(`\n Inserting ${docs.length} documents...`);
await insertBatched(proseCol, docs);
const totalDocs = await proseCol.countDocuments({ book: "ej" });
console.log(` ✅ EJ: ${totalDocs} prose entries in MongoDB`);
}
// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------
async function main() {
console.log("🌱 Seeding Easy Journey to Other Planets...\n");
const client = new MongoClient(DATABASE_URL!);
await client.connect();
console.log("📡 Connected to MongoDB\n");
const db = client.db();
const start = Date.now();
await seedEj(db);
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
console.log(`\n🎉 Done in ${elapsed}s`);
await client.close();
}
main().catch((err) => {
console.error("❌ Seed failed:", err);
process.exit(1);
});
scripts/seed-ek.ts
+204
View File
@@ -0,0 +1,204 @@
/**
* seed-ek.ts — Push Elevation to Kṛṣṇa Consciousness JSON data into MongoDB.
*
* Uses the native MongoDB driver (same pattern as seed-ej.ts) to avoid
* Prisma replica-set write requirements.
*
* EK is a prose book (like EJ) with no volume structure.
* We use volume=0 as the sentinel for "no volumes".
*
* Collections written:
* - book_meta → one doc { book: "ek", data: <intro.json> }
* - prose_chapters → 1 intro segment + 6 chapters = 7 docs
*
* Usage:
* npx tsx scripts/seed-ek.ts
*/
import dotenv from "dotenv";
import fs from "fs";
import path from "path";
import { MongoClient } from "mongodb";
// Load env
dotenv.config({ path: ".env.local" });
dotenv.config({ path: ".env" });
const DATABASE_URL = process.env.DATABASE_URL;
if (!DATABASE_URL) {
console.error("❌ DATABASE_URL is not set in .env");
process.exit(1);
}
const BASE_DIR = path.resolve(__dirname, "..");
const EK_DIR = path.join(BASE_DIR, "lib", "jsons", "books", "ek");
/** Volume sentinel for books without volumes. */
const NO_VOLUME = 0;
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function readJson<T>(filePath: string): T {
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
}
/**
* Insert documents one at a time with a small delay every N docs.
* Uses insertOne to avoid BSON wire-protocol corruption.
*/
async function insertBatched(
col: ReturnType<ReturnType<MongoClient["db"]>["collection"]>,
docs: Record<string, unknown>[],
) {
for (let i = 0; i < docs.length; i++) {
await col.insertOne(docs[i]);
if ((i + 1) % 100 === 0) {
await new Promise((r) => setTimeout(r, 100));
}
}
}
// ---------------------------------------------------------------------------
// Types for the JSON files
// ---------------------------------------------------------------------------
type EkIntroJson = {
book: string;
title: string;
author: string;
year: number;
description: string;
totalChapters: number;
chapters: { number: number; title: string }[];
};
type EkIntroSegmentJson = {
slug: string;
title: string;
type: string;
book: string;
order: number;
paragraphs: string[];
};
type EkChapterJson = {
book: string;
chapter: number;
title: string;
type: string;
paragraphs: string[];
};
// ---------------------------------------------------------------------------
// Seed function
// ---------------------------------------------------------------------------
async function seedEk(db: ReturnType<MongoClient["db"]>) {
const introPath = path.join(EK_DIR, "intro.json");
if (!fs.existsSync(introPath)) {
console.error("❌ EK intro.json not found at", introPath);
process.exit(1);
}
const intro = readJson<EkIntroJson>(introPath);
// ---- 1. Book meta ----
const metaCol = db.collection("book_meta");
await metaCol.deleteMany({ book: "ek" });
await metaCol.insertOne({ book: "ek", data: intro });
console.log(" 📖 EK book meta inserted");
// ---- 2. Prose chapters collection ----
const proseCol = db.collection("prose_chapters");
await proseCol.deleteMany({ book: "ek" });
const docs: Record<string, unknown>[] = [];
// Intro segments
const introSegmentFiles = ["front-pages.json"];
const introDir = path.join(EK_DIR, "intro");
for (const file of introSegmentFiles) {
const filePath = path.join(introDir, file);
if (!fs.existsSync(filePath)) {
console.warn(` ⚠️ Intro segment not found, skipping: ${file}`);
continue;
}
const seg = readJson<EkIntroSegmentJson>(filePath);
docs.push({
book: "ek",
volume: NO_VOLUME,
chapter: null,
title: seg.title,
type: "intro-segment",
slug: `intro-${seg.slug}`,
order: seg.order,
paragraphs: seg.paragraphs,
});
}
console.log(` 📄 Intro segments: ${docs.length}`);
// Chapters (01.json through 06.json)
const chaptersDir = path.join(EK_DIR, "chapters");
if (fs.existsSync(chaptersDir)) {
const chapterFiles = fs
.readdirSync(chaptersDir)
.filter((f) => f.endsWith(".json"))
.sort();
for (const file of chapterFiles) {
const ch = readJson<EkChapterJson>(path.join(chaptersDir, file));
const chNum = ch.chapter ?? parseInt(file.replace(".json", ""), 10);
docs.push({
book: "ek",
volume: NO_VOLUME,
chapter: chNum,
title: ch.title,
type: "chapter",
slug: String(chNum),
order: 100 + chNum,
paragraphs: ch.paragraphs,
});
}
console.log(` 📖 Chapters: ${chapterFiles.length}`);
}
// Insert all docs
console.log(`\n Inserting ${docs.length} documents...`);
await insertBatched(proseCol, docs);
const totalDocs = await proseCol.countDocuments({ book: "ek" });
console.log(` ✅ EK: ${totalDocs} prose entries in MongoDB`);
}
// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------
async function main() {
console.log("🌱 Seeding Elevation to Kṛṣṇa Consciousness...\n");
const client = new MongoClient(DATABASE_URL!);
await client.connect();
console.log("📡 Connected to MongoDB\n");
const db = client.db();
const start = Date.now();
await seedEk(db);
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
console.log(`\n🎉 Done in ${elapsed}s`);
await client.close();
}
main().catch((err) => {
console.error("❌ Seed failed:", err);
process.exit(1);
});
scripts/seed-hkh.ts
+204
View File
@@ -0,0 +1,204 @@
/**
* seed-hkh.ts — Push The Kṛṣṇa Consciousness Handbook JSON data into MongoDB.
*
* Uses the native MongoDB driver (same pattern as seed-owk.ts) to avoid
* Prisma replica-set write requirements.
*
* HKH is a prose book with no volume structure.
* We use volume=0 as the sentinel for "no volumes".
*
* Collections written:
* - book_meta → one doc { book: "hkh", data: <intro.json> }
* - prose_chapters → 2 intro segments + 7 chapters = 9 docs
*
* Usage:
* npx tsx scripts/seed-hkh.ts
*/
import dotenv from "dotenv";
import fs from "fs";
import path from "path";
import { MongoClient } from "mongodb";
// Load env
dotenv.config({ path: ".env.local" });
dotenv.config({ path: ".env" });
const DATABASE_URL = process.env.DATABASE_URL;
if (!DATABASE_URL) {
console.error("❌ DATABASE_URL is not set in .env");
process.exit(1);
}
const BASE_DIR = path.resolve(__dirname, "..");
const HKH_DIR = path.join(BASE_DIR, "lib", "jsons", "books", "hkh");
/** Volume sentinel for books without volumes. */
const NO_VOLUME = 0;
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function readJson<T>(filePath: string): T {
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
}
/**
* Insert documents one at a time with a small delay every N docs.
* Uses insertOne to avoid BSON wire-protocol corruption.
*/
async function insertBatched(
col: ReturnType<ReturnType<MongoClient["db"]>["collection"]>,
docs: Record<string, unknown>[],
) {
for (let i = 0; i < docs.length; i++) {
await col.insertOne(docs[i]);
if ((i + 1) % 100 === 0) {
await new Promise((r) => setTimeout(r, 100));
}
}
}
// ---------------------------------------------------------------------------
// Types for the JSON files
// ---------------------------------------------------------------------------
type HkhIntroJson = {
book: string;
title: string;
author: string;
year: number;
description: string;
totalChapters: number;
chapters: { number: number; title: string }[];
};
type HkhIntroSegmentJson = {
slug: string;
title: string;
type: string;
book: string;
order: number;
paragraphs: string[];
};
type HkhChapterJson = {
book: string;
chapter: number;
title: string;
type: string;
paragraphs: string[];
};
// ---------------------------------------------------------------------------
// Seed function
// ---------------------------------------------------------------------------
async function seedHkh(db: ReturnType<MongoClient["db"]>) {
const introPath = path.join(HKH_DIR, "intro.json");
if (!fs.existsSync(introPath)) {
console.error("❌ HKH intro.json not found at", introPath);
process.exit(1);
}
const intro = readJson<HkhIntroJson>(introPath);
// ---- 1. Book meta ----
const metaCol = db.collection("book_meta");
await metaCol.deleteMany({ book: "hkh" });
await metaCol.insertOne({ book: "hkh", data: intro });
console.log(" 📖 HKH book meta inserted");
// ---- 2. Prose chapters collection ----
const proseCol = db.collection("prose_chapters");
await proseCol.deleteMany({ book: "hkh" });
const docs: Record<string, unknown>[] = [];
// Intro segments
const introSegmentFiles = ["front-pages.json", "introduction.json"];
const introDir = path.join(HKH_DIR, "intro");
for (const file of introSegmentFiles) {
const filePath = path.join(introDir, file);
if (!fs.existsSync(filePath)) {
console.warn(` ⚠️ Intro segment not found, skipping: ${file}`);
continue;
}
const seg = readJson<HkhIntroSegmentJson>(filePath);
docs.push({
book: "hkh",
volume: NO_VOLUME,
chapter: null,
title: seg.title,
type: "intro-segment",
slug: `intro-${seg.slug}`,
order: seg.order,
paragraphs: seg.paragraphs,
});
}
console.log(` 📄 Intro segments: ${docs.length}`);
// Chapters (01.json through 07.json)
const chaptersDir = path.join(HKH_DIR, "chapters");
if (fs.existsSync(chaptersDir)) {
const chapterFiles = fs
.readdirSync(chaptersDir)
.filter((f) => f.endsWith(".json"))
.sort();
for (const file of chapterFiles) {
const ch = readJson<HkhChapterJson>(path.join(chaptersDir, file));
const chNum = ch.chapter ?? parseInt(file.replace(".json", ""), 10);
docs.push({
book: "hkh",
volume: NO_VOLUME,
chapter: chNum,
title: ch.title,
type: "chapter",
slug: String(chNum),
order: 100 + chNum,
paragraphs: ch.paragraphs,
});
}
console.log(` 📖 Chapters: ${chapterFiles.length}`);
}
// Insert all docs
console.log(`\n Inserting ${docs.length} documents...`);
await insertBatched(proseCol, docs);
const totalDocs = await proseCol.countDocuments({ book: "hkh" });
console.log(` ✅ HKH: ${totalDocs} prose entries in MongoDB`);
}
// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------
async function main() {
console.log("🌱 Seeding The Kṛṣṇa Consciousness Handbook...\n");
const client = new MongoClient(DATABASE_URL!);
await client.connect();
console.log("📡 Connected to MongoDB\n");
const db = client.db();
const start = Date.now();
await seedHkh(db);
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
console.log(`\n🎉 Done in ${elapsed}s`);
await client.close();
}
main().catch((err) => {
console.error("❌ Seed failed:", err);
process.exit(1);
});
scripts/seed-iso.ts
+235
View File
@@ -0,0 +1,235 @@
/**
* seed-iso.ts — Push Śrī Īśopaniṣad JSON data into MongoDB.
*
* Uses the native MongoDB driver (same pattern as seed-nod.ts) to avoid
* Prisma replica-set write requirements.
*
* ISO is a verse book (like BG) but also has prose intro segments.
* We use volume=0 as the sentinel for "no volumes" on the prose side.
*
* Collections written:
* - book_meta → one doc { book: "iso", data: <intro.json> }
* - verses → 19 verse docs (Invocation + Mantras 1-18)
* - prose_chapters → 3 intro segment docs (front-pages, introduction, about-the-author)
*
* Usage:
* npx tsx scripts/seed-iso.ts
*/
import dotenv from "dotenv";
import fs from "fs";
import path from "path";
import { MongoClient } from "mongodb";
// Load env
dotenv.config({ path: ".env.local" });
dotenv.config({ path: ".env" });
const DATABASE_URL = process.env.DATABASE_URL;
if (!DATABASE_URL) {
console.error("❌ DATABASE_URL is not set in .env");
process.exit(1);
}
const BASE_DIR = path.resolve(__dirname, "..");
const ISO_DIR = path.join(BASE_DIR, "lib", "jsons", "books", "iso");
/** Volume sentinel for books without volumes. */
const NO_VOLUME = 0;
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function readJson<T>(filePath: string): T {
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
}
/**
* Insert documents one at a time with a small delay every N docs.
* Uses insertOne to avoid BSON wire-protocol corruption.
*/
async function insertBatched(
col: ReturnType<ReturnType<MongoClient["db"]>["collection"]>,
docs: Record<string, unknown>[],
) {
for (let i = 0; i < docs.length; i++) {
await col.insertOne(docs[i]);
if ((i + 1) % 100 === 0) {
await new Promise((r) => setTimeout(r, 100));
}
}
}
// ---------------------------------------------------------------------------
// Types for the JSON files
// ---------------------------------------------------------------------------
type IsoIntroJson = {
book: string;
title: string;
author: string;
year: number;
description: string;
totalMantras: number;
mantras: { number: number; title: string; slug: string }[];
};
type IsoVerseJson = {
book: string;
book_title: string;
mantra: number;
title: string;
reference: string;
devanagari: string;
transliteration: string;
synonyms: { word: string; meanings: string[] }[];
translation: string;
purport: string[];
};
type IsoIntroSegmentJson = {
slug: string;
title: string;
type: string;
book: string;
order: number;
paragraphs: string[];
};
// ---------------------------------------------------------------------------
// Seed function
// ---------------------------------------------------------------------------
async function seedIso(db: ReturnType<MongoClient["db"]>) {
const introPath = path.join(ISO_DIR, "intro.json");
if (!fs.existsSync(introPath)) {
console.error("❌ ISO intro.json not found at", introPath);
process.exit(1);
}
const intro = readJson<IsoIntroJson>(introPath);
// ---- 1. Book meta ----
const metaCol = db.collection("book_meta");
await metaCol.deleteMany({ book: "iso" });
await metaCol.insertOne({ book: "iso", data: intro });
console.log(" 📖 ISO book meta inserted");
// ---- 2. Verses ----
const versesCol = db.collection("verses");
await versesCol.deleteMany({ book: "iso" });
const verseDocs: Record<string, unknown>[] = [];
// Verse files: invocation.json, 01.json through 18.json
const verseFiles = [
"invocation.json",
...Array.from({ length: 18 }, (_, i) =>
String(i + 1).padStart(2, "0") + ".json"
),
];
const versesDir = path.join(ISO_DIR, "verses");
for (const file of verseFiles) {
const filePath = path.join(versesDir, file);
if (!fs.existsSync(filePath)) {
console.warn(` ⚠️ Verse file not found, skipping: ${file}`);
continue;
}
const v = readJson<IsoVerseJson>(filePath);
verseDocs.push({
book: "iso",
division: null,
chapter: v.mantra,
verse: 1,
verseEnd: null,
reference: v.reference,
bookTitle: v.book_title,
devanagari: v.devanagari,
bengali: null,
transliteration: v.transliteration,
synonyms: v.synonyms,
translation: v.translation,
purport: v.purport,
});
}
console.log(` 📜 Verses prepared: ${verseDocs.length}`);
console.log(` Inserting ${verseDocs.length} verse documents...`);
await insertBatched(versesCol, verseDocs);
const totalVerses = await versesCol.countDocuments({ book: "iso" });
console.log(` ✅ ISO verses: ${totalVerses} in MongoDB`);
// ---- 3. Prose intro segments ----
const proseCol = db.collection("prose_chapters");
await proseCol.deleteMany({ book: "iso" });
const proseDocs: Record<string, unknown>[] = [];
const introSegmentFiles = [
"front-pages.json",
"introduction.json",
"about-the-author.json",
];
const introDir = path.join(ISO_DIR, "intro");
for (const file of introSegmentFiles) {
const filePath = path.join(introDir, file);
if (!fs.existsSync(filePath)) {
console.warn(` ⚠️ Intro segment not found, skipping: ${file}`);
continue;
}
const seg = readJson<IsoIntroSegmentJson>(filePath);
proseDocs.push({
book: "iso",
volume: NO_VOLUME,
chapter: null,
title: seg.title,
type: seg.type || "intro-segment",
slug: `intro-${seg.slug}`,
order: seg.order,
paragraphs: seg.paragraphs,
});
}
console.log(` 📄 Intro segments prepared: ${proseDocs.length}`);
console.log(` Inserting ${proseDocs.length} prose documents...`);
await insertBatched(proseCol, proseDocs);
const totalProse = await proseCol.countDocuments({ book: "iso" });
console.log(` ✅ ISO prose: ${totalProse} entries in MongoDB`);
}
// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------
async function main() {
console.log("🌱 Seeding Śrī Īśopaniṣad...\n");
const client = new MongoClient(DATABASE_URL!);
await client.connect();
console.log("📡 Connected to MongoDB\n");
const db = client.db();
const start = Date.now();
await seedIso(db);
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
console.log(`\n🎉 Done in ${elapsed}s`);
await client.close();
}
main().catch((err) => {
console.error("❌ Seed failed:", err);
process.exit(1);
});
scripts/seed-mg.ts
+204
View File
@@ -0,0 +1,204 @@
/**
* seed-mg.ts — Push Kṛṣṇa Consciousness: The Matchless Gift JSON data into MongoDB.
*
* Uses the native MongoDB driver (same pattern as seed-ek.ts) to avoid
* Prisma replica-set write requirements.
*
* MG is a prose book (like EK) with no volume structure.
* We use volume=0 as the sentinel for "no volumes".
*
* Collections written:
* - book_meta → one doc { book: "mg", data: <intro.json> }
* - prose_chapters → 1 intro segment + 7 chapters = 8 docs
*
* Usage:
* npx tsx scripts/seed-mg.ts
*/
import dotenv from "dotenv";
import fs from "fs";
import path from "path";
import { MongoClient } from "mongodb";
// Load env
dotenv.config({ path: ".env.local" });
dotenv.config({ path: ".env" });
const DATABASE_URL = process.env.DATABASE_URL;
if (!DATABASE_URL) {
console.error("❌ DATABASE_URL is not set in .env");
process.exit(1);
}
const BASE_DIR = path.resolve(__dirname, "..");
const MG_DIR = path.join(BASE_DIR, "lib", "jsons", "books", "mg");
/** Volume sentinel for books without volumes. */
const NO_VOLUME = 0;
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function readJson<T>(filePath: string): T {
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
}
/**
* Insert documents one at a time with a small delay every N docs.
* Uses insertOne to avoid BSON wire-protocol corruption.
*/
async function insertBatched(
col: ReturnType<ReturnType<MongoClient["db"]>["collection"]>,
docs: Record<string, unknown>[],
) {
for (let i = 0; i < docs.length; i++) {
await col.insertOne(docs[i]);
if ((i + 1) % 100 === 0) {
await new Promise((r) => setTimeout(r, 100));
}
}
}
// ---------------------------------------------------------------------------
// Types for the JSON files
// ---------------------------------------------------------------------------
type MgIntroJson = {
book: string;
title: string;
author: string;
year: number;
description: string;
totalChapters: number;
chapters: { number: number; title: string }[];
};
type MgIntroSegmentJson = {
slug: string;
title: string;
type: string;
book: string;
order: number;
paragraphs: string[];
};
type MgChapterJson = {
book: string;
chapter: number;
title: string;
type: string;
paragraphs: string[];
};
// ---------------------------------------------------------------------------
// Seed function
// ---------------------------------------------------------------------------
async function seedMg(db: ReturnType<MongoClient["db"]>) {
const introPath = path.join(MG_DIR, "intro.json");
if (!fs.existsSync(introPath)) {
console.error("❌ MG intro.json not found at", introPath);
process.exit(1);
}
const intro = readJson<MgIntroJson>(introPath);
// ---- 1. Book meta ----
const metaCol = db.collection("book_meta");
await metaCol.deleteMany({ book: "mg" });
await metaCol.insertOne({ book: "mg", data: intro });
console.log(" 📖 MG book meta inserted");
// ---- 2. Prose chapters collection ----
const proseCol = db.collection("prose_chapters");
await proseCol.deleteMany({ book: "mg" });
const docs: Record<string, unknown>[] = [];
// Intro segments
const introSegmentFiles = ["front-pages.json"];
const introDir = path.join(MG_DIR, "intro");
for (const file of introSegmentFiles) {
const filePath = path.join(introDir, file);
if (!fs.existsSync(filePath)) {
console.warn(` ⚠️ Intro segment not found, skipping: ${file}`);
continue;
}
const seg = readJson<MgIntroSegmentJson>(filePath);
docs.push({
book: "mg",
volume: NO_VOLUME,
chapter: null,
title: seg.title,
type: "intro-segment",
slug: `intro-${seg.slug}`,
order: seg.order,
paragraphs: seg.paragraphs,
});
}
console.log(` 📄 Intro segments: ${docs.length}`);
// Chapters (01.json through 07.json)
const chaptersDir = path.join(MG_DIR, "chapters");
if (fs.existsSync(chaptersDir)) {
const chapterFiles = fs
.readdirSync(chaptersDir)
.filter((f) => f.endsWith(".json"))
.sort();
for (const file of chapterFiles) {
const ch = readJson<MgChapterJson>(path.join(chaptersDir, file));
const chNum = ch.chapter ?? parseInt(file.replace(".json", ""), 10);
docs.push({
book: "mg",
volume: NO_VOLUME,
chapter: chNum,
title: ch.title,
type: "chapter",
slug: String(chNum),
order: 100 + chNum,
paragraphs: ch.paragraphs,
});
}
console.log(` 📖 Chapters: ${chapterFiles.length}`);
}
// Insert all docs
console.log(`\n Inserting ${docs.length} documents...`);
await insertBatched(proseCol, docs);
const totalDocs = await proseCol.countDocuments({ book: "mg" });
console.log(` ✅ MG: ${totalDocs} prose entries in MongoDB`);
}
// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------
async function main() {
console.log("🌱 Seeding Kṛṣṇa Consciousness: The Matchless Gift...\n");
const client = new MongoClient(DATABASE_URL!);
await client.connect();
console.log("📡 Connected to MongoDB\n");
const db = client.db();
const start = Date.now();
await seedMg(db);
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
console.log(`\n🎉 Done in ${elapsed}s`);
await client.close();
}
main().catch((err) => {
console.error("❌ Seed failed:", err);
process.exit(1);
});
scripts/seed-nod.ts
+222
View File
@@ -0,0 +1,222 @@
/**
* seed-nod.ts — Push The Nectar of Devotion JSON data into MongoDB.
*
* Uses the native MongoDB driver (same pattern as seed-db.ts) to avoid
* Prisma replica-set write requirements.
*
* NoD is a prose book (like KB) with no volume structure.
* We use volume=0 as the sentinel for "no volumes".
*
* Collections written:
* - book_meta → one doc { book: "nod", data: <intro.json> }
* - prose_chapters → intro segments + 51 chapters + concluding words
*
* Usage:
* npx tsx scripts/seed-nod.ts
*/
import dotenv from "dotenv";
import fs from "fs";
import path from "path";
import { MongoClient } from "mongodb";
// Load env
dotenv.config({ path: ".env.local" });
dotenv.config({ path: ".env" });
const DATABASE_URL = process.env.DATABASE_URL;
if (!DATABASE_URL) {
console.error("❌ DATABASE_URL is not set in .env");
process.exit(1);
}
const BASE_DIR = path.resolve(__dirname, "..");
const NOD_DIR = path.join(BASE_DIR, "lib", "jsons", "books", "nod");
/** Volume sentinel for books without volumes. */
const NO_VOLUME = 0;
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function readJson<T>(filePath: string): T {
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
}
/**
* Insert documents one at a time with a small delay every N docs.
* Uses insertOne to avoid BSON wire-protocol corruption.
*/
async function insertBatched(
col: ReturnType<ReturnType<MongoClient["db"]>["collection"]>,
docs: Record<string, unknown>[],
) {
for (let i = 0; i < docs.length; i++) {
await col.insertOne(docs[i]);
if ((i + 1) % 100 === 0) {
await new Promise((r) => setTimeout(r, 100));
}
}
}
// ---------------------------------------------------------------------------
// Types for the JSON files
// ---------------------------------------------------------------------------
type NodIntroJson = {
book: string;
title: string;
subtitle: string;
author: string;
year: number;
description: string;
totalChapters: number;
parts: { number: number; title: string; chapters: number[] }[];
chapters: { number: number; title: string }[];
};
type NodIntroSegmentJson = {
slug: string;
title: string;
type: string;
book: string;
order: number;
paragraphs: string[];
};
type NodChapterJson = {
book: string;
chapter: number | null;
title: string;
type: string;
slug?: string;
paragraphs: string[];
};
// ---------------------------------------------------------------------------
// Seed function
// ---------------------------------------------------------------------------
async function seedNod(db: ReturnType<MongoClient["db"]>) {
const introPath = path.join(NOD_DIR, "intro.json");
if (!fs.existsSync(introPath)) {
console.error("❌ NoD intro.json not found at", introPath);
process.exit(1);
}
const intro = readJson<NodIntroJson>(introPath);
// ---- Book meta ----
const metaCol = db.collection("book_meta");
await metaCol.deleteMany({ book: "nod" });
await metaCol.insertOne({ book: "nod", data: intro });
console.log(" 📖 NoD book meta inserted");
// ---- Prose chapters collection ----
const proseCol = db.collection("prose_chapters");
await proseCol.deleteMany({ book: "nod" });
const docs: Record<string, unknown>[] = [];
// Intro segments
const introDir = path.join(NOD_DIR, "intro");
if (fs.existsSync(introDir)) {
const introFiles = fs
.readdirSync(introDir)
.filter((f) => f.endsWith(".json"))
.sort();
for (const file of introFiles) {
const seg = readJson<NodIntroSegmentJson>(path.join(introDir, file));
docs.push({
book: "nod",
volume: NO_VOLUME,
chapter: null,
title: seg.title,
type: seg.type || "intro-segment",
slug: `intro-${seg.slug || file.replace(".json", "")}`,
order: seg.order ?? 0,
paragraphs: seg.paragraphs,
});
}
console.log(` 📄 Intro segments: ${introFiles.length}`);
}
// Chapters (01.json – 51.json)
const chaptersDir = path.join(NOD_DIR, "chapters");
if (fs.existsSync(chaptersDir)) {
const chapterFiles = fs
.readdirSync(chaptersDir)
.filter((f) => f.endsWith(".json") && f !== "concluding-words.json")
.sort();
for (const file of chapterFiles) {
const ch = readJson<NodChapterJson>(path.join(chaptersDir, file));
const chNum = ch.chapter ?? parseInt(file.replace(".json", ""), 10);
docs.push({
book: "nod",
volume: NO_VOLUME,
chapter: chNum,
title: ch.title,
type: ch.type || "chapter",
slug: String(chNum),
order: 100 + chNum,
paragraphs: ch.paragraphs,
});
}
console.log(` 📖 Chapters: ${chapterFiles.length}`);
// Concluding words
const concludingPath = path.join(chaptersDir, "concluding-words.json");
if (fs.existsSync(concludingPath)) {
const cw = readJson<NodChapterJson>(concludingPath);
docs.push({
book: "nod",
volume: NO_VOLUME,
chapter: null,
title: cw.title,
type: cw.type || "conclusion",
slug: "concluding-words",
order: 9999,
paragraphs: cw.paragraphs,
});
console.log(" 📝 Concluding words: 1");
}
}
// Insert all docs
console.log(`\n Inserting ${docs.length} documents...`);
await insertBatched(proseCol, docs);
const totalDocs = await proseCol.countDocuments({ book: "nod" });
console.log(` ✅ NoD: ${totalDocs} prose entries in MongoDB`);
}
// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------
async function main() {
console.log("🌱 Seeding The Nectar of Devotion...\n");
const client = new MongoClient(DATABASE_URL!);
await client.connect();
console.log("📡 Connected to MongoDB\n");
const db = client.db();
const start = Date.now();
await seedNod(db);
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
console.log(`\n🎉 Done in ${elapsed}s`);
await client.close();
}
main().catch((err) => {
console.error("❌ Seed failed:", err);
process.exit(1);
});
scripts/seed-noi.ts
+233
View File
@@ -0,0 +1,233 @@
/**
* seed-noi.ts — Push The Nectar of Instruction JSON data into MongoDB.
*
* Uses the native MongoDB driver (same pattern as seed-nod.ts) to avoid
* Prisma replica-set write requirements.
*
* NoI is a verse book (like ISO) with prose intro segments.
* We use volume=0 as the sentinel for "no volumes" on the prose side.
*
* Collections written:
* - book_meta → one doc { book: "noi", data: <intro.json> }
* - verses → 11 verse docs (Texts 1-11)
* - prose_chapters → 3 intro segment docs (front-pages, preface, appendixes)
*
* Usage:
* npx tsx scripts/seed-noi.ts
*/
import dotenv from "dotenv";
import fs from "fs";
import path from "path";
import { MongoClient } from "mongodb";
// Load env
dotenv.config({ path: ".env.local" });
dotenv.config({ path: ".env" });
const DATABASE_URL = process.env.DATABASE_URL;
if (!DATABASE_URL) {
console.error("❌ DATABASE_URL is not set in .env");
process.exit(1);
}
const BASE_DIR = path.resolve(__dirname, "..");
const NOI_DIR = path.join(BASE_DIR, "lib", "jsons", "books", "noi");
/** Volume sentinel for books without volumes. */
const NO_VOLUME = 0;
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function readJson<T>(filePath: string): T {
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
}
/**
* Insert documents one at a time with a small delay every N docs.
* Uses insertOne to avoid BSON wire-protocol corruption.
*/
async function insertBatched(
col: ReturnType<ReturnType<MongoClient["db"]>["collection"]>,
docs: Record<string, unknown>[],
) {
for (let i = 0; i < docs.length; i++) {
await col.insertOne(docs[i]);
if ((i + 1) % 100 === 0) {
await new Promise((r) => setTimeout(r, 100));
}
}
}
// ---------------------------------------------------------------------------
// Types for the JSON files
// ---------------------------------------------------------------------------
type NoiIntroJson = {
book: string;
title: string;
subtitle: string;
author: string;
year: number;
description: string;
totalTexts: number;
texts: { number: number; title: string; slug: string }[];
};
type NoiVerseJson = {
book: string;
book_title: string;
text: number;
title: string;
reference: string;
devanagari: string;
transliteration: string;
synonyms: { word: string; meanings: string[] }[];
translation: string;
purport: string[];
};
type NoiIntroSegmentJson = {
slug: string;
title: string;
type: string;
book: string;
order: number;
paragraphs: string[];
};
// ---------------------------------------------------------------------------
// Seed function
// ---------------------------------------------------------------------------
async function seedNoi(db: ReturnType<MongoClient["db"]>) {
const introPath = path.join(NOI_DIR, "intro.json");
if (!fs.existsSync(introPath)) {
console.error("❌ NoI intro.json not found at", introPath);
process.exit(1);
}
const intro = readJson<NoiIntroJson>(introPath);
// ---- 1. Book meta ----
const metaCol = db.collection("book_meta");
await metaCol.deleteMany({ book: "noi" });
await metaCol.insertOne({ book: "noi", data: intro });
console.log(" 📖 NoI book meta inserted");
// ---- 2. Verses ----
const versesCol = db.collection("verses");
await versesCol.deleteMany({ book: "noi" });
const verseDocs: Record<string, unknown>[] = [];
// Verse files: 01.json through 11.json
const verseFiles = Array.from({ length: 11 }, (_, i) =>
String(i + 1).padStart(2, "0") + ".json"
);
const versesDir = path.join(NOI_DIR, "verses");
for (const file of verseFiles) {
const filePath = path.join(versesDir, file);
if (!fs.existsSync(filePath)) {
console.warn(` ⚠️ Verse file not found, skipping: ${file}`);
continue;
}
const v = readJson<NoiVerseJson>(filePath);
verseDocs.push({
book: "noi",
division: null,
chapter: v.text,
verse: 1,
verseEnd: null,
reference: v.reference,
bookTitle: v.book_title,
devanagari: v.devanagari,
bengali: null,
transliteration: v.transliteration,
synonyms: v.synonyms,
translation: v.translation,
purport: v.purport,
});
}
console.log(` 📜 Verses prepared: ${verseDocs.length}`);
console.log(` Inserting ${verseDocs.length} verse documents...`);
await insertBatched(versesCol, verseDocs);
const totalVerses = await versesCol.countDocuments({ book: "noi" });
console.log(` ✅ NoI verses: ${totalVerses} in MongoDB`);
// ---- 3. Prose intro segments ----
const proseCol = db.collection("prose_chapters");
await proseCol.deleteMany({ book: "noi" });
const proseDocs: Record<string, unknown>[] = [];
const introSegmentFiles = [
"front-pages.json",
"preface.json",
"appendixes.json",
];
const introDir = path.join(NOI_DIR, "intro");
for (const file of introSegmentFiles) {
const filePath = path.join(introDir, file);
if (!fs.existsSync(filePath)) {
console.warn(` ⚠️ Intro segment not found, skipping: ${file}`);
continue;
}
const seg = readJson<NoiIntroSegmentJson>(filePath);
proseDocs.push({
book: "noi",
volume: NO_VOLUME,
chapter: null,
title: seg.title,
type: seg.type || "intro-segment",
slug: `intro-${seg.slug}`,
order: seg.order,
paragraphs: seg.paragraphs,
});
}
console.log(` 📄 Intro segments prepared: ${proseDocs.length}`);
console.log(` Inserting ${proseDocs.length} prose documents...`);
await insertBatched(proseCol, proseDocs);
const totalProse = await proseCol.countDocuments({ book: "noi" });
console.log(` ✅ NoI prose: ${totalProse} entries in MongoDB`);
}
// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------
async function main() {
console.log("🌱 Seeding The Nectar of Instruction...\n");
const client = new MongoClient(DATABASE_URL!);
await client.connect();
console.log("📡 Connected to MongoDB\n");
const db = client.db();
const start = Date.now();
await seedNoi(db);
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
console.log(`\n🎉 Done in ${elapsed}s`);
await client.close();
}
main().catch((err) => {
console.error("❌ Seed failed:", err);
process.exit(1);
});
scripts/seed-ochkm.ts
+204
View File
@@ -0,0 +1,204 @@
/**
* seed-ochkm.ts — Push On Chanting the Hare Kṛṣṇa Mantra JSON data into MongoDB.
*
* Uses the native MongoDB driver (same pattern as seed-poy.ts) to avoid
* Prisma replica-set write requirements.
*
* OCHKM is a prose book (like POY) with no volume structure.
* We use volume=0 as the sentinel for "no volumes".
*
* Collections written:
* - book_meta → one doc { book: "ochkm", data: <intro.json> }
* - prose_chapters → 1 intro segment + 4 chapters = 5 docs
*
* Usage:
* npx tsx scripts/seed-ochkm.ts
*/
import dotenv from "dotenv";
import fs from "fs";
import path from "path";
import { MongoClient } from "mongodb";
// Load env
dotenv.config({ path: ".env.local" });
dotenv.config({ path: ".env" });
const DATABASE_URL = process.env.DATABASE_URL;
if (!DATABASE_URL) {
console.error("❌ DATABASE_URL is not set in .env");
process.exit(1);
}
const BASE_DIR = path.resolve(__dirname, "..");
const OCHKM_DIR = path.join(BASE_DIR, "lib", "jsons", "books", "ochkm");
/** Volume sentinel for books without volumes. */
const NO_VOLUME = 0;
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function readJson<T>(filePath: string): T {
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
}
/**
* Insert documents one at a time with a small delay every N docs.
* Uses insertOne to avoid BSON wire-protocol corruption.
*/
async function insertBatched(
col: ReturnType<ReturnType<MongoClient["db"]>["collection"]>,
docs: Record<string, unknown>[],
) {
for (let i = 0; i < docs.length; i++) {
await col.insertOne(docs[i]);
if ((i + 1) % 100 === 0) {
await new Promise((r) => setTimeout(r, 100));
}
}
}
// ---------------------------------------------------------------------------
// Types for the JSON files
// ---------------------------------------------------------------------------
type OchkmIntroJson = {
book: string;
title: string;
author: string;
year: number;
description: string;
totalChapters: number;
chapters: { number: number; title: string }[];
};
type OchkmIntroSegmentJson = {
slug: string;
title: string;
type: string;
book: string;
order: number;
paragraphs: string[];
};
type OchkmChapterJson = {
book: string;
chapter: number;
title: string;
type: string;
paragraphs: string[];
};
// ---------------------------------------------------------------------------
// Seed function
// ---------------------------------------------------------------------------
async function seedOchkm(db: ReturnType<MongoClient["db"]>) {
const introPath = path.join(OCHKM_DIR, "intro.json");
if (!fs.existsSync(introPath)) {
console.error("❌ OCHKM intro.json not found at", introPath);
process.exit(1);
}
const intro = readJson<OchkmIntroJson>(introPath);
// ---- 1. Book meta ----
const metaCol = db.collection("book_meta");
await metaCol.deleteMany({ book: "ochkm" });
await metaCol.insertOne({ book: "ochkm", data: intro });
console.log(" 📖 OCHKM book meta inserted");
// ---- 2. Prose chapters collection ----
const proseCol = db.collection("prose_chapters");
await proseCol.deleteMany({ book: "ochkm" });
const docs: Record<string, unknown>[] = [];
// Intro segments
const introSegmentFiles = ["front-pages.json"];
const introDir = path.join(OCHKM_DIR, "intro");
for (const file of introSegmentFiles) {
const filePath = path.join(introDir, file);
if (!fs.existsSync(filePath)) {
console.warn(` ⚠️ Intro segment not found, skipping: ${file}`);
continue;
}
const seg = readJson<OchkmIntroSegmentJson>(filePath);
docs.push({
book: "ochkm",
volume: NO_VOLUME,
chapter: null,
title: seg.title,
type: "intro-segment",
slug: `intro-${seg.slug}`,
order: seg.order,
paragraphs: seg.paragraphs,
});
}
console.log(` 📄 Intro segments: ${docs.length}`);
// Chapters (01.json through 04.json)
const chaptersDir = path.join(OCHKM_DIR, "chapters");
if (fs.existsSync(chaptersDir)) {
const chapterFiles = fs
.readdirSync(chaptersDir)
.filter((f) => f.endsWith(".json"))
.sort();
for (const file of chapterFiles) {
const ch = readJson<OchkmChapterJson>(path.join(chaptersDir, file));
const chNum = ch.chapter ?? parseInt(file.replace(".json", ""), 10);
docs.push({
book: "ochkm",
volume: NO_VOLUME,
chapter: chNum,
title: ch.title,
type: "chapter",
slug: String(chNum),
order: 100 + chNum,
paragraphs: ch.paragraphs,
});
}
console.log(` 📖 Chapters: ${chapterFiles.length}`);
}
// Insert all docs
console.log(`\n Inserting ${docs.length} documents...`);
await insertBatched(proseCol, docs);
const totalDocs = await proseCol.countDocuments({ book: "ochkm" });
console.log(` ✅ OCHKM: ${totalDocs} prose entries in MongoDB`);
}
// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------
async function main() {
console.log("🌱 Seeding On Chanting the Hare Kṛṣṇa Mantra...\n");
const client = new MongoClient(DATABASE_URL!);
await client.connect();
console.log("📡 Connected to MongoDB\n");
const db = client.db();
const start = Date.now();
await seedOchkm(db);
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
console.log(`\n🎉 Done in ${elapsed}s`);
await client.close();
}
main().catch((err) => {
console.error("❌ Seed failed:", err);
process.exit(1);
});
scripts/seed-owk.ts
+204
View File
@@ -0,0 +1,204 @@
/**
* seed-owk.ts — Push On the Way to Kṛṣṇa JSON data into MongoDB.
*
* Uses the native MongoDB driver (same pattern as seed-bbd.ts) to avoid
* Prisma replica-set write requirements.
*
* OWK is a prose book (like BBD) with no volume structure.
* We use volume=0 as the sentinel for "no volumes".
*
* Collections written:
* - book_meta → one doc { book: "owk", data: <intro.json> }
* - prose_chapters → 1 intro segment + 5 chapters = 6 docs
*
* Usage:
* npx tsx scripts/seed-owk.ts
*/
import dotenv from "dotenv";
import fs from "fs";
import path from "path";
import { MongoClient } from "mongodb";
// Load env
dotenv.config({ path: ".env.local" });
dotenv.config({ path: ".env" });
const DATABASE_URL = process.env.DATABASE_URL;
if (!DATABASE_URL) {
console.error("❌ DATABASE_URL is not set in .env");
process.exit(1);
}
const BASE_DIR = path.resolve(__dirname, "..");
const OWK_DIR = path.join(BASE_DIR, "lib", "jsons", "books", "owk");
/** Volume sentinel for books without volumes. */
const NO_VOLUME = 0;
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function readJson<T>(filePath: string): T {
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
}
/**
* Insert documents one at a time with a small delay every N docs.
* Uses insertOne to avoid BSON wire-protocol corruption.
*/
async function insertBatched(
col: ReturnType<ReturnType<MongoClient["db"]>["collection"]>,
docs: Record<string, unknown>[],
) {
for (let i = 0; i < docs.length; i++) {
await col.insertOne(docs[i]);
if ((i + 1) % 100 === 0) {
await new Promise((r) => setTimeout(r, 100));
}
}
}
// ---------------------------------------------------------------------------
// Types for the JSON files
// ---------------------------------------------------------------------------
type OwkIntroJson = {
book: string;
title: string;
author: string;
year: number;
description: string;
totalChapters: number;
chapters: { number: number; title: string }[];
};
type OwkIntroSegmentJson = {
slug: string;
title: string;
type: string;
book: string;
order: number;
paragraphs: string[];
};
type OwkChapterJson = {
book: string;
chapter: number;
title: string;
type: string;
paragraphs: string[];
};
// ---------------------------------------------------------------------------
// Seed function
// ---------------------------------------------------------------------------
async function seedOwk(db: ReturnType<MongoClient["db"]>) {
const introPath = path.join(OWK_DIR, "intro.json");
if (!fs.existsSync(introPath)) {
console.error("❌ OWK intro.json not found at", introPath);
process.exit(1);
}
const intro = readJson<OwkIntroJson>(introPath);
// ---- 1. Book meta ----
const metaCol = db.collection("book_meta");
await metaCol.deleteMany({ book: "owk" });
await metaCol.insertOne({ book: "owk", data: intro });
console.log(" 📖 OWK book meta inserted");
// ---- 2. Prose chapters collection ----
const proseCol = db.collection("prose_chapters");
await proseCol.deleteMany({ book: "owk" });
const docs: Record<string, unknown>[] = [];
// Intro segments
const introSegmentFiles = ["front-pages.json"];
const introDir = path.join(OWK_DIR, "intro");
for (const file of introSegmentFiles) {
const filePath = path.join(introDir, file);
if (!fs.existsSync(filePath)) {
console.warn(` ⚠️ Intro segment not found, skipping: ${file}`);
continue;
}
const seg = readJson<OwkIntroSegmentJson>(filePath);
docs.push({
book: "owk",
volume: NO_VOLUME,
chapter: null,
title: seg.title,
type: "intro-segment",
slug: `intro-${seg.slug}`,
order: seg.order,
paragraphs: seg.paragraphs,
});
}
console.log(` 📄 Intro segments: ${docs.length}`);
// Chapters (01.json through 05.json)
const chaptersDir = path.join(OWK_DIR, "chapters");
if (fs.existsSync(chaptersDir)) {
const chapterFiles = fs
.readdirSync(chaptersDir)
.filter((f) => f.endsWith(".json"))
.sort();
for (const file of chapterFiles) {
const ch = readJson<OwkChapterJson>(path.join(chaptersDir, file));
const chNum = ch.chapter ?? parseInt(file.replace(".json", ""), 10);
docs.push({
book: "owk",
volume: NO_VOLUME,
chapter: chNum,
title: ch.title,
type: "chapter",
slug: String(chNum),
order: 100 + chNum,
paragraphs: ch.paragraphs,
});
}
console.log(` 📖 Chapters: ${chapterFiles.length}`);
}
// Insert all docs
console.log(`\n Inserting ${docs.length} documents...`);
await insertBatched(proseCol, docs);
const totalDocs = await proseCol.countDocuments({ book: "owk" });
console.log(` ✅ OWK: ${totalDocs} prose entries in MongoDB`);
}
// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------
async function main() {
console.log("🌱 Seeding On the Way to Kṛṣṇa...\n");
const client = new MongoClient(DATABASE_URL!);
await client.connect();
console.log("📡 Connected to MongoDB\n");
const db = client.db();
const start = Date.now();
await seedOwk(db);
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
console.log(`\n🎉 Done in ${elapsed}s`);
await client.close();
}
main().catch((err) => {
console.error("❌ Seed failed:", err);
process.exit(1);
});
scripts/seed-poy.ts
+204
View File
@@ -0,0 +1,204 @@
/**
* seed-poy.ts — Push The Perfection of Yoga JSON data into MongoDB.
*
* Uses the native MongoDB driver (same pattern as seed-bbd.ts) to avoid
* Prisma replica-set write requirements.
*
* POY is a prose book (like BBD) with no volume structure.
* We use volume=0 as the sentinel for "no volumes".
*
* Collections written:
* - book_meta → one doc { book: "poy", data: <intro.json> }
* - prose_chapters → 1 intro segment + 11 chapters = 12 docs
*
* Usage:
* npx tsx scripts/seed-poy.ts
*/
import dotenv from "dotenv";
import fs from "fs";
import path from "path";
import { MongoClient } from "mongodb";
// Load env
dotenv.config({ path: ".env.local" });
dotenv.config({ path: ".env" });
const DATABASE_URL = process.env.DATABASE_URL;
if (!DATABASE_URL) {
console.error("❌ DATABASE_URL is not set in .env");
process.exit(1);
}
const BASE_DIR = path.resolve(__dirname, "..");
const POY_DIR = path.join(BASE_DIR, "lib", "jsons", "books", "poy");
/** Volume sentinel for books without volumes. */
const NO_VOLUME = 0;
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function readJson<T>(filePath: string): T {
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
}
/**
* Insert documents one at a time with a small delay every N docs.
* Uses insertOne to avoid BSON wire-protocol corruption.
*/
async function insertBatched(
col: ReturnType<ReturnType<MongoClient["db"]>["collection"]>,
docs: Record<string, unknown>[],
) {
for (let i = 0; i < docs.length; i++) {
await col.insertOne(docs[i]);
if ((i + 1) % 100 === 0) {
await new Promise((r) => setTimeout(r, 100));
}
}
}
// ---------------------------------------------------------------------------
// Types for the JSON files
// ---------------------------------------------------------------------------
type PoyIntroJson = {
book: string;
title: string;
author: string;
year: number;
description: string;
totalChapters: number;
chapters: { number: number; title: string }[];
};
type PoyIntroSegmentJson = {
slug: string;
title: string;
type: string;
book: string;
order: number;
paragraphs: string[];
};
type PoyChapterJson = {
book: string;
chapter: number;
title: string;
type: string;
paragraphs: string[];
};
// ---------------------------------------------------------------------------
// Seed function
// ---------------------------------------------------------------------------
async function seedPoy(db: ReturnType<MongoClient["db"]>) {
const introPath = path.join(POY_DIR, "intro.json");
if (!fs.existsSync(introPath)) {
console.error("❌ POY intro.json not found at", introPath);
process.exit(1);
}
const intro = readJson<PoyIntroJson>(introPath);
// ---- 1. Book meta ----
const metaCol = db.collection("book_meta");
await metaCol.deleteMany({ book: "poy" });
await metaCol.insertOne({ book: "poy", data: intro });
console.log(" 📖 POY book meta inserted");
// ---- 2. Prose chapters collection ----
const proseCol = db.collection("prose_chapters");
await proseCol.deleteMany({ book: "poy" });
const docs: Record<string, unknown>[] = [];
// Intro segments
const introSegmentFiles = ["front-pages.json"];
const introDir = path.join(POY_DIR, "intro");
for (const file of introSegmentFiles) {
const filePath = path.join(introDir, file);
if (!fs.existsSync(filePath)) {
console.warn(` ⚠️ Intro segment not found, skipping: ${file}`);
continue;
}
const seg = readJson<PoyIntroSegmentJson>(filePath);
docs.push({
book: "poy",
volume: NO_VOLUME,
chapter: null,
title: seg.title,
type: "intro-segment",
slug: `intro-${seg.slug}`,
order: seg.order,
paragraphs: seg.paragraphs,
});
}
console.log(` 📄 Intro segments: ${docs.length}`);
// Chapters (01.json through 11.json)
const chaptersDir = path.join(POY_DIR, "chapters");
if (fs.existsSync(chaptersDir)) {
const chapterFiles = fs
.readdirSync(chaptersDir)
.filter((f) => f.endsWith(".json"))
.sort();
for (const file of chapterFiles) {
const ch = readJson<PoyChapterJson>(path.join(chaptersDir, file));
const chNum = ch.chapter ?? parseInt(file.replace(".json", ""), 10);
docs.push({
book: "poy",
volume: NO_VOLUME,
chapter: chNum,
title: ch.title,
type: "chapter",
slug: String(chNum),
order: 100 + chNum,
paragraphs: ch.paragraphs,
});
}
console.log(` 📖 Chapters: ${chapterFiles.length}`);
}
// Insert all docs
console.log(`\n Inserting ${docs.length} documents...`);
await insertBatched(proseCol, docs);
const totalDocs = await proseCol.countDocuments({ book: "poy" });
console.log(` ✅ POY: ${totalDocs} prose entries in MongoDB`);
}
// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------
async function main() {
console.log("🌱 Seeding The Perfection of Yoga...\n");
const client = new MongoClient(DATABASE_URL!);
await client.connect();
console.log("📡 Connected to MongoDB\n");
const db = client.db();
const start = Date.now();
await seedPoy(db);
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
console.log(`\n🎉 Done in ${elapsed}s`);
await client.close();
}
main().catch((err) => {
console.error("❌ Seed failed:", err);
process.exit(1);
});
scripts/seed-pqpa.ts
+220
View File
@@ -0,0 +1,220 @@
/**
* seed-pqpa.ts — Push Perfect Questions Perfect Answers JSON data into MongoDB.
*
* Uses the native MongoDB driver (same pattern as seed-nod.ts) to avoid
* Prisma replica-set write requirements.
*
* PQPA is a prose/dialogue book with no volume structure.
* We use volume=0 as the sentinel for "no volumes".
*
* Collections written:
* - book_meta → one doc { book: "pqpa", data: <intro.json> }
* - prose_chapters → intro segments + 9 chapters + concluding words
*
* Usage:
* npx tsx scripts/seed-pqpa.ts
*/
import dotenv from "dotenv";
import fs from "fs";
import path from "path";
import { MongoClient } from "mongodb";
// Load env
dotenv.config({ path: ".env.local" });
dotenv.config({ path: ".env" });
const DATABASE_URL = process.env.DATABASE_URL;
if (!DATABASE_URL) {
console.error("❌ DATABASE_URL is not set in .env");
process.exit(1);
}
const BASE_DIR = path.resolve(__dirname, "..");
const PQPA_DIR = path.join(BASE_DIR, "lib", "jsons", "books", "pqpa");
/** Volume sentinel for books without volumes. */
const NO_VOLUME = 0;
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function readJson<T>(filePath: string): T {
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
}
/**
* Insert documents one at a time with a small delay every N docs.
* Uses insertOne to avoid BSON wire-protocol corruption.
*/
async function insertBatched(
col: ReturnType<ReturnType<MongoClient["db"]>["collection"]>,
docs: Record<string, unknown>[],
) {
for (let i = 0; i < docs.length; i++) {
await col.insertOne(docs[i]);
if ((i + 1) % 100 === 0) {
await new Promise((r) => setTimeout(r, 100));
}
}
}
// ---------------------------------------------------------------------------
// Types for the JSON files
// ---------------------------------------------------------------------------
type PqpaIntroJson = {
book: string;
title: string;
author: string;
year: number;
description: string;
totalChapters: number;
chapters: { number: number; title: string }[];
};
type PqpaIntroSegmentJson = {
slug: string;
title: string;
type: string;
book: string;
order: number;
paragraphs: string[];
};
type PqpaChapterJson = {
book: string;
chapter: number | null;
title: string;
type: string;
slug?: string;
paragraphs: string[];
};
// ---------------------------------------------------------------------------
// Seed function
// ---------------------------------------------------------------------------
async function seedPqpa(db: ReturnType<MongoClient["db"]>) {
const introPath = path.join(PQPA_DIR, "intro.json");
if (!fs.existsSync(introPath)) {
console.error("❌ PQPA intro.json not found at", introPath);
process.exit(1);
}
const intro = readJson<PqpaIntroJson>(introPath);
// ---- Book meta ----
const metaCol = db.collection("book_meta");
await metaCol.deleteMany({ book: "pqpa" });
await metaCol.insertOne({ book: "pqpa", data: intro });
console.log(" 📖 PQPA book meta inserted");
// ---- Prose chapters collection ----
const proseCol = db.collection("prose_chapters");
await proseCol.deleteMany({ book: "pqpa" });
const docs: Record<string, unknown>[] = [];
// Intro segments
const introDir = path.join(PQPA_DIR, "intro");
if (fs.existsSync(introDir)) {
const introFiles = fs
.readdirSync(introDir)
.filter((f) => f.endsWith(".json"))
.sort();
for (const file of introFiles) {
const seg = readJson<PqpaIntroSegmentJson>(path.join(introDir, file));
docs.push({
book: "pqpa",
volume: NO_VOLUME,
chapter: null,
title: seg.title,
type: seg.type || "intro-segment",
slug: `intro-${seg.slug || file.replace(".json", "")}`,
order: seg.order ?? 0,
paragraphs: seg.paragraphs,
});
}
console.log(` 📄 Intro segments: ${introFiles.length}`);
}
// Chapters (01.json – 09.json)
const chaptersDir = path.join(PQPA_DIR, "chapters");
if (fs.existsSync(chaptersDir)) {
const chapterFiles = fs
.readdirSync(chaptersDir)
.filter((f) => f.endsWith(".json") && f !== "concluding-words.json")
.sort();
for (const file of chapterFiles) {
const ch = readJson<PqpaChapterJson>(path.join(chaptersDir, file));
const chNum = ch.chapter ?? parseInt(file.replace(".json", ""), 10);
docs.push({
book: "pqpa",
volume: NO_VOLUME,
chapter: chNum,
title: ch.title,
type: ch.type || "chapter",
slug: String(chNum),
order: 100 + chNum,
paragraphs: ch.paragraphs,
});
}
console.log(` 📖 Chapters: ${chapterFiles.length}`);
// Concluding words
const concludingPath = path.join(chaptersDir, "concluding-words.json");
if (fs.existsSync(concludingPath)) {
const cw = readJson<PqpaChapterJson>(concludingPath);
docs.push({
book: "pqpa",
volume: NO_VOLUME,
chapter: null,
title: cw.title,
type: cw.type || "conclusion",
slug: "concluding-words",
order: 9999,
paragraphs: cw.paragraphs,
});
console.log(" 📝 Concluding words: 1");
}
}
// Insert all docs
console.log(`\n Inserting ${docs.length} documents...`);
await insertBatched(proseCol, docs);
const totalDocs = await proseCol.countDocuments({ book: "pqpa" });
console.log(` ✅ PQPA: ${totalDocs} prose entries in MongoDB`);
}
// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------
async function main() {
console.log("🌱 Seeding Perfect Questions Perfect Answers...\n");
const client = new MongoClient(DATABASE_URL!);
await client.connect();
console.log("📡 Connected to MongoDB\n");
const db = client.db();
const start = Date.now();
await seedPqpa(db);
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
console.log(`\n🎉 Done in ${elapsed}s`);
await client.close();
}
main().catch((err) => {
console.error("❌ Seed failed:", err);
process.exit(1);
});
scripts/seed-rv.ts
+204
View File
@@ -0,0 +1,204 @@
/**
* seed-rv.ts — Push Rāja-vidyā: The King of Knowledge JSON data into MongoDB.
*
* Uses the native MongoDB driver (same pattern as seed-bbd.ts) to avoid
* Prisma replica-set write requirements.
*
* RV is a prose book (like BBD) with no volume structure.
* We use volume=0 as the sentinel for "no volumes".
*
* Collections written:
* - book_meta → one doc { book: "rv", data: <intro.json> }
* - prose_chapters → 1 intro segment + 8 chapters = 9 docs
*
* Usage:
* npx tsx scripts/seed-rv.ts
*/
import dotenv from "dotenv";
import fs from "fs";
import path from "path";
import { MongoClient } from "mongodb";
// Load env
dotenv.config({ path: ".env.local" });
dotenv.config({ path: ".env" });
const DATABASE_URL = process.env.DATABASE_URL;
if (!DATABASE_URL) {
console.error("❌ DATABASE_URL is not set in .env");
process.exit(1);
}
const BASE_DIR = path.resolve(__dirname, "..");
const RV_DIR = path.join(BASE_DIR, "lib", "jsons", "books", "rv");
/** Volume sentinel for books without volumes. */
const NO_VOLUME = 0;
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function readJson<T>(filePath: string): T {
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
}
/**
* Insert documents one at a time with a small delay every N docs.
* Uses insertOne to avoid BSON wire-protocol corruption.
*/
async function insertBatched(
col: ReturnType<ReturnType<MongoClient["db"]>["collection"]>,
docs: Record<string, unknown>[],
) {
for (let i = 0; i < docs.length; i++) {
await col.insertOne(docs[i]);
if ((i + 1) % 100 === 0) {
await new Promise((r) => setTimeout(r, 100));
}
}
}
// ---------------------------------------------------------------------------
// Types for the JSON files
// ---------------------------------------------------------------------------
type RvIntroJson = {
book: string;
title: string;
author: string;
year: number;
description: string;
totalChapters: number;
chapters: { number: number; title: string }[];
};
type RvIntroSegmentJson = {
slug: string;
title: string;
type: string;
book: string;
order: number;
paragraphs: string[];
};
type RvChapterJson = {
book: string;
chapter: number;
title: string;
type: string;
paragraphs: string[];
};
// ---------------------------------------------------------------------------
// Seed function
// ---------------------------------------------------------------------------
async function seedRv(db: ReturnType<MongoClient["db"]>) {
const introPath = path.join(RV_DIR, "intro.json");
if (!fs.existsSync(introPath)) {
console.error("❌ RV intro.json not found at", introPath);
process.exit(1);
}
const intro = readJson<RvIntroJson>(introPath);
// ---- 1. Book meta ----
const metaCol = db.collection("book_meta");
await metaCol.deleteMany({ book: "rv" });
await metaCol.insertOne({ book: "rv", data: intro });
console.log(" 📖 RV book meta inserted");
// ---- 2. Prose chapters collection ----
const proseCol = db.collection("prose_chapters");
await proseCol.deleteMany({ book: "rv" });
const docs: Record<string, unknown>[] = [];
// Intro segments
const introSegmentFiles = ["front-pages.json"];
const introDir = path.join(RV_DIR, "intro");
for (const file of introSegmentFiles) {
const filePath = path.join(introDir, file);
if (!fs.existsSync(filePath)) {
console.warn(` ⚠️ Intro segment not found, skipping: ${file}`);
continue;
}
const seg = readJson<RvIntroSegmentJson>(filePath);
docs.push({
book: "rv",
volume: NO_VOLUME,
chapter: null,
title: seg.title,
type: "intro-segment",
slug: `intro-${seg.slug}`,
order: seg.order,
paragraphs: seg.paragraphs,
});
}
console.log(` 📄 Intro segments: ${docs.length}`);
// Chapters (01.json through 08.json)
const chaptersDir = path.join(RV_DIR, "chapters");
if (fs.existsSync(chaptersDir)) {
const chapterFiles = fs
.readdirSync(chaptersDir)
.filter((f) => f.endsWith(".json"))
.sort();
for (const file of chapterFiles) {
const ch = readJson<RvChapterJson>(path.join(chaptersDir, file));
const chNum = ch.chapter ?? parseInt(file.replace(".json", ""), 10);
docs.push({
book: "rv",
volume: NO_VOLUME,
chapter: chNum,
title: ch.title,
type: "chapter",
slug: String(chNum),
order: 100 + chNum,
paragraphs: ch.paragraphs,
});
}
console.log(` 📖 Chapters: ${chapterFiles.length}`);
}
// Insert all docs
console.log(`\n Inserting ${docs.length} documents...`);
await insertBatched(proseCol, docs);
const totalDocs = await proseCol.countDocuments({ book: "rv" });
console.log(` ✅ RV: ${totalDocs} prose entries in MongoDB`);
}
// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------
async function main() {
console.log("🌱 Seeding Rāja-vidyā: The King of Knowledge...\n");
const client = new MongoClient(DATABASE_URL!);
await client.connect();
console.log("📡 Connected to MongoDB\n");
const db = client.db();
const start = Date.now();
await seedRv(db);
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
console.log(`\n🎉 Done in ${elapsed}s`);
await client.close();
}
main().catch((err) => {
console.error("❌ Seed failed:", err);
process.exit(1);
});
scripts/seed-ssr.ts
+239
View File
@@ -0,0 +1,239 @@
/**
* seed-ssr.ts — Push The Science of Self-Realization JSON data into MongoDB.
*
* Uses the native MongoDB driver (same pattern as seed-hkh.ts) to avoid
* Prisma replica-set write requirements.
*
* SSR is a large prose book with no volume structure.
* We use volume=0 as the sentinel for "no volumes".
*
* Sub-chapters are stored with slugs like "1-1", "1-2", "2-1", etc.
* Each sub-chapter JSON has parentChapter and subChapter fields used to
* compute a stable ordering: parentChapter * 100 + subChapter.
*
* Collections written:
* - book_meta → one doc { book: "ssr", data: <intro.json> }
* - prose_chapters → 6 intro segments + ~50 sub-chapters
*
* Usage:
* npx tsx scripts/seed-ssr.ts
*/
import dotenv from "dotenv";
import fs from "fs";
import path from "path";
import { MongoClient } from "mongodb";
// Load env
dotenv.config({ path: ".env.local" });
dotenv.config({ path: ".env" });
const DATABASE_URL = process.env.DATABASE_URL;
if (!DATABASE_URL) {
console.error("❌ DATABASE_URL is not set in .env");
process.exit(1);
}
const BASE_DIR = path.resolve(__dirname, "..");
const SSR_DIR = path.join(BASE_DIR, "lib", "jsons", "books", "ssr");
/** Volume sentinel for books without volumes. */
const NO_VOLUME = 0;
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function readJson<T>(filePath: string): T {
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
}
/**
* Insert documents one at a time with a small delay every N docs.
* Uses insertOne to avoid BSON wire-protocol corruption.
*/
async function insertBatched(
col: ReturnType<ReturnType<MongoClient["db"]>["collection"]>,
docs: Record<string, unknown>[],
) {
for (let i = 0; i < docs.length; i++) {
await col.insertOne(docs[i]);
if ((i + 1) % 100 === 0) {
await new Promise((r) => setTimeout(r, 100));
}
}
}
// ---------------------------------------------------------------------------
// Types for the JSON files
// ---------------------------------------------------------------------------
type SsrIntroJson = {
book: string;
title: string;
author: string;
year: number;
description: string;
totalChapters: number;
chapters: {
number: number;
title: string;
subChapters: { slug: string; title: string }[];
}[];
};
type SsrIntroSegmentJson = {
slug: string;
title: string;
type: string;
book: string;
order: number;
paragraphs: string[];
};
type SsrChapterJson = {
book: string;
chapter: string; // "1-1", "2-3", etc.
parentChapter: number;
subChapter: number;
title: string;
parentTitle: string;
type: string;
paragraphs: string[];
};
// ---------------------------------------------------------------------------
// Seed function
// ---------------------------------------------------------------------------
async function seedSsr(db: ReturnType<MongoClient["db"]>) {
const introPath = path.join(SSR_DIR, "intro.json");
if (!fs.existsSync(introPath)) {
console.error("❌ SSR intro.json not found at", introPath);
process.exit(1);
}
const intro = readJson<SsrIntroJson>(introPath);
// ---- 1. Book meta ----
const metaCol = db.collection("book_meta");
await metaCol.deleteMany({ book: "ssr" });
await metaCol.insertOne({ book: "ssr", data: intro });
console.log(" 📖 SSR book meta inserted");
// ---- 2. Prose chapters collection ----
const proseCol = db.collection("prose_chapters");
await proseCol.deleteMany({ book: "ssr" });
const docs: Record<string, unknown>[] = [];
// Intro segments (6 total)
const introSegmentFiles = [
"front-pages.json",
"dedication.json",
"foreword.json",
"introduction.json",
"appreciations.json",
"about-the-author.json",
];
const introDir = path.join(SSR_DIR, "intro");
for (const file of introSegmentFiles) {
const filePath = path.join(introDir, file);
if (!fs.existsSync(filePath)) {
console.warn(` ⚠️ Intro segment not found, skipping: ${file}`);
continue;
}
const seg = readJson<SsrIntroSegmentJson>(filePath);
docs.push({
book: "ssr",
volume: NO_VOLUME,
chapter: null,
title: seg.title,
type: "intro-segment",
slug: `intro-${seg.slug}`,
order: seg.order,
paragraphs: seg.paragraphs,
});
}
console.log(` 📄 Intro segments: ${docs.length}`);
// Sub-chapters (1-1.json, 1-2.json, 2-1.json, etc.)
const chaptersDir = path.join(SSR_DIR, "chapters");
let chapterCount = 0;
if (fs.existsSync(chaptersDir)) {
const chapterFiles = fs
.readdirSync(chaptersDir)
.filter((f) => f.endsWith(".json"))
.sort((a, b) => {
// Sort by parentChapter then subChapter numerically
const parseSlug = (name: string) => {
const slug = name.replace(".json", "");
const parts = slug.split("-");
return {
parent: parseInt(parts[0], 10),
sub: parseInt(parts[1], 10),
};
};
const aa = parseSlug(a);
const bb = parseSlug(b);
if (aa.parent !== bb.parent) return aa.parent - bb.parent;
return aa.sub - bb.sub;
});
for (const file of chapterFiles) {
const ch = readJson<SsrChapterJson>(path.join(chaptersDir, file));
const order = ch.parentChapter * 100 + ch.subChapter;
docs.push({
book: "ssr",
volume: NO_VOLUME,
chapter: null, // null since chapter is a string "1-1" not an int
title: ch.title,
type: "chapter",
slug: ch.chapter, // "1-1", "1-2", "2-1", etc.
order,
paragraphs: ch.paragraphs,
});
chapterCount++;
}
console.log(` 📖 Sub-chapters: ${chapterCount}`);
}
// Insert all docs
console.log(`\n Inserting ${docs.length} documents...`);
await insertBatched(proseCol, docs);
const totalDocs = await proseCol.countDocuments({ book: "ssr" });
console.log(` ✅ SSR: ${totalDocs} prose entries in MongoDB`);
}
// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------
async function main() {
console.log("🌱 Seeding The Science of Self-Realization...\n");
const client = new MongoClient(DATABASE_URL!);
await client.connect();
console.log("📡 Connected to MongoDB\n");
const db = client.db();
const start = Date.now();
await seedSsr(db);
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
console.log(`\n🎉 Done in ${elapsed}s`);
await client.close();
}
main().catch((err) => {
console.error("❌ Seed failed:", err);
process.exit(1);
});
scripts/seed-te.ts
+169
View File
@@ -0,0 +1,169 @@
/**
* seed-te.ts — Push Two Essays JSON data into MongoDB.
*
* Uses the native MongoDB driver (same pattern as seed-ej.ts) to avoid
* Prisma replica-set write requirements.
*
* Two Essays is a collection of 2 standalone essays using the NEW `essays`
* collection (not `prose_chapters`).
*
* Collections written:
* - book_meta → one doc { book: "te", data: <intro.json> }
* - essays → 2 essay docs
*
* Usage:
* npx tsx scripts/seed-te.ts
*/
import dotenv from "dotenv";
import fs from "fs";
import path from "path";
import { MongoClient } from "mongodb";
// Load env
dotenv.config({ path: ".env.local" });
dotenv.config({ path: ".env" });
const DATABASE_URL = process.env.DATABASE_URL;
if (!DATABASE_URL) {
console.error("❌ DATABASE_URL is not set in .env");
process.exit(1);
}
const BASE_DIR = path.resolve(__dirname, "..");
const TE_DIR = path.join(BASE_DIR, "lib", "jsons", "books", "te");
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function readJson<T>(filePath: string): T {
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
}
/**
* Insert documents one at a time with a small delay every N docs.
* Uses insertOne to avoid BSON wire-protocol corruption.
*/
async function insertBatched(
col: ReturnType<ReturnType<MongoClient["db"]>["collection"]>,
docs: Record<string, unknown>[],
) {
for (let i = 0; i < docs.length; i++) {
await col.insertOne(docs[i]);
if ((i + 1) % 100 === 0) {
await new Promise((r) => setTimeout(r, 100));
}
}
}
// ---------------------------------------------------------------------------
// Types for the JSON files
// ---------------------------------------------------------------------------
type TeIntroJson = {
group: string;
title: string;
author: string;
year: number;
description: string;
essays: { slug: string; title: string; order: number }[];
};
type TeEssayJson = {
slug: string;
title: string;
author: string;
year: number;
group: string;
order: number;
paragraphs: string[];
};
// ---------------------------------------------------------------------------
// Seed function
// ---------------------------------------------------------------------------
async function seedTe(db: ReturnType<MongoClient["db"]>) {
const introPath = path.join(TE_DIR, "intro.json");
if (!fs.existsSync(introPath)) {
console.error("❌ TE intro.json not found at", introPath);
process.exit(1);
}
const intro = readJson<TeIntroJson>(introPath);
// ---- 1. Book meta ----
const metaCol = db.collection("book_meta");
await metaCol.deleteMany({ book: "te" });
await metaCol.insertOne({ book: "te", data: intro });
console.log(" 📖 TE book meta inserted");
// ---- 2. Essays collection ----
const essaysCol = db.collection("essays");
await essaysCol.deleteMany({ group: "two-essays" });
const docs: Record<string, unknown>[] = [];
const essaysDir = path.join(TE_DIR, "essays");
if (!fs.existsSync(essaysDir)) {
console.error("❌ TE essays directory not found at", essaysDir);
process.exit(1);
}
const essayFiles = fs
.readdirSync(essaysDir)
.filter((f) => f.endsWith(".json"))
.sort();
for (const file of essayFiles) {
const essay = readJson<TeEssayJson>(path.join(essaysDir, file));
docs.push({
slug: essay.slug,
title: essay.title,
author: essay.author,
year: essay.year,
group: essay.group,
order: essay.order,
paragraphs: essay.paragraphs,
});
}
console.log(` 📝 Essays found: ${docs.length}`);
// Insert all docs
console.log(`\n Inserting ${docs.length} essay documents...`);
await insertBatched(essaysCol, docs);
const totalDocs = await essaysCol.countDocuments({ group: "two-essays" });
console.log(` ✅ TE: ${totalDocs} essays in MongoDB`);
}
// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------
async function main() {
console.log("🌱 Seeding Two Essays...\n");
const client = new MongoClient(DATABASE_URL!);
await client.connect();
console.log("📡 Connected to MongoDB\n");
const db = client.db();
const start = Date.now();
await seedTe(db);
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
console.log(`\n🎉 Done in ${elapsed}s`);
await client.close();
}
main().catch((err) => {
console.error("❌ Seed failed:", err);
process.exit(1);
});
scripts/seed-tlc.ts
+207
View File
@@ -0,0 +1,207 @@
/**
* seed-tlc.ts — Push Teachings of Lord Caitanya JSON data into MongoDB.
*
* Uses the native MongoDB driver (same pattern as seed-nod.ts) to avoid
* Prisma replica-set write requirements.
*
* TLC is a prose book (like NoD) with no volume structure.
* We use volume=0 as the sentinel for "no volumes".
*
* Collections written:
* - book_meta → one doc { book: "tlc", data: <intro.json> }
* - prose_chapters → 9 intro segments + 32 chapters
*
* Usage:
* npx tsx scripts/seed-tlc.ts
*/
import dotenv from "dotenv";
import fs from "fs";
import path from "path";
import { MongoClient } from "mongodb";
// Load env
dotenv.config({ path: ".env.local" });
dotenv.config({ path: ".env" });
const DATABASE_URL = process.env.DATABASE_URL;
if (!DATABASE_URL) {
console.error("❌ DATABASE_URL is not set in .env");
process.exit(1);
}
const BASE_DIR = path.resolve(__dirname, "..");
const TLC_DIR = path.join(BASE_DIR, "lib", "jsons", "books", "tlc");
/** Volume sentinel for books without volumes. */
const NO_VOLUME = 0;
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function readJson<T>(filePath: string): T {
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
}
/**
* Insert documents one at a time with a small delay every N docs.
* Uses insertOne to avoid BSON wire-protocol corruption.
*/
async function insertBatched(
col: ReturnType<ReturnType<MongoClient["db"]>["collection"]>,
docs: Record<string, unknown>[],
) {
for (let i = 0; i < docs.length; i++) {
await col.insertOne(docs[i]);
if ((i + 1) % 100 === 0) {
await new Promise((r) => setTimeout(r, 100));
}
}
}
// ---------------------------------------------------------------------------
// Types for the JSON files
// ---------------------------------------------------------------------------
type TlcIntroJson = {
book: string;
title: string;
subtitle: string;
author: string;
year: number;
description: string;
totalChapters: number;
parts: { number: number; title: string; chapters: number[] }[];
chapters: { number: number; title: string }[];
};
type TlcIntroSegmentJson = {
slug: string;
title: string;
type: string;
book: string;
order: number;
paragraphs: string[];
};
type TlcChapterJson = {
book: string;
chapter: number | null;
title: string;
type: string;
slug?: string;
paragraphs: string[];
};
// ---------------------------------------------------------------------------
// Seed function
// ---------------------------------------------------------------------------
async function seedTlc(db: ReturnType<MongoClient["db"]>) {
const introPath = path.join(TLC_DIR, "intro.json");
if (!fs.existsSync(introPath)) {
console.error("❌ TLC intro.json not found at", introPath);
process.exit(1);
}
const intro = readJson<TlcIntroJson>(introPath);
// ---- Book meta ----
const metaCol = db.collection("book_meta");
await metaCol.deleteMany({ book: "tlc" });
await metaCol.insertOne({ book: "tlc", data: intro });
console.log(" 📖 TLC book meta inserted");
// ---- Prose chapters collection ----
const proseCol = db.collection("prose_chapters");
await proseCol.deleteMany({ book: "tlc" });
const docs: Record<string, unknown>[] = [];
// Intro segments
const introDir = path.join(TLC_DIR, "intro");
if (fs.existsSync(introDir)) {
const introFiles = fs
.readdirSync(introDir)
.filter((f) => f.endsWith(".json"))
.sort();
let fallbackOrder = 0;
for (const file of introFiles) {
const seg = readJson<TlcIntroSegmentJson>(path.join(introDir, file));
fallbackOrder++;
docs.push({
book: "tlc",
volume: NO_VOLUME,
chapter: null,
title: seg.title,
type: seg.type || "intro-segment",
slug: `intro-${seg.slug || file.replace(".json", "")}`,
order: seg.order ?? fallbackOrder,
paragraphs: seg.paragraphs,
});
}
console.log(` 📄 Intro segments: ${introFiles.length}`);
}
// Chapters (01.json – 32.json)
const chaptersDir = path.join(TLC_DIR, "chapters");
if (fs.existsSync(chaptersDir)) {
const chapterFiles = fs
.readdirSync(chaptersDir)
.filter((f) => f.endsWith(".json"))
.sort();
for (const file of chapterFiles) {
const ch = readJson<TlcChapterJson>(path.join(chaptersDir, file));
const chNum = ch.chapter ?? parseInt(file.replace(".json", ""), 10);
docs.push({
book: "tlc",
volume: NO_VOLUME,
chapter: chNum,
title: ch.title,
type: ch.type || "chapter",
slug: String(chNum),
order: 100 + chNum,
paragraphs: ch.paragraphs,
});
}
console.log(` 📖 Chapters: ${chapterFiles.length}`);
}
// Insert all docs
console.log(`\n Inserting ${docs.length} documents...`);
await insertBatched(proseCol, docs);
const totalDocs = await proseCol.countDocuments({ book: "tlc" });
console.log(` ✅ TLC: ${totalDocs} prose entries in MongoDB`);
}
// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------
async function main() {
console.log("🌱 Seeding Teachings of Lord Caitanya...\n");
const client = new MongoClient(DATABASE_URL!);
await client.connect();
console.log("📡 Connected to MongoDB\n");
const db = client.db();
const start = Date.now();
await seedTlc(db);
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
console.log(`\n🎉 Done in ${elapsed}s`);
await client.close();
}
main().catch((err) => {
console.error("❌ Seed failed:", err);
process.exit(1);
});
scripts/seed-ttp.ts
+204
View File
@@ -0,0 +1,204 @@
/**
* seed-ttp.ts — Push Transcendental Teachings of Prahlāda Mahārāja JSON data into MongoDB.
*
* Uses the native MongoDB driver (same pattern as seed-poy.ts) to avoid
* Prisma replica-set write requirements.
*
* TTP is a prose book (like POY) with no volume structure.
* We use volume=0 as the sentinel for "no volumes".
*
* Collections written:
* - book_meta → one doc { book: "ttp", data: <intro.json> }
* - prose_chapters → 1 intro segment + 5 chapters = 6 docs
*
* Usage:
* npx tsx scripts/seed-ttp.ts
*/
import dotenv from "dotenv";
import fs from "fs";
import path from "path";
import { MongoClient } from "mongodb";
// Load env
dotenv.config({ path: ".env.local" });
dotenv.config({ path: ".env" });
const DATABASE_URL = process.env.DATABASE_URL;
if (!DATABASE_URL) {
console.error("❌ DATABASE_URL is not set in .env");
process.exit(1);
}
const BASE_DIR = path.resolve(__dirname, "..");
const TTP_DIR = path.join(BASE_DIR, "lib", "jsons", "books", "ttp");
/** Volume sentinel for books without volumes. */
const NO_VOLUME = 0;
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function readJson<T>(filePath: string): T {
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
}
/**
* Insert documents one at a time with a small delay every N docs.
* Uses insertOne to avoid BSON wire-protocol corruption.
*/
async function insertBatched(
col: ReturnType<ReturnType<MongoClient["db"]>["collection"]>,
docs: Record<string, unknown>[],
) {
for (let i = 0; i < docs.length; i++) {
await col.insertOne(docs[i]);
if ((i + 1) % 100 === 0) {
await new Promise((r) => setTimeout(r, 100));
}
}
}
// ---------------------------------------------------------------------------
// Types for the JSON files
// ---------------------------------------------------------------------------
type TtpIntroJson = {
book: string;
title: string;
author: string;
year: number;
description: string;
totalChapters: number;
chapters: { number: number; title: string }[];
};
type TtpIntroSegmentJson = {
slug: string;
title: string;
type: string;
book: string;
order: number;
paragraphs: string[];
};
type TtpChapterJson = {
book: string;
chapter: number;
title: string;
type: string;
paragraphs: string[];
};
// ---------------------------------------------------------------------------
// Seed function
// ---------------------------------------------------------------------------
async function seedTtp(db: ReturnType<MongoClient["db"]>) {
const introPath = path.join(TTP_DIR, "intro.json");
if (!fs.existsSync(introPath)) {
console.error("❌ TTP intro.json not found at", introPath);
process.exit(1);
}
const intro = readJson<TtpIntroJson>(introPath);
// ---- 1. Book meta ----
const metaCol = db.collection("book_meta");
await metaCol.deleteMany({ book: "ttp" });
await metaCol.insertOne({ book: "ttp", data: intro });
console.log(" 📖 TTP book meta inserted");
// ---- 2. Prose chapters collection ----
const proseCol = db.collection("prose_chapters");
await proseCol.deleteMany({ book: "ttp" });
const docs: Record<string, unknown>[] = [];
// Intro segments
const introSegmentFiles = ["front-pages.json"];
const introDir = path.join(TTP_DIR, "intro");
for (const file of introSegmentFiles) {
const filePath = path.join(introDir, file);
if (!fs.existsSync(filePath)) {
console.warn(` ⚠️ Intro segment not found, skipping: ${file}`);
continue;
}
const seg = readJson<TtpIntroSegmentJson>(filePath);
docs.push({
book: "ttp",
volume: NO_VOLUME,
chapter: null,
title: seg.title,
type: "intro-segment",
slug: `intro-${seg.slug}`,
order: seg.order,
paragraphs: seg.paragraphs,
});
}
console.log(` 📄 Intro segments: ${docs.length}`);
// Chapters (01.json through 05.json)
const chaptersDir = path.join(TTP_DIR, "chapters");
if (fs.existsSync(chaptersDir)) {
const chapterFiles = fs
.readdirSync(chaptersDir)
.filter((f) => f.endsWith(".json"))
.sort();
for (const file of chapterFiles) {
const ch = readJson<TtpChapterJson>(path.join(chaptersDir, file));
const chNum = ch.chapter ?? parseInt(file.replace(".json", ""), 10);
docs.push({
book: "ttp",
volume: NO_VOLUME,
chapter: chNum,
title: ch.title,
type: "chapter",
slug: String(chNum),
order: 100 + chNum,
paragraphs: ch.paragraphs,
});
}
console.log(` 📖 Chapters: ${chapterFiles.length}`);
}
// Insert all docs
console.log(`\n Inserting ${docs.length} documents...`);
await insertBatched(proseCol, docs);
const totalDocs = await proseCol.countDocuments({ book: "ttp" });
console.log(` ✅ TTP: ${totalDocs} prose entries in MongoDB`);
}
// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------
async function main() {
console.log("🌱 Seeding Transcendental Teachings of Prahlāda Mahārāja...\n");
const client = new MongoClient(DATABASE_URL!);
await client.connect();
console.log("📡 Connected to MongoDB\n");
const db = client.db();
const start = Date.now();
await seedTtp(db);
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
console.log(`\n🎉 Done in ${elapsed}s`);
await client.close();
}
main().catch((err) => {
console.error("❌ Seed failed:", err);
process.exit(1);
});
scripts/ssr_extract.py
+1071
View File
File diff suppressed because it is too large Load Diff
scripts/te_extract.py
+445
View File
@@ -0,0 +1,445 @@
#!/usr/bin/env python3
"""Extract Two Essays (1966) from other.html.
The book contains 2 essays:
1. "Krishna The Reservoir Of Pleasure"
2. "Who Is Crazy?"
These are plain English essays from 1966 — NO ScaGoudy encoding.
Output:
lib/jsons/books/te/intro.json — Book metadata
lib/jsons/books/te/essays/krishna-reservoir-of-pleasure.json
lib/jsons/books/te/essays/who-is-crazy.json
Usage:
python3 scripts/te_extract.py
"""
import json
import os
import re
import sys
from bs4 import BeautifulSoup, Tag
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def get_css_class(tag: Tag) -> str:
"""Get the first CSS class of a tag, or empty string."""
classes = tag.get("class", [])
return classes[0] if classes else ""
def is_empty(tag: Tag) -> bool:
"""Check if a tag is empty or whitespace-only."""
text = tag.get_text().strip()
return not text or text == "\xa0"
def clean_html_content(tag: Tag) -> str:
"""Extract inner HTML from a tag, strip span class artifacts, preserve <i>, <b>, <br>."""
inner = "".join(str(child) for child in tag.children)
# Remove span tags with CSS classes (RTF conversion artifacts)
inner = re.sub(
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
)
# Clean up Apple-tab-span leftovers (tab characters)
inner = inner.replace("\t", " ")
# Normalize whitespace (but preserve <br> tags)
inner = re.sub(r"\s+", " ", inner).strip()
return inner
def write_json(data, filepath):
"""Write data as formatted JSON to filepath, creating directories as needed."""
os.makedirs(os.path.dirname(filepath), exist_ok=True)
with open(filepath, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False, indent=2)
# ---------------------------------------------------------------------------
# CSS classes to SKIP entirely (ad block, structural headers, etc.)
# ---------------------------------------------------------------------------
SKIP_CLASSES = {"p12", "p121", "p122", "p13", "p123", "p124", "p23", "p18"}
# CSS classes that contain essay body content
BODY_CLASSES = {"p28", "p6", "p22", "p3"}
# ---------------------------------------------------------------------------
# Boundary detection
# ---------------------------------------------------------------------------
def find_te_boundaries(all_paragraphs: list[Tag]) -> tuple[int, int]:
"""Find the start and end paragraph indices for Two Essays content.
Start: <p class="p12"><b>TWO ESSAYS</b></p>
End: Next <p class="p12"> or <p class="p24"> or <p class="p34"> after our content,
or just past the last essay paragraph.
Returns (start_idx, end_idx) — exclusive end.
"""
te_start = None
te_end = None
for i, p in enumerate(all_paragraphs):
css = get_css_class(p)
text = p.get_text().strip().upper()
# Find the TWO ESSAYS header
if css == "p12" and "TWO ESSAYS" in text:
te_start = i
continue
# Once we've found the start, look for the end
if te_start is not None and te_end is None:
# A new p12 header means we've left Two Essays
if css == "p12" and i > te_start:
te_end = i
break
# Collier end markers
if css in ("p24", "p34"):
te_end = i
break
if te_start is None:
raise ValueError("Could not find TWO ESSAYS header in HTML")
# If we didn't find an explicit end marker, scan for the last body paragraph
if te_end is None:
# Walk backwards from the end to find the last relevant paragraph
for i in range(len(all_paragraphs) - 1, te_start, -1):
css = get_css_class(all_paragraphs[i])
if css in BODY_CLASSES:
te_end = i + 1
break
if te_end is None:
te_end = len(all_paragraphs)
return te_start, te_end
def find_essay_headings(
all_paragraphs: list[Tag], start: int, end: int
) -> list[tuple[int, str]]:
"""Find p27 headings within the Two Essays range.
Returns list of (paragraph_index, title_text).
"""
headings = []
for i in range(start, end):
p = all_paragraphs[i]
css = get_css_class(p)
if css == "p27":
title = p.get_text().strip()
headings.append((i, title))
return headings
# ---------------------------------------------------------------------------
# Paragraph extraction
# ---------------------------------------------------------------------------
def extract_essay_paragraphs(
all_paragraphs: list[Tag], start: int, end: int
) -> list[str]:
"""Extract body paragraphs from the given range.
Includes: p28, p6, p22, p3
Skips: SKIP_CLASSES, p27 (headings), empty paragraphs
"""
paragraphs = []
for i in range(start, end):
p = all_paragraphs[i]
css = get_css_class(p)
# Skip classes we don't want
if css in SKIP_CLASSES:
continue
# Skip headings (already handled)
if css == "p27":
continue
# Only accept body classes
if css not in BODY_CLASSES:
continue
# Skip empty paragraphs
if is_empty(p):
continue
html = clean_html_content(p)
if html and html != "&nbsp;":
paragraphs.append(html)
return paragraphs
# ---------------------------------------------------------------------------
# Main extraction
# ---------------------------------------------------------------------------
def extract_te(soup: BeautifulSoup) -> dict:
"""Extract Two Essays from the parsed HTML.
Returns a dict with:
"essays": list of essay dicts
"essay_listing": list of {slug, title, order} for intro.json
"""
all_paragraphs = list(soup.find_all("p"))
# Step 1: Find Two Essays boundaries
te_start, te_end = find_te_boundaries(all_paragraphs)
print(f" Two Essays content: paragraphs {te_start} to {te_end}")
print(f" Total paragraphs in range: {te_end - te_start}")
# Step 2: Find essay headings (p27)
headings = find_essay_headings(all_paragraphs, te_start, te_end)
print(f" Found {len(headings)} essay headings")
for pos, title in headings:
print(f" paragraph {pos}: {title}")
if len(headings) != 2:
print(f" WARNING: Expected 2 essay headings, found {len(headings)}")
# Step 3: Extract essays
essays = []
essay_configs = [
{
"slug": "krishna-reservoir-of-pleasure",
"title": "Krishna The Reservoir Of Pleasure",
"order": 1,
},
{
"slug": "who-is-crazy",
"title": "Who Is Crazy?",
"order": 2,
},
]
for idx, (heading_pos, heading_title) in enumerate(headings):
config = essay_configs[idx] if idx < len(essay_configs) else None
# Content starts after the heading
content_start = heading_pos + 1
# Content ends at next heading or te_end
if idx + 1 < len(headings):
content_end = headings[idx + 1][0]
else:
content_end = te_end
paragraphs = extract_essay_paragraphs(
all_paragraphs, content_start, content_end
)
if config:
slug = config["slug"]
title = config["title"]
order = config["order"]
else:
# Fallback for unexpected essays
slug = re.sub(r"[^a-z0-9]+", "-", heading_title.lower()).strip("-")
title = heading_title
order = idx + 1
essay = {
"slug": slug,
"title": title,
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
"year": 1966,
"group": "two-essays",
"order": order,
"paragraphs": paragraphs,
}
essays.append(essay)
# Step 4: Build essay listing for intro.json
essay_listing = [
{"slug": e["slug"], "title": e["title"], "order": e["order"]} for e in essays
]
return {
"essays": essays,
"essay_listing": essay_listing,
}
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
def main():
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
html_path = os.path.join(base_dir, "public", "html", "other.html")
output_base = os.path.join(base_dir, "lib", "jsons", "books", "te")
if not os.path.exists(html_path):
print(f"ERROR: {html_path} not found!")
sys.exit(1)
print(f"Reading {html_path}...")
with open(html_path, "r", encoding="utf-8") as f:
soup = BeautifulSoup(f, "html.parser")
print("Extracting Two Essays (1966)...\n")
result = extract_te(soup)
essays = result["essays"]
essay_listing = result["essay_listing"]
# --- Write intro.json ---
intro_meta = {
"group": "two-essays",
"title": "Two Essays",
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
"year": 1966,
"description": "Two foundational essays: 'Krishna The Reservoir Of Pleasure' and 'Who Is Crazy?'",
"essays": essay_listing,
}
intro_path = os.path.join(output_base, "intro.json")
write_json(intro_meta, intro_path)
print(f"\n Written: {intro_path}")
# --- Write essay files ---
print(f"\n--- Essays ---")
for essay in essays:
filepath = os.path.join(output_base, "essays", f"{essay['slug']}.json")
write_json(essay, filepath)
print(
f" {essay['order']}. {essay['title']}: "
f"{len(essay['paragraphs'])} paragraphs → {filepath}"
)
# --- Summary ---
total_paras = sum(len(e["paragraphs"]) for e in essays)
print(f"\n{'=' * 60}")
print(f" EXTRACTION SUMMARY")
print(f"{'=' * 60}")
print(f" Essays: {len(essays)}")
print(f" Total paragraphs: {total_paras}")
for e in essays:
print(f" {e['order']}. {e['title']}: {len(e['paragraphs'])} paragraphs")
if essays:
print(f" Avg paras/essay: {total_paras / len(essays):.1f}")
print(f" Output directory: {output_base}/")
# --- Samples ---
for essay in essays:
print(f"\n--- Sample: {essay['title']} ---")
for i, p in enumerate(essay["paragraphs"][:3]):
preview = re.sub(r"<[^>]+>", "", p)[:120]
print(f" [{i}] {preview}...")
# --- Quality Checks ---
print(f"\n--- Quality Checks ---")
# Check both essays present
expected_slugs = {"krishna-reservoir-of-pleasure", "who-is-crazy"}
actual_slugs = {e["slug"] for e in essays}
missing = expected_slugs - actual_slugs
extra = actual_slugs - expected_slugs
if missing:
print(f" ⚠ Missing essays: {missing}")
if extra:
print(f" ⚠ Extra essays: {extra}")
if not missing and not extra:
print(f" ✅ Both essays present")
# Check empty essays
empty_essays = [e["slug"] for e in essays if len(e["paragraphs"]) == 0]
if empty_essays:
print(f" ⚠ Empty essays (0 paragraphs): {empty_essays}")
else:
print(f" ✅ No empty essays")
# Check paragraph counts are reasonable
for e in essays:
count = len(e["paragraphs"])
if count < 5:
print(
f" ⚠ Essay '{e['slug']}' has only {count} paragraphs (suspiciously low)"
)
elif count > 100:
print(f" ⚠ Essay '{e['slug']}' has {count} paragraphs (suspiciously high)")
else:
print(f" ✅ Essay '{e['slug']}' has {count} paragraphs (reasonable)")
# Check for residual span class artifacts
span_count = 0
for e in essays:
for p in e["paragraphs"]:
if 'class="' in p and "<span" in p:
span_count += 1
if span_count == 0:
print(f" ✅ No residual span class artifacts found")
else:
print(f" ⚠ {span_count} paragraphs still contain span class artifacts")
# Check no ScaGoudy encoding present (these are plain English essays)
# But verify key words appear as plain English (not IAST)
all_text = " ".join(" ".join(e["paragraphs"][:10]) for e in essays)
if "Krishna" in all_text:
print(f" ✅ 'Krishna' found (plain English, as expected for 1966)")
else:
print(f" ⚠ 'Krishna' not found in text")
if "Bhagavad Gita" in all_text or "Bhagavad-Gita" in all_text:
print(f" ✅ 'Bhagavad Gita' found (plain English)")
# Check for ad content leaking in
ad_markers = ["ISKCON", "26 2nd Ave", "SRIMAD-BHAGAWATAM", "$3.25", "$16.80"]
leaked_ads = []
for e in essays:
for p in e["paragraphs"]:
plain = re.sub(r"<[^>]+>", "", p)
for marker in ad_markers:
if marker in plain:
leaked_ads.append((e["slug"], marker, plain[:80]))
break
if leaked_ads:
print(f" ⚠ {len(leaked_ads)} paragraphs may contain ad content:")
for slug, marker, preview in leaked_ads[:3]:
print(f" [{slug}] matched '{marker}': {preview}...")
else:
print(f" ✅ No ad content detected in essays")
# Check for SKIP_CLASSES content leaking in
# (p18 paragraphs contain ad text like "BLISS—what was once only a word...")
bliss_check = False
for e in essays:
for p in e["paragraphs"]:
plain = re.sub(r"<[^>]+>", "", p)
if plain.startswith("BLISS") or "SOUNDS OF KRISHNA CONSCIOUSNESS" in plain:
bliss_check = True
break
if bliss_check:
print(f" ⚠ Ad text ('BLISS...') leaked into essay paragraphs")
else:
print(f" ✅ Ad text properly excluded")
# Show last paragraph of each essay
print(f"\n--- Last Paragraph of Each Essay ---")
for e in essays:
if e["paragraphs"]:
last = re.sub(r"<[^>]+>", "", e["paragraphs"][-1])[:150]
print(f" {e['title']}:")
print(f" ...{last}")
print(f"\nDone! ✨")
if __name__ == "__main__":
main()
scripts/tlc_extract.py
+868
View File
@@ -0,0 +1,868 @@
#!/usr/bin/env python3
"""Extract Teachings of Lord Caitanya (1968 Edition) from other.html.
The book has 9 intro segments, 32 chapters in 2 parts.
IMPORTANT: This is the 1968 edition with plain English transliterations.
NO ScaGoudy font decoding is needed.
Output:
lib/jsons/books/tlc/intro.json — Book metadata
lib/jsons/books/tlc/intro/*.json — Intro segments
lib/jsons/books/tlc/chapters/*.json — Chapters
Usage:
python3 scripts/tlc_extract.py
"""
import json
import os
import re
import sys
from bs4 import BeautifulSoup, Tag
# ---------------------------------------------------------------------------
# CSS class categorization for TLC
# ---------------------------------------------------------------------------
# Bookmark class — used for section boundary detection
BOOKMARK_CLASS = "p13"
# Book title header class
BOOK_HEADER_CLASS = "p12"
# Body paragraph classes (primary content)
BODY_CLASSES = frozenset(["p6", "p44", "p18", "p30"])
# Section heading (bold) — "Chapter One", "Preface", etc.
SECTION_HEADING_CLASS = "p31"
# Chapter title (bold) — "Teachings to Rupa Goswami", etc.
CHAPTER_TITLE_CLASS = "p27"
# Part header class — "PART I", "PART II"
PART_HEADER_CLASS = "p34"
# Title page / copyright text classes (in intro area)
TITLE_PAGE_CLASS = "p3"
COPYRIGHT_CLASS = "p32"
# Italic verse quote and attribution
VERSE_QUOTE_CLASS = "p39"
VERSE_ATTRIBUTION_CLASS = "p40"
# Photo/painting credits
CREDITS_CLASS = "p41"
# Attribution text ("The Publishers")
ATTRIBUTION_CLASS = "p42"
# Section title bold text
SECTION_TITLE_CLASS = "p43"
# Numbered items (in Mission section)
NUMBERED_ITEM_CLASS = "p24"
# Italic text (e.g., "Hari! Hari!")
ITALIC_TEXT_CLASS = "p19"
# Classes to skip entirely (empty/decorative)
SKIP_CLASSES = frozenset(["p9", "p15"])
# ---------------------------------------------------------------------------
# Intro segment definitions
# ---------------------------------------------------------------------------
INTRO_SEGMENTS = [
{
"slug": "front-flap",
"title": "Front Flap & Pages",
"order": 1,
"bookmark_text": "TLC-1968 Front flap & pages",
},
{
"slug": "the-publishers",
"title": "The Publishers",
"order": 2,
"bookmark_text": "TLC-1968 The Publishers",
},
{
"slug": "dedication",
"title": "Dedication",
"order": 3,
"bookmark_text": "TLC-1968 DEDICATION",
},
{
"slug": "acknowledgement",
"title": "Acknowledgement",
"order": 4,
"bookmark_text": "TLC-1968 ACKNOWLEDGEMENT",
},
{
"slug": "the-author",
"title": "The Author",
"order": 5,
"bookmark_text": "TLC-1968 The Author",
},
{
"slug": "preface",
"title": "Preface",
"order": 6,
"bookmark_text": "TLC-1968 Preface",
},
{
"slug": "prologue",
"title": "Prologue",
"order": 7,
"bookmark_text": "TLC-1968 Prologue",
},
{
"slug": "mission",
"title": "Lord Chaitanya's Mission",
"order": 8,
"bookmark_text": "TLC-1968 Mission",
},
{
"slug": "introduction",
"title": "Introduction",
"order": 9,
"bookmark_text": "TLC-1968 Intoduction", # Note: typo in original
},
]
# ---------------------------------------------------------------------------
# HTML content helpers
# ---------------------------------------------------------------------------
def clean_html_content(tag: Tag) -> str:
"""Extract inner HTML from a tag, strip span artifacts, preserve formatting tags.
IMPORTANT: No ScaGoudy decoding — this is the 1968 plain English edition.
"""
inner = "".join(str(child) for child in tag.children)
# Remove span tags with CSS classes (RTF conversion artifacts)
inner = re.sub(
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
)
# Clean up Apple-tab-span leftovers (tab characters)
inner = inner.replace("\t", " ")
# Normalize whitespace (but preserve <br> tags)
inner = re.sub(r"\s+", " ", inner).strip()
return inner
def get_plain_text(tag: Tag) -> str:
"""Get plain text from a tag (no decoding needed for 1968 edition)."""
return tag.get_text().strip()
def get_css_class(tag: Tag) -> str:
"""Get the first CSS class of a tag, or empty string."""
classes = tag.get("class", [])
return classes[0] if classes else ""
def is_empty(tag: Tag) -> bool:
"""Check if a tag is empty or whitespace-only."""
text = tag.get_text().strip()
return not text or text == "\xa0"
def is_dash_line(text: str) -> bool:
"""Return True if text is only dashes, spaces, and/or whitespace."""
cleaned = text.strip()
if not cleaned:
return False
return all(c in "-—–\u2014\u2013 \t" for c in cleaned)
# ---------------------------------------------------------------------------
# Extraction helpers
# ---------------------------------------------------------------------------
def find_tlc_boundaries(all_paragraphs: list) -> tuple:
"""Find the start and end paragraph indices of the TLC content.
Start: <p class="p12"> containing "Teachings of Lord Caitanya"
End: next <p class="p12"> after start, or a dashed line followed by p3,
or end of file
"""
start_idx = None
end_idx = len(all_paragraphs)
for i, p in enumerate(all_paragraphs):
cls = get_css_class(p)
if cls != BOOK_HEADER_CLASS:
continue
text = get_plain_text(p)
if "Teachings of Lord Caitanya" in text:
start_idx = i
elif start_idx is not None:
# Next p12 after TLC start = end of TLC
end_idx = i
break
if start_idx is None:
print(
"ERROR: Could not find 'Teachings of Lord Caitanya' header in other.html!"
)
sys.exit(1)
# Also check for the dashed-line boundary before end_idx
# The last content is followed by a dashed line, then the next book starts
for i in range(start_idx + 1, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
text = get_plain_text(p)
# A dashed line in a p6 paragraph followed by a p3 paragraph = end boundary
if cls == "p6" and is_dash_line(text):
# Check if next non-empty paragraph is p3 (start of next book)
for j in range(i + 1, min(i + 5, end_idx)):
next_cls = get_css_class(all_paragraphs[j])
if next_cls == "p3":
end_idx = i # End before the dashed line
break
if end_idx == i:
break
return start_idx, end_idx
def find_all_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
"""Find all p13 bookmark tags in the TLC range.
Returns list of (paragraph_index, bookmark_text).
"""
bookmarks = []
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
if cls == BOOKMARK_CLASS:
text = get_plain_text(p)
if "TLC" in text:
bookmarks.append((i, text))
return bookmarks
def parse_chapter_bookmark(text: str) -> tuple:
"""Parse a chapter bookmark like 'TLC 1-1968: Teachings to Rupa Goswami'.
Returns (chapter_number, title) or None if not a chapter bookmark.
"""
m = re.match(r"TLC\s+(\d+)-1968:\s*(.+)", text.strip())
if m:
return int(m.group(1)), m.group(2).strip()
return None
def parse_intro_bookmark(text: str) -> str:
"""Parse an intro bookmark like 'TLC-1968 Preface'.
Returns the segment name or None if not an intro bookmark.
"""
m = re.match(r"TLC-1968\s+(.+)", text.strip())
if m:
return m.group(1).strip()
return None
def extract_paragraphs(
all_paragraphs: list,
start_idx: int,
end_idx: int,
is_intro: bool = False,
) -> list:
"""Extract paragraphs for a section (chapter or intro segment).
Handles all CSS classes within the TLC content.
No ScaGoudy decoding is applied.
"""
result = []
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
# Skip empty paragraphs
if is_empty(p):
continue
# Skip noise classes
if cls in SKIP_CLASSES:
continue
# Skip bookmark tags
if cls == BOOKMARK_CLASS:
continue
# Skip book title header
if cls == BOOK_HEADER_CLASS:
continue
# Check for dash lines — skip them
text = get_plain_text(p)
if is_dash_line(text):
continue
# Part headers (p34) — include as bold paragraph
if cls == PART_HEADER_CLASS:
html = clean_html_content(p)
if html:
result.append(html)
continue
# Section heading (p31) — include as paragraph
if cls == SECTION_HEADING_CLASS:
html = clean_html_content(p)
if html:
result.append(html)
continue
# Chapter title (p27) — include as bold paragraph
if cls == CHAPTER_TITLE_CLASS:
html = clean_html_content(p)
if html:
result.append(html)
continue
# Verse quote (p39) — format as centered italic div
if cls == VERSE_QUOTE_CLASS:
html = clean_html_content(p)
if html:
# Strip inner <i></i> tags before re-wrapping
stripped = re.sub(r"</?i>", "", html)
result.append(f'<div style="text-align:center"><i>{stripped}</i></div>')
continue
# Verse attribution (p40) — format as centered italic div
if cls == VERSE_ATTRIBUTION_CLASS:
html = clean_html_content(p)
if html:
stripped = re.sub(r"</?i>", "", html)
result.append(f'<div style="text-align:center"><i>{stripped}</i></div>')
continue
# Title page / copyright text (in intro area)
if cls == TITLE_PAGE_CLASS:
html = clean_html_content(p)
if html and not is_dash_line(get_plain_text(p)):
# Detect Sanskrit verses disguised as title-page paragraphs
stripped_html = html.strip()
if stripped_html.startswith("<i>") and "<br" in stripped_html:
inner = re.sub(r"</?i>", "", stripped_html)
result.append(
f'<div style="text-align:center"><i>{inner}</i></div>'
)
else:
result.append(html)
continue
# Copyright text (p32)
if cls == COPYRIGHT_CLASS:
html = clean_html_content(p)
if html:
result.append(html)
continue
# Photo/painting credits (p41)
if cls == CREDITS_CLASS:
html = clean_html_content(p)
if html:
result.append(html)
continue
# Attribution text (p42) — "The Publishers"
if cls == ATTRIBUTION_CLASS:
html = clean_html_content(p)
if html:
result.append(html)
continue
# Section title bold text (p43)
if cls == SECTION_TITLE_CLASS:
html = clean_html_content(p)
if html:
result.append(html)
continue
# Numbered items (p24) — include as regular paragraphs
if cls == NUMBERED_ITEM_CLASS:
html = clean_html_content(p)
if html:
result.append(html)
continue
# Italic text like "Hari! Hari!" (p19)
if cls == ITALIC_TEXT_CLASS:
html = clean_html_content(p)
if html:
result.append(html)
continue
# Body paragraphs (p6, p44, p18, p30)
if cls in BODY_CLASSES:
html = clean_html_content(p)
if html:
# Detect Sanskrit verses disguised as body paragraphs:
# they start with <i> and contain <br> (multi-line italic = verse)
stripped_html = html.strip()
if stripped_html.startswith("<i>") and "<br" in stripped_html:
# This is a verse — wrap in centered italic div
inner = re.sub(r"</?i>", "", stripped_html)
result.append(
f'<div style="text-align:center"><i>{inner}</i></div>'
)
else:
result.append(html)
continue
# Any other class — try to extract content (be permissive)
html = clean_html_content(p)
if html and not is_dash_line(text):
result.append(html)
return result
# ---------------------------------------------------------------------------
# Main extraction logic
# ---------------------------------------------------------------------------
def extract_tlc(soup: BeautifulSoup) -> dict:
"""Extract the entire Teachings of Lord Caitanya from the parsed HTML.
Returns a dict with:
"intro_segments": list of intro segment dicts
"chapters": list of chapter dicts
"parts": list of part assignment dicts
"chapter_listing": list of {number, title} for intro.json
"""
all_paragraphs = list(soup.find_all("p"))
# Step 1: Find TLC boundaries
tlc_start, tlc_end = find_tlc_boundaries(all_paragraphs)
print(f" TLC content: paragraphs {tlc_start} to {tlc_end}")
print(f" Total paragraphs in range: {tlc_end - tlc_start}")
# Step 2: Find all bookmarks
bookmarks = find_all_bookmarks(all_paragraphs, tlc_start, tlc_end)
print(f" Found {len(bookmarks)} TLC bookmarks")
# Step 3: Categorize bookmarks into intro vs chapter
intro_bookmarks = [] # (para_idx, bookmark_text)
chapter_bookmarks = [] # (para_idx, chapter_num, title)
for pos, text in bookmarks:
ch = parse_chapter_bookmark(text)
if ch:
chapter_bookmarks.append((pos, ch[0], ch[1]))
else:
intro_name = parse_intro_bookmark(text)
if intro_name:
intro_bookmarks.append((pos, text))
print(f" Intro bookmarks: {len(intro_bookmarks)}")
print(f" Chapter bookmarks: {len(chapter_bookmarks)}")
# Step 4: Track PART assignments by scanning for p34 tags
part_assignments = {} # chapter_num -> part_num
parts_data = []
part_positions = [] # (para_idx, part_title)
for i in range(tlc_start, tlc_end):
p = all_paragraphs[i]
cls = get_css_class(p)
if cls == PART_HEADER_CLASS:
text = get_plain_text(p)
if text:
part_positions.append((i, text))
print(f" Part headers found: {len(part_positions)}")
for pos, title in part_positions:
print(f" Paragraph ~{pos}: {title}")
# Map part positions to part numbers
part_number_map = {}
for pos, title in part_positions:
title_upper = title.upper().strip()
if "II" in title_upper and "III" not in title_upper:
part_number_map[pos] = 2
elif "I" in title_upper:
part_number_map[pos] = 1
# Assign parts to chapters based on position
sorted_part_positions = sorted(part_number_map.keys())
for ch_pos, ch_num, ch_title in chapter_bookmarks:
assigned_part = None
for pp in sorted_part_positions:
if pp < ch_pos:
assigned_part = part_number_map[pp]
else:
break
if assigned_part:
part_assignments[ch_num] = assigned_part
# Build parts_data
for part_num in sorted(set(part_assignments.values())):
chs = sorted([ch for ch, pn in part_assignments.items() if pn == part_num])
part_title = f"Part {'I' if part_num == 1 else 'II'}"
parts_data.append(
{
"number": part_num,
"title": part_title,
"chapters": chs,
}
)
# Step 5: Collect all section start positions for boundary detection
all_section_starts = sorted(
[pos for pos, _ in intro_bookmarks] + [pos for pos, _, _ in chapter_bookmarks]
)
# Step 6: Extract intro segments
intro_segments = []
for seg_def in INTRO_SEGMENTS:
# Find matching bookmark
bookmark_pos = None
for pos, btext in intro_bookmarks:
if btext.strip() == seg_def["bookmark_text"]:
bookmark_pos = pos
break
if bookmark_pos is None:
print(f" WARNING: Could not find bookmark for '{seg_def['slug']}'")
continue
# Content starts after the bookmark
content_start = bookmark_pos + 1
# Content ends at the next bookmark or part header or tlc_end
content_end = tlc_end
for s in all_section_starts:
if s > bookmark_pos:
content_end = s
break
# Also check if a PART header comes before the next bookmark
for pp in sorted_part_positions:
if pp > bookmark_pos and pp < content_end:
content_end = pp
break
paragraphs = extract_paragraphs(
all_paragraphs, content_start, content_end, is_intro=True
)
# Use the corrected title (e.g., "Introduction" not "Intoduction")
seg = {
"slug": seg_def["slug"],
"title": seg_def["title"],
"type": "intro-segment",
"book": "tlc",
"order": seg_def["order"],
"paragraphs": paragraphs,
}
intro_segments.append(seg)
# Step 7: Extract chapters
chapters = []
for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_bookmarks):
content_start = ch_pos + 1
# Content ends at next chapter bookmark or tlc_end
if idx + 1 < len(chapter_bookmarks):
content_end = chapter_bookmarks[idx + 1][0]
else:
content_end = tlc_end
# For the PART header: if a PART header appears right before this
# chapter's bookmark, include it in this chapter's content range.
actual_start = content_start
for pp in sorted_part_positions:
if pp < ch_pos and pp >= content_start - 5:
# Check that there's no other bookmark between pp and ch_pos
has_other_bookmark = False
for bpos, _ in bookmarks:
if bpos > pp and bpos < ch_pos:
has_other_bookmark = True
break
if not has_other_bookmark:
actual_start = pp
paragraphs = extract_paragraphs(all_paragraphs, actual_start, content_end)
ch = {
"book": "tlc",
"chapter": ch_num,
"title": ch_title,
"type": "chapter",
"paragraphs": paragraphs,
}
chapters.append(ch)
# Step 8: Build chapter listing for intro.json
chapter_listing = [
{"number": ch["chapter"], "title": ch["title"]} for ch in chapters
]
return {
"intro_segments": intro_segments,
"chapters": chapters,
"parts": parts_data,
"chapter_listing": chapter_listing,
}
# ---------------------------------------------------------------------------
# Output
# ---------------------------------------------------------------------------
def write_json(data, filepath):
"""Write data as formatted JSON to filepath, creating directories as needed."""
os.makedirs(os.path.dirname(filepath), exist_ok=True)
with open(filepath, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False, indent=2)
def main():
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
html_path = os.path.join(base_dir, "public", "html", "other.html")
output_base = os.path.join(base_dir, "lib", "jsons", "books", "tlc")
if not os.path.exists(html_path):
print(f"ERROR: {html_path} not found!")
sys.exit(1)
print(f"Reading {html_path}...")
with open(html_path, "r", encoding="utf-8") as f:
soup = BeautifulSoup(f, "html.parser")
print("Extracting Teachings of Lord Caitanya (1968 Edition)...\n")
result = extract_tlc(soup)
intro_segments = result["intro_segments"]
chapters = result["chapters"]
parts = result["parts"]
chapter_listing = result["chapter_listing"]
# --- Write intro.json ---
intro_meta = {
"book": "tlc",
"title": "Teachings of Lord Caitanya",
"subtitle": "A Treatise on Factual Spiritual Life",
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
"year": 1968,
"description": "The sublime life and teachings of Lord Caitanya Mahāprabhu, the most munificent incarnation of the Supreme Lord.",
"totalChapters": 32,
"parts": parts,
"chapters": chapter_listing,
}
intro_path = os.path.join(output_base, "intro.json")
write_json(intro_meta, intro_path)
print(f"\n Written: {intro_path}")
# --- Write intro segments ---
print(f"\n--- Intro Segments ---")
for seg in intro_segments:
filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
write_json(seg, filepath)
print(
f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}"
)
# --- Write chapters ---
print(f"\n--- Chapters ---")
for ch in chapters:
ch_num_str = str(ch["chapter"]).zfill(2)
filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
write_json(ch, filepath)
part_info = ""
for p in parts:
if ch["chapter"] in p["chapters"]:
part_info = f" [Part {p['number']}]"
break
print(
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
f"{ch['title']}{part_info}"
)
# --- Summary ---
total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments)
total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
total_paras = total_intro_paras + total_chapter_paras
print(f"\n{'=' * 60}")
print(f" EXTRACTION SUMMARY")
print(f"{'=' * 60}")
print(f" Intro segments: {len(intro_segments)}")
print(f" Chapters: {len(chapters)}")
print(f" Parts: {len(parts)}")
for p in parts:
print(
f" Part {p['number']} ({p['title']}): chapters {p['chapters'][0]}-{p['chapters'][-1]} ({len(p['chapters'])} chapters)"
)
print(f" Total paragraphs: {total_paras}")
print(f" Intro: {total_intro_paras}")
print(f" Chapters: {total_chapter_paras}")
if chapters:
print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
print(f" Output directory: {output_base}/")
# --- Samples ---
print(f"\n--- Sample: Chapter 1 ---")
if chapters:
ch1 = chapters[0]
for i, p in enumerate(ch1["paragraphs"][:4]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
if chapters and len(chapters) > 1:
print(f"\n--- Sample: Chapter {chapters[-1]['chapter']} (last) ---")
last = chapters[-1]
for i, p in enumerate(last["paragraphs"][:3]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
if intro_segments:
print(f"\n--- Sample: First intro segment ({intro_segments[0]['title']}) ---")
first_intro = intro_segments[0]
for i, p in enumerate(first_intro["paragraphs"][:3]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
# --- Quality Checks ---
print(f"\n--- Quality Checks ---")
# Check all 32 chapters present
expected_chapters = set(range(1, 33))
actual_chapters = set(ch["chapter"] for ch in chapters)
missing = expected_chapters - actual_chapters
extra = actual_chapters - expected_chapters
if missing:
print(f" ⚠ Missing chapters: {sorted(missing)}")
if extra:
print(f" ⚠ Extra chapters: {sorted(extra)}")
if not missing and not extra:
print(f" ✅ All 32 chapters present")
# Check all 9 intro segments present
expected_intros = {
"front-flap",
"the-publishers",
"dedication",
"acknowledgement",
"the-author",
"preface",
"prologue",
"mission",
"introduction",
}
actual_intros = {s["slug"] for s in intro_segments}
missing_intros = expected_intros - actual_intros
extra_intros = actual_intros - expected_intros
if missing_intros:
print(f" ⚠ Missing intro segments: {missing_intros}")
if extra_intros:
print(f" ⚠ Extra intro segments: {extra_intros}")
if not missing_intros and not extra_intros:
print(f" ✅ All 9 intro segments present")
# Check empty chapters
empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
if empty_chapters:
print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
else:
print(f" ✅ No empty chapters")
# Check empty intro segments
empty_intros = [s["slug"] for s in intro_segments if len(s["paragraphs"]) == 0]
if empty_intros:
print(f" ⚠ Empty intro segments: {empty_intros}")
else:
print(f" ✅ No empty intro segments")
# Check part assignments
all_part_chapters = set()
for p in parts:
all_part_chapters.update(p["chapters"])
unassigned = [
ch["chapter"] for ch in chapters if ch["chapter"] not in all_part_chapters
]
if unassigned:
print(f" ⚠ Chapters not assigned to any part: {unassigned}")
else:
print(f" ✅ All chapters assigned to parts")
# Verify Part I has chapters 1-16 and Part II has chapters 17-32
for p in parts:
if p["number"] == 1:
expected = list(range(1, 17))
if p["chapters"] == expected:
print(f" ✅ Part I correctly has chapters 1-16")
else:
print(f" ⚠ Part I has chapters {p['chapters']}, expected {expected}")
elif p["number"] == 2:
expected = list(range(17, 33))
if p["chapters"] == expected:
print(f" ✅ Part II correctly has chapters 17-32")
else:
print(f" ⚠ Part II has chapters {p['chapters']}, expected {expected}")
# Verify no ScaGoudy artifacts (there shouldn't be any for 1968 edition)
# Look for common ScaGoudy encoded words that would indicate wrongly applied decoding
print(f"\n--- 1968 Edition Check ---")
if chapters:
ch1_text = " ".join(chapters[0]["paragraphs"][:10])
if "Krishna" in ch1_text:
print(f" ✅ 'Krishna' found in Chapter 1 (plain English as expected)")
if "Kṛṣṇa" in ch1_text:
print(
f" ℹ 'Kṛṣṇa' (IAST) found in Chapter 1 — unexpected for 1968 edition"
)
if "Goswami" in ch1_text or "Rupa Goswami" in ch1_text:
print(f" ✅ 'Goswami' found (plain English as expected)")
# Check for residual span tags
span_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
if 'class="' in p and "<span" in p:
span_count += 1
for seg in intro_segments:
for p in seg["paragraphs"]:
if 'class="' in p and "<span" in p:
span_count += 1
if span_count == 0:
print(f" ✅ No residual span class artifacts found")
else:
print(f" ⚠ {span_count} paragraphs still contain span class artifacts")
# Check for dash-line paragraphs that shouldn't be there
dash_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
plain = re.sub(r"<[^>]+>", "", p)
if is_dash_line(plain):
dash_count += 1
for seg in intro_segments:
for p in seg["paragraphs"]:
plain = re.sub(r"<[^>]+>", "", p)
if is_dash_line(plain):
dash_count += 1
if dash_count == 0:
print(f" ✅ No dashed-line paragraphs in output")
else:
print(f" ⚠ {dash_count} dashed-line paragraphs found in output")
print(f"\nDone! ✨")
if __name__ == "__main__":
main()
scripts/ttp_extract.py
+641
View File
@@ -0,0 +1,641 @@
#!/usr/bin/env python3
"""Extract Transcendental Teachings of Prahlāda Mahārāja (TTP) from other.html.
The book has 1 intro segment (front-pages) and 5 chapters.
There are NO p13 bookmarks — chapters are detected via p27 headings.
Output:
lib/jsons/books/ttp/intro.json — Book metadata
lib/jsons/books/ttp/intro/*.json — Intro segments
lib/jsons/books/ttp/chapters/*.json — Chapters
Usage:
python3 scripts/ttp_extract.py
"""
import json
import os
import re
import sys
from bs4 import BeautifulSoup, Tag
# ---------------------------------------------------------------------------
# ScaGoudy → IAST Unicode mapping
# ---------------------------------------------------------------------------
SCAGOUDY_TO_IAST = {
"ä": "ā",
"Ä": "Ā",
"å": "ṛ",
"Å": "Ṛ",
"ñ": "ṣ",
"Ñ": "Ṣ",
"ö": "ṭ",
"Ö": "Ṭ",
"ë": "ṇ",
"Ë": "Ṇ",
"ò": "ḍ",
"Ò": "Ḍ",
"ç": "ś",
"Ç": "Ś",
"ì": "ṅ",
"ï": "ñ",
"ü": "ū",
"Ü": "Ū",
"é": "ī",
"É": "Ī",
"à": "ṁ",
"ù": "ḥ",
"è": "ṝ",
"û": "ṁl",
"ÿ": "ḷ",
}
def decode_scagoudy(text: str) -> str:
"""Replace ScaGoudy-encoded characters with proper IAST Unicode."""
for src, dst in SCAGOUDY_TO_IAST.items():
text = text.replace(src, dst)
return text
def decode_html_content(tag: Tag) -> str:
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
inner = "".join(str(child) for child in tag.children)
inner = decode_scagoudy(inner)
# Remove span tags with CSS classes (RTF conversion artifacts)
inner = re.sub(
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
)
# Clean up Apple-tab-span leftovers (tab characters)
inner = inner.replace("\t", " ")
# Normalize whitespace (but preserve <br> tags)
inner = re.sub(r"\s+", " ", inner).strip()
return inner
def decode_text(tag: Tag) -> str:
"""Get plain text from a tag with ScaGoudy decoding."""
return decode_scagoudy(tag.get_text().strip())
def get_css_class(tag: Tag) -> str:
"""Get the first CSS class of a tag, or empty string."""
classes = tag.get("class", [])
return classes[0] if classes else ""
def is_empty(tag: Tag) -> bool:
"""Check if a tag is empty or whitespace-only."""
text = tag.get_text().strip()
return not text or text == "\xa0"
def is_dash_line(text: str) -> bool:
"""Return True if text is only dashes, spaces, and/or whitespace."""
cleaned = text.strip()
if not cleaned:
return False
return all(c in "-—–\u2014\u2013 \t" for c in cleaned)
# ---------------------------------------------------------------------------
# CSS class categorization for TTP
# ---------------------------------------------------------------------------
# Book title header class
BOOK_HEADER_CLASS = "p12"
# Chapter heading class (used as chapter boundary since no p13 bookmarks)
CHAPTER_HEADING_CLASS = "p27"
# Body paragraph classes (all treated as paragraphs)
BODY_CLASSES = frozenset(["p48", "p6", "p99", "p98", "p71"])
# Front-page specific classes to include
FRONT_PAGE_CLASSES = frozenset(["p69", "p76", "p18"])
# Classes to skip entirely
SKIP_CLASSES = frozenset(["p9", "p15", "p70"])
# Chapter titles mapping (proper IAST decoded)
CHAPTER_TITLES = {
1: "The Dearmost Person",
2: "Move on to the Perfect Goal",
3: "The Highest Welfare Activity",
4: "Cutting Off Ties of Affection",
5: "Prayers by Prahlāda Mahārāja",
}
# ---------------------------------------------------------------------------
# Extraction helpers
# ---------------------------------------------------------------------------
def find_ttp_boundaries(all_paragraphs: list) -> tuple:
"""Find the start and end paragraph indices of the TTP content.
Start: <p class="p12"> containing "Transcendental Teachings of Prahläda"
End: next <p class="p12"> after start
"""
start_idx = None
end_idx = len(all_paragraphs)
for i, p in enumerate(all_paragraphs):
cls = get_css_class(p)
if cls != BOOK_HEADER_CLASS:
continue
text = p.get_text().strip()
if "Transcendental Teachings of Prahl" in text:
start_idx = i
elif start_idx is not None:
# Next p12 after TTP start = end of TTP
end_idx = i
break
if start_idx is None:
print(
"ERROR: Could not find 'Transcendental Teachings of Prahlāda' header in other.html!"
)
sys.exit(1)
return start_idx, end_idx
def find_chapter_headings(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
"""Find all chapter headings (p27) in the TTP range.
Parses heading text with pattern: "N.Title" or "N. Title"
Returns list of (paragraph_index, chapter_number, chapter_title).
"""
headings = []
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
if cls != CHAPTER_HEADING_CLASS:
continue
text = decode_text(p)
# Remove bold tags from text content
text = re.sub(r"</?b>", "", text).strip()
# Try to parse chapter number and title
m = re.match(r"(\d+)\s*[./]\s*(.+)", text)
if m:
ch_num = int(m.group(1))
ch_title_raw = m.group(2).strip()
# Strip surrounding quotes if present
ch_title_raw = ch_title_raw.strip('"').strip('"').strip('"')
# Use pre-defined proper IAST titles if available
ch_title = CHAPTER_TITLES.get(ch_num, ch_title_raw)
headings.append((i, ch_num, ch_title))
return headings
def extract_paragraphs(
all_paragraphs: list,
start_idx: int,
end_idx: int,
is_front_pages: bool = False,
) -> list:
"""Extract paragraphs for a section (chapter or intro segment).
Handles all CSS classes within the TTP content.
Detects italic+br verses in any body class → centered divs.
"""
result = []
verse_accumulator = []
def flush_verse():
if verse_accumulator:
joined = "<br>".join(verse_accumulator)
result.append(f'<div style="text-align:center"><i>{joined}</i></div>')
verse_accumulator.clear()
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
# Skip empty paragraphs
if is_empty(p):
continue
# Skip noise classes
if cls in SKIP_CLASSES:
continue
# Skip book title header (p12)
if cls == BOOK_HEADER_CLASS:
continue
# Skip chapter headings (p27) — they are boundaries, not content
if cls == CHAPTER_HEADING_CLASS:
continue
# Check for dash lines — skip them
text = decode_text(p)
if is_dash_line(text):
continue
# Non-verse paragraph — flush any accumulated verse first
flush_verse()
# Front-page specific classes
if is_front_pages and cls in FRONT_PAGE_CLASSES:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Body paragraphs (p48, p6, p99, p98, p71)
if cls in BODY_CLASSES:
html = decode_html_content(p)
if html:
# Detect Sanskrit verses disguised as body paragraphs:
# they start with <i> and contain <br> (multi-line italic = verse)
stripped_html = html.strip()
if stripped_html.startswith("<i>") and "<br" in stripped_html:
# This is a verse — wrap in centered italic div
inner = re.sub(r"</?i>", "", stripped_html)
result.append(
f'<div style="text-align:center"><i>{inner}</i></div>'
)
else:
result.append(html)
continue
# Front-page classes even outside is_front_pages context
if cls in FRONT_PAGE_CLASSES:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Any other class — try to extract content (be permissive)
html = decode_html_content(p)
if html and not is_dash_line(text):
result.append(html)
# Flush any trailing verse
flush_verse()
return result
# ---------------------------------------------------------------------------
# Main extraction logic
# ---------------------------------------------------------------------------
def extract_ttp(soup: BeautifulSoup) -> dict:
"""Extract the entire TTP book from the parsed HTML.
Returns a dict with:
"intro_segments": list of intro segment dicts
"chapters": list of chapter dicts
"chapter_listing": list of {number, title} for intro.json
"""
all_paragraphs = list(soup.find_all("p"))
# Step 1: Find TTP boundaries
ttp_start, ttp_end = find_ttp_boundaries(all_paragraphs)
print(f" TTP content: paragraphs {ttp_start} to {ttp_end}")
print(f" Total paragraphs in range: {ttp_end - ttp_start}")
# Step 2: Find all chapter headings (p27)
chapter_headings = find_chapter_headings(all_paragraphs, ttp_start, ttp_end)
print(f" Found {len(chapter_headings)} chapter headings")
for pos, ch_num, ch_title in chapter_headings:
print(f" [chapter {ch_num}] paragraph {pos}: {ch_title}")
# Step 3: Extract front pages
# From book start (ttp_start) to first chapter heading
intro_segments = []
if chapter_headings:
front_pages_end = chapter_headings[0][0]
else:
front_pages_end = ttp_end
front_pages_paragraphs = extract_paragraphs(
all_paragraphs, ttp_start, front_pages_end, is_front_pages=True
)
intro_segments.append(
{
"slug": "front-pages",
"title": "Front Pages",
"type": "intro-segment",
"book": "ttp",
"order": 1,
"paragraphs": front_pages_paragraphs,
}
)
# Step 4: Extract chapters
chapters = []
for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_headings):
# Content starts after the heading
content_start = ch_pos + 1
# Content ends at next chapter heading or ttp_end
if idx + 1 < len(chapter_headings):
content_end = chapter_headings[idx + 1][0]
else:
content_end = ttp_end
paragraphs = extract_paragraphs(all_paragraphs, content_start, content_end)
ch = {
"book": "ttp",
"chapter": ch_num,
"title": ch_title,
"type": "chapter",
"paragraphs": paragraphs,
}
chapters.append(ch)
# Step 5: Build chapter listing for intro.json
chapter_listing = [
{"number": ch["chapter"], "title": ch["title"]} for ch in chapters
]
return {
"intro_segments": intro_segments,
"chapters": chapters,
"chapter_listing": chapter_listing,
}
# ---------------------------------------------------------------------------
# Output
# ---------------------------------------------------------------------------
def write_json(data, filepath):
"""Write data as formatted JSON to filepath, creating directories as needed."""
os.makedirs(os.path.dirname(filepath), exist_ok=True)
with open(filepath, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False, indent=2)
def main():
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
html_path = os.path.join(base_dir, "public", "html", "other.html")
output_base = os.path.join(base_dir, "lib", "jsons", "books", "ttp")
if not os.path.exists(html_path):
print(f"ERROR: {html_path} not found!")
sys.exit(1)
print(f"Reading {html_path}...")
with open(html_path, "r", encoding="utf-8") as f:
soup = BeautifulSoup(f, "html.parser")
print("Extracting Transcendental Teachings of Prahlāda Mahārāja...\n")
result = extract_ttp(soup)
intro_segments = result["intro_segments"]
chapters = result["chapters"]
chapter_listing = result["chapter_listing"]
# --- Write intro.json ---
intro_meta = {
"book": "ttp",
"title": "Transcendental Teachings of Prahlāda Mahārāja",
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
"year": 1972,
"description": "The profound spiritual teachings of the great devotee Prahlāda Mahārāja from the Śrīmad-Bhāgavatam.",
"totalChapters": len(chapters),
"chapters": chapter_listing,
}
intro_path = os.path.join(output_base, "intro.json")
write_json(intro_meta, intro_path)
print(f"\n Written: {intro_path}")
# --- Write intro segments ---
print(f"\n--- Intro Segments ---")
for seg in intro_segments:
filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
write_json(seg, filepath)
print(
f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}"
)
# --- Write chapters ---
print(f"\n--- Chapters ---")
for ch in chapters:
ch_num_str = str(ch["chapter"]).zfill(2)
filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
write_json(ch, filepath)
print(
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
f"{ch['title']}"
)
# --- Summary ---
total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments)
total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
total_paras = total_intro_paras + total_chapter_paras
print(f"\n{'=' * 60}")
print(f" EXTRACTION SUMMARY")
print(f"{'=' * 60}")
print(f" Intro segments: {len(intro_segments)}")
print(f" Chapters: {len(chapters)}")
print(f" Total paragraphs: {total_paras}")
print(f" Intro: {total_intro_paras}")
print(f" Chapters: {total_chapter_paras}")
if chapters:
print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
print(f" Output directory: {output_base}/")
# --- Samples ---
print(f"\n--- Sample: Front Pages ---")
if intro_segments:
front = intro_segments[0]
for i, p in enumerate(front["paragraphs"][:5]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
if chapters:
print(f"\n--- Sample: Chapter 1 ({chapters[0]['title']}) ---")
ch1 = chapters[0]
for i, p in enumerate(ch1["paragraphs"][:5]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
if chapters and len(chapters) > 1:
print(
f"\n--- Sample: Chapter {chapters[-1]['chapter']} ({chapters[-1]['title']}) ---"
)
last = chapters[-1]
for i, p in enumerate(last["paragraphs"][:5]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
# --- Quality Checks ---
print(f"\n--- Quality Checks ---")
# Check all 5 chapters present
expected_chapters = set(range(1, 6))
actual_chapters = set(ch["chapter"] for ch in chapters)
missing = expected_chapters - actual_chapters
extra = actual_chapters - expected_chapters
if missing:
print(f" ⚠ Missing chapters: {sorted(missing)}")
if extra:
print(f" ⚠ Extra chapters: {sorted(extra)}")
if not missing and not extra:
print(f" ✅ All 5 chapters present")
# Check all 1 intro segment present
expected_intros = {"front-pages"}
actual_intros = {s["slug"] for s in intro_segments}
missing_intros = expected_intros - actual_intros
extra_intros = actual_intros - expected_intros
if missing_intros:
print(f" ⚠ Missing intro segments: {missing_intros}")
if extra_intros:
print(f" ⚠ Extra intro segments: {extra_intros}")
if not missing_intros and not extra_intros:
print(f" ✅ All 1 intro segments present")
# Check empty chapters
empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
if empty_chapters:
print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
else:
print(f" ✅ No empty chapters")
# Check empty intro segments
empty_intros = [s["slug"] for s in intro_segments if len(s["paragraphs"]) == 0]
if empty_intros:
print(f" ⚠ Empty intro segments: {empty_intros}")
else:
print(f" ✅ No empty intro segments")
# Check ScaGoudy decoding
print(f"\n--- ScaGoudy Decoding Check ---")
all_text = ""
for ch in chapters:
all_text += " ".join(ch["paragraphs"][:20])
for seg in intro_segments:
all_text += " ".join(seg["paragraphs"][:10])
if "Kṛṣṇa" in all_text:
print(f" ✅ 'Kṛṣṇa' found (ScaGoudy properly decoded)")
else:
print(f" ⚠ 'Kṛṣṇa' not found — ScaGoudy decoding may have failed")
if "Prahlāda" in all_text or "Prahlād" in all_text:
print(f" ✅ 'Prahlāda' found (ScaGoudy properly decoded)")
else:
print(f" ⚠ 'Prahlāda' not found — ScaGoudy decoding may have failed")
if "Prabhupāda" in all_text:
print(f" ✅ 'Prabhupāda' found (ScaGoudy properly decoded)")
# Check for un-decoded ScaGoudy remnants
scagoudy_remnants = 0
for ch in chapters:
for p in ch["paragraphs"]:
if "Kåñëa" in p or "Prabhupäda" in p:
scagoudy_remnants += 1
for seg in intro_segments:
for p in seg["paragraphs"]:
if "Kåñëa" in p or "Prabhupäda" in p:
scagoudy_remnants += 1
if scagoudy_remnants == 0:
print(f" ✅ No ScaGoudy remnants detected")
else:
print(f" ⚠ {scagoudy_remnants} paragraphs with ScaGoudy remnants")
# Check for centered verse blocks
verse_block_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
if '<div style="text-align:center"><i>' in p:
verse_block_count += 1
for seg in intro_segments:
for p in seg["paragraphs"]:
if '<div style="text-align:center"><i>' in p:
verse_block_count += 1
if verse_block_count > 0:
print(f" ✅ {verse_block_count} centered Sanskrit verse blocks found")
else:
print(f" ⚠ No centered verse blocks found — verse grouping may have failed")
# Check for residual span tags
span_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
if 'class="' in p and "<span" in p:
span_count += 1
for seg in intro_segments:
for p in seg["paragraphs"]:
if 'class="' in p and "<span" in p:
span_count += 1
if span_count == 0:
print(f" ✅ No residual span class artifacts found")
else:
print(f" ⚠ {span_count} paragraphs still contain span class artifacts")
# Check for dash-line paragraphs that shouldn't be there
dash_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
plain = re.sub(r"<[^>]+>", "", p)
if is_dash_line(plain):
dash_count += 1
for seg in intro_segments:
for p in seg["paragraphs"]:
plain = re.sub(r"<[^>]+>", "", p)
if is_dash_line(plain):
dash_count += 1
if dash_count == 0:
print(f" ✅ No dashed-line paragraphs in output")
else:
print(f" ⚠ {dash_count} dashed-line paragraphs found in output")
# Show a verse block sample
print(f"\n--- Sample: Verse Block ---")
found_verse = False
for ch in chapters:
for p in ch["paragraphs"]:
if '<div style="text-align:center"><i>' in p:
preview = re.sub(r"<[^>]+>", "", p)[:120]
print(f" {preview}...")
found_verse = True
break
if found_verse:
break
if not found_verse:
for seg in intro_segments:
for p in seg["paragraphs"]:
if '<div style="text-align:center"><i>' in p:
preview = re.sub(r"<[^>]+>", "", p)[:120]
print(f" {preview}...")
found_verse = True
break
if found_verse:
break
if not found_verse:
print(f" (no verse blocks found)")
# Chapter title decoding check
print(f"\n--- Chapter Title Decoding Check ---")
for ch in chapters:
title = ch["title"]
if "å" in title or "ë" in title or "ñ" in title or "ä" in title:
print(f" ⚠ Chapter {ch['chapter']} title has un-decoded ScaGoudy: {title}")
else:
print(f" ✅ Ch {ch['chapter']}: {title}")
print(f"\nDone! ✨")
if __name__ == "__main__":
main()