841 lines
28 KiB
Python
841 lines
28 KiB
Python
#!/usr/bin/env python3
|
||
"""Extract Śrīmad-Bhāgavatam verses from HTML into structured JSON files.
|
||
|
||
Structure: Canto → Chapter → Verse
|
||
Output goes to app/lib/books/sb/
|
||
|
||
The SB HTML uses a different class mapping than BG but shares the same
|
||
ScaGoudy font encoding and the same verse structure:
|
||
Devanagari (skip) → Transliteration pādas → Synonyms → Translation → Purport
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import re
|
||
import sys
|
||
|
||
from bs4 import BeautifulSoup, Tag
|
||
from indic_transliteration import sanscript
|
||
from indic_transliteration.sanscript import transliterate
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# ScaGoudy → IAST Unicode mapping (shared with bg_extract.py)
|
||
# ---------------------------------------------------------------------------
|
||
SCAGOUDY_TO_IAST = {
|
||
"ä": "ā",
|
||
"Ä": "Ā",
|
||
"å": "ṛ",
|
||
"Å": "Ṛ",
|
||
"ñ": "ṣ",
|
||
"Ñ": "Ṣ",
|
||
"ö": "ṭ",
|
||
"Ö": "Ṭ",
|
||
"ë": "ṇ",
|
||
"Ë": "Ṇ",
|
||
"ò": "ḍ",
|
||
"Ò": "Ḍ",
|
||
"ç": "ś",
|
||
"Ç": "Ś",
|
||
"ì": "ṅ",
|
||
"ï": "ñ",
|
||
"ü": "ū",
|
||
"Ü": "Ū",
|
||
"é": "ī",
|
||
"É": "Ī",
|
||
"à": "ṁ",
|
||
"ù": "ḥ",
|
||
"è": "ṝ", # long vocalic r (U+1E5D)
|
||
"û": "ṁl", # anusvara + l (ScaGoudy ligature glyph)
|
||
"ÿ": "ḷ", # vocalic l (U+1E37)
|
||
}
|
||
|
||
|
||
def decode_scagoudy(text: str) -> str:
|
||
"""Convert ScaGoudy-encoded text to proper IAST Unicode."""
|
||
for src, dst in SCAGOUDY_TO_IAST.items():
|
||
text = text.replace(src, dst)
|
||
return text
|
||
|
||
|
||
def decode_html_content(tag: Tag) -> str:
|
||
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
|
||
inner = "".join(str(child) for child in tag.children)
|
||
inner = decode_scagoudy(inner)
|
||
# Remove span tags with CSS classes (artifacts from RTF conversion)
|
||
inner = re.sub(r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner)
|
||
inner = re.sub(r"\s+", " ", inner).strip()
|
||
return inner
|
||
|
||
|
||
def decode_text(tag: Tag) -> str:
|
||
"""Extract plain text from a tag and decode ScaGoudy."""
|
||
text = tag.get_text()
|
||
return decode_scagoudy(text).strip()
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Devanagari helpers (same as bg_extract.py)
|
||
# ---------------------------------------------------------------------------
|
||
def normalize_devanagari_anusvara(text: str) -> str:
|
||
"""Replace explicit class nasals before same-class consonants with anusvāra."""
|
||
replacements = [
|
||
(r"ङ्([कखगघ])", r"ं\1"),
|
||
(r"ञ्([चछजझ])", r"ं\1"),
|
||
]
|
||
for pattern, repl in replacements:
|
||
text = re.sub(pattern, repl, text)
|
||
return text
|
||
|
||
|
||
def join_virama_words(text: str) -> str:
|
||
"""Join Devanagari words where the previous word ends with virāma.
|
||
|
||
Handles both virāma+consonant and virāma+vowel cases.
|
||
In Devanagari script, a halant (virāma) consonant always absorbs the
|
||
following letter — whether consonant or vowel — without a space.
|
||
|
||
virāma + consonant: श्रीर् विजयो → श्रीर्विजयो
|
||
virāma + vowel: तैर् अलातायुधैः → तैरलातायुधैः
|
||
"""
|
||
# Join virāma + space + consonant (क U+0915 through ह U+0939)
|
||
# Use [ \t]+ instead of \s+ to avoid joining across newlines (line breaks
|
||
# separate paired pādas and must be preserved).
|
||
text = re.sub(r"्[ \t]+([क-ह])", r"्\1", text)
|
||
# Join virāma + space + independent vowel (अ U+0905 through औ U+0914, plus ॠ ॡ)
|
||
# The transliterator will have produced virāma on the consonant and a separate
|
||
# independent vowel glyph. We need to:
|
||
# 1. Drop the virāma (the consonant takes the vowel)
|
||
# 2. Replace the independent vowel with its dependent mātrā form
|
||
# However, a simpler approach: just remove the space and let the
|
||
# virāma+independent-vowel stay — this is visually wrong.
|
||
# Instead, we do the proper mapping:
|
||
_VOWEL_TO_MATRA = {
|
||
"अ": "", # inherent vowel — just drop virāma and the अ
|
||
"आ": "ा",
|
||
"इ": "ि",
|
||
"ई": "ी",
|
||
"उ": "ु",
|
||
"ऊ": "ू",
|
||
"ऋ": "ृ",
|
||
"ॠ": "ॄ",
|
||
"ऌ": "ॢ",
|
||
"ॡ": "ॣ",
|
||
"ए": "े",
|
||
"ऐ": "ै",
|
||
"ओ": "ो",
|
||
"औ": "ौ",
|
||
}
|
||
|
||
def _virama_vowel_replace(m):
|
||
vowel = m.group(1)
|
||
matra = _VOWEL_TO_MATRA.get(vowel, vowel)
|
||
# Remove the virāma (्) and replace independent vowel with mātrā
|
||
return matra
|
||
|
||
text = re.sub(r"्[ \t]+([अ-औॠॡ])", _virama_vowel_replace, text)
|
||
return text
|
||
|
||
|
||
def iast_to_devanagari(iast_text: str) -> str:
|
||
"""Convert IAST transliteration to Devanagari script."""
|
||
lines = iast_text.split("\n")
|
||
result_lines = []
|
||
for line in lines:
|
||
clean = line.replace("-", "")
|
||
clean = clean.replace("ṁ", "ṃ")
|
||
deva = transliterate(clean, sanscript.IAST, sanscript.DEVANAGARI)
|
||
deva = normalize_devanagari_anusvara(deva)
|
||
deva = join_virama_words(deva)
|
||
result_lines.append(deva)
|
||
return "\n".join(result_lines)
|
||
|
||
|
||
def pair_padas(padas: list) -> str:
|
||
"""Pair pādas into verse lines (2 pādas per line)."""
|
||
if not padas:
|
||
return ""
|
||
lines = []
|
||
i = 0
|
||
if len(padas) % 2 == 1:
|
||
lines.append(padas[0])
|
||
i = 1
|
||
while i < len(padas) - 1:
|
||
lines.append(padas[i] + " " + padas[i + 1])
|
||
i += 2
|
||
if i < len(padas):
|
||
lines.append(padas[i])
|
||
return "\n".join(lines)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Synonym parser (same as bg_extract.py)
|
||
# ---------------------------------------------------------------------------
|
||
def parse_synonyms(text: str) -> list:
|
||
"""Parse synonym text into structured array."""
|
||
result = []
|
||
entries = text.split(";")
|
||
for entry in entries:
|
||
entry = entry.strip()
|
||
if not entry:
|
||
continue
|
||
if "—" in entry:
|
||
parts = entry.split("—", 1)
|
||
elif "--" in entry:
|
||
parts = entry.split("--", 1)
|
||
else:
|
||
continue
|
||
if len(parts) == 2:
|
||
word = parts[0].strip().strip("., ")
|
||
meaning = parts[1].strip()
|
||
if word and meaning:
|
||
result.append({"word": word, "meanings": [meaning]})
|
||
return result
|
||
|
||
|
||
def parse_synonym_html(tag: Tag) -> list:
|
||
text = decode_text(tag)
|
||
return parse_synonyms(text)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# CSS class helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# All Devanagari classes (Indevr + RM Devanagari) — content to skip
|
||
DEVANAGARI_CLASSES = frozenset(
|
||
[
|
||
"p23",
|
||
"p81",
|
||
"p91",
|
||
"p101",
|
||
"p102",
|
||
"p105",
|
||
"p113", # Indevr
|
||
"p126",
|
||
"p128",
|
||
"p129", # RM Devanagari
|
||
]
|
||
)
|
||
|
||
# Transliteration pāda classes (center-aligned italic ScaGoudy)
|
||
TRANSLITERATION_CLASSES = frozenset(["p24"])
|
||
|
||
# Classes that are sometimes transliteration pādas (when they appear between
|
||
# Devanagari and synonyms) but otherwise are embedded verse quotes in purport.
|
||
# p41 and p92 are used for prose-style transliterations (common in Canto 5).
|
||
# p59 is a rare center-aligned variant also used for transliteration.
|
||
# p17 is primarily a purport class (780 uses) but occasionally holds transliteration
|
||
# pādas (e.g. SB 5.24.7, SB 6.19.8) — the state machine ensures it's only
|
||
# captured as transliteration when between Devanagari and synonyms.
|
||
# p114 is an indented ScaGoudy paragraph used once for a transliteration pāda.
|
||
MAYBE_TRANSLITERATION_CLASSES = frozenset(["p33", "p41", "p92", "p59", "p17", "p114"])
|
||
|
||
# Synonym classes (color #000080)
|
||
# p117 is a rare variant (2 occurrences) with same margins as p25 but no color
|
||
SYNONYM_CLASSES = frozenset(["p25", "p124", "p117"])
|
||
|
||
# Translation/purport section header classes
|
||
SECTION_HEADER_CLASSES = frozenset(["p26", "p28"])
|
||
|
||
# Translation text classes (color #800000, bold)
|
||
# p108 is maroon-colored ScaGoudy (3 occurrences) used as translation text
|
||
# p118 is a rare variant (1 occurrence) with same margins as p27 but no color
|
||
TRANSLATION_CLASSES = frozenset(["p27", "p98", "p108", "p118"])
|
||
|
||
# Purport paragraph classes — a wide set because the HTML uses many variants
|
||
PURPORT_CLASSES = frozenset(
|
||
[
|
||
"p29",
|
||
"p14",
|
||
"p17",
|
||
"p18",
|
||
"p30",
|
||
"p11",
|
||
"p45",
|
||
"p67",
|
||
"p77",
|
||
"p80",
|
||
"p117",
|
||
"p118",
|
||
"p119",
|
||
"p84",
|
||
"p79",
|
||
"p133",
|
||
"p135",
|
||
"p136",
|
||
"p62",
|
||
"p44",
|
||
"p83",
|
||
"p96", # chapter summaries (inside Summary entries)
|
||
"p5", # general text paragraphs (used in purport/summary context)
|
||
]
|
||
)
|
||
|
||
# Embedded verse quotations inside purport (center-aligned) — include in purport
|
||
PURPORT_VERSE_CLASSES = frozenset(
|
||
[
|
||
"p46",
|
||
"p33",
|
||
"p41",
|
||
"p12",
|
||
"p13", # verse quotes & numbered lists in purport
|
||
"p15",
|
||
"p47",
|
||
"p55",
|
||
"p56",
|
||
"p58", # center-aligned purport variants
|
||
"p66",
|
||
"p68", # center-aligned purport variants
|
||
"p34", # small center-aligned text in purport
|
||
"p69", # purport paragraph variant
|
||
"p93", # purport paragraph variant
|
||
"p107", # purport paragraph variant
|
||
]
|
||
)
|
||
|
||
# End-of-verse citation references (green, e.g. "[Bhāg. 1962:1.1.1]")
|
||
CITATION_CLASSES = frozenset(["p31"])
|
||
|
||
# Chapter-end markers ("Thus end the Bhaktivedanta purports…")
|
||
CHAPTER_END_CLASS = "p32"
|
||
|
||
# Chapter header bookmark (e.g. "SB 1.1: Questions by the Sages")
|
||
CHAPTER_BOOKMARK_CLASS = "p8"
|
||
|
||
# Chapter heading (e.g. "CHAPTER ONE\nQuestions by the Sages")
|
||
CHAPTER_HEADING_CLASS = "p19"
|
||
|
||
# Verse reference marker (e.g. "SB 1.1.1" or "SB. 6.1.1")
|
||
VERSE_REF_CLASS = "p20"
|
||
|
||
# TEXT N header
|
||
TEXT_HEADER_CLASS = "p21"
|
||
|
||
# "TEXT" label
|
||
TEXT_LABEL_CLASS = "p22"
|
||
|
||
# Canto/volume boundary markers — skip
|
||
CANTO_MARKER_CLASSES = frozenset(["p3", "p74"])
|
||
|
||
# Section title classes that are NOT verse content — skip
|
||
SKIP_CLASSES = frozenset(
|
||
[
|
||
"p4",
|
||
"p9",
|
||
"p49",
|
||
"p50", # canto/volume headers, dedication, glossary
|
||
"p10",
|
||
"p35",
|
||
"p95",
|
||
"p134", # decorative / Times font
|
||
"p48", # glossary/author markers
|
||
"p22", # "TEXT" label — we use p21 for TEXT N
|
||
]
|
||
)
|
||
|
||
|
||
def get_css_class(tag: Tag) -> str:
|
||
"""Get the primary CSS class of a tag."""
|
||
classes = tag.get("class", [])
|
||
return classes[0] if classes else ""
|
||
|
||
|
||
def _tag_uses_devanagari_font(tag: Tag) -> bool:
|
||
"""Check if a tag's CSS class is a known Devanagari class."""
|
||
return get_css_class(tag) in DEVANAGARI_CLASSES
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Reference parsing
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# Matches "SB 1.1.1" or "SB. 6.1.1" — captures canto, chapter, verse
|
||
_VERSE_REF_RE = re.compile(r"SB\.?\s*(\d+)\.(\d+)\.(\d+)")
|
||
# Matches verse range like "SB 1.4.17-18" — captures start and end verse nums
|
||
_VERSE_RANGE_RE = re.compile(r"SB\.?\s*\d+\.\d+\.(\d+)\s*-\s*(\d+)")
|
||
# Matches chapter reference like "SB 1.1:" or "SB. 6.1:"
|
||
_CHAPTER_REF_RE = re.compile(r"SB\.?\s*(\d+)\.(\d+)\s*:")
|
||
# Matches summary reference like "SB 5.1 Summary"
|
||
_SUMMARY_RE = re.compile(r"SB\.?\s*(\d+)\.(\d+)\s+Summary", re.IGNORECASE)
|
||
# Matches invocation reference like "SB 2.1.invo"
|
||
_INVO_RE = re.compile(r"SB\.?\s*(\d+)\.(\d+)\.invo", re.IGNORECASE)
|
||
|
||
|
||
def _parse_verse_ref(text: str):
|
||
"""Parse a verse reference string.
|
||
|
||
Returns (canto, chapter, verse, verse_end, clean_ref, ref_type) or None.
|
||
ref_type is one of: 'verse', 'summary', 'invocation'
|
||
"""
|
||
text = text.strip()
|
||
|
||
# Check for Summary
|
||
m = _SUMMARY_RE.search(text)
|
||
if m:
|
||
canto = int(m.group(1))
|
||
chapter = int(m.group(2))
|
||
return (canto, chapter, 0, None, f"SB {canto}.{chapter} Summary", "summary")
|
||
|
||
# Check for Invocation
|
||
m = _INVO_RE.search(text)
|
||
if m:
|
||
canto = int(m.group(1))
|
||
chapter = int(m.group(2))
|
||
return (
|
||
canto,
|
||
chapter,
|
||
0,
|
||
None,
|
||
f"SB {canto}.{chapter} Invocation",
|
||
"invocation",
|
||
)
|
||
|
||
# Check for verse range in the reference text
|
||
# The ref may contain multiple individual refs then a range at the end:
|
||
# "SB 1.4.17, SB 1.4.18, SB 1.4.17-18"
|
||
# Or just a plain range: "SB 10.1.5-7"
|
||
# We want the last meaningful reference.
|
||
verse_refs = list(_VERSE_REF_RE.finditer(text))
|
||
if not verse_refs:
|
||
return None
|
||
|
||
# Use the first match for canto/chapter/verse
|
||
first = verse_refs[0]
|
||
canto = int(first.group(1))
|
||
chapter = int(first.group(2))
|
||
verse = int(first.group(3))
|
||
|
||
# Check for range
|
||
verse_end = None
|
||
range_match = _VERSE_RANGE_RE.search(text)
|
||
if range_match:
|
||
verse_end = int(range_match.group(2))
|
||
# The first verse number from the range
|
||
verse = int(range_match.group(1))
|
||
|
||
# Build clean reference
|
||
if verse_end is not None:
|
||
clean_ref = f"SB {canto}.{chapter}.{verse}-{verse_end}"
|
||
else:
|
||
clean_ref = f"SB {canto}.{chapter}.{verse}"
|
||
|
||
return (canto, chapter, verse, verse_end, clean_ref, "verse")
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Chapter title parsing from p8 tags
|
||
# ---------------------------------------------------------------------------
|
||
def _parse_chapter_bookmark(text: str):
|
||
"""Parse a chapter bookmark like 'SB 1.1: Questions by the Sages'.
|
||
|
||
Returns (canto, chapter, title) or None.
|
||
"""
|
||
text = text.strip()
|
||
m = re.match(r"SB\.?\s*(\d+)\.(\d+)\s*:\s*(.+)", text)
|
||
if m:
|
||
canto = int(m.group(1))
|
||
chapter = int(m.group(2))
|
||
title = decode_scagoudy(m.group(3).strip())
|
||
return (canto, chapter, title)
|
||
return None
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# State machine
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def _finalize_verse(verse: dict):
|
||
"""Build transliteration and Devanagari from pādas. Remove internal fields."""
|
||
padas = verse.get("_padas", [])
|
||
verse["transliteration"] = "\n".join(padas)
|
||
if padas:
|
||
verse["devanagari"] = iast_to_devanagari(pair_padas(padas))
|
||
else:
|
||
verse["devanagari"] = ""
|
||
if "_padas" in verse:
|
||
del verse["_padas"]
|
||
if "_state" in verse:
|
||
del verse["_state"]
|
||
if "_ref_type" in verse:
|
||
del verse["_ref_type"]
|
||
|
||
|
||
def extract_verses(soup: BeautifulSoup, max_canto=99, max_chapter=999, max_verse=9999):
|
||
"""Extract verses from parsed SB HTML using a state machine.
|
||
|
||
Returns (verses, chapter_info) where chapter_info is:
|
||
{ (canto, chapter): title_string }
|
||
"""
|
||
paragraphs = soup.find_all("p")
|
||
|
||
verses = []
|
||
current_verse = None
|
||
state = "IDLE" # IDLE, DEVANAGARI, TRANSLITERATION, SYNONYMS,
|
||
# TRANSLATION, PURPORT, SUMMARY
|
||
|
||
chapter_titles = {} # (canto, chapter) -> title
|
||
|
||
for p in paragraphs:
|
||
cls = get_css_class(p)
|
||
text = p.get_text().strip()
|
||
|
||
# --- Chapter bookmark (p8) ---
|
||
if cls == CHAPTER_BOOKMARK_CLASS:
|
||
parsed = _parse_chapter_bookmark(text)
|
||
if parsed:
|
||
canto, chapter, title = parsed
|
||
chapter_titles[(canto, chapter)] = title
|
||
# Don't change state — bookmarks appear before the chapter heading
|
||
continue
|
||
|
||
# --- Canto / volume markers — skip ---
|
||
if cls in CANTO_MARKER_CLASSES or cls in SKIP_CLASSES:
|
||
continue
|
||
|
||
# --- Chapter heading (p19) — skip ---
|
||
if cls == CHAPTER_HEADING_CLASS:
|
||
continue
|
||
|
||
# --- Verse reference (p20) ---
|
||
if cls == VERSE_REF_CLASS:
|
||
# Finalize previous verse
|
||
if current_verse and current_verse.get("_ref_type") != "summary":
|
||
_finalize_verse(current_verse)
|
||
verses.append(current_verse)
|
||
elif current_verse and current_verse.get("_ref_type") == "summary":
|
||
# Discard summary entries — they're chapter preambles, not verses
|
||
pass
|
||
current_verse = None
|
||
state = "IDLE"
|
||
|
||
ref = _parse_verse_ref(text)
|
||
if ref is None:
|
||
continue
|
||
|
||
canto, chapter, verse_num, verse_end, clean_ref, ref_type = ref
|
||
|
||
if canto > max_canto:
|
||
break
|
||
if canto == max_canto and chapter > max_chapter:
|
||
break
|
||
if canto == max_canto and chapter == max_chapter and verse_num > max_verse:
|
||
break
|
||
|
||
if ref_type == "summary":
|
||
# Start collecting summary text but mark it so we skip it
|
||
current_verse = {
|
||
"_ref_type": "summary",
|
||
"_state": "SUMMARY",
|
||
}
|
||
state = "SUMMARY"
|
||
continue
|
||
|
||
current_verse = {
|
||
"book_title": "Śrīmad-Bhāgavatam",
|
||
"canto": canto,
|
||
"chapter": chapter,
|
||
"verse": verse_num,
|
||
"reference": clean_ref,
|
||
"devanagari": "",
|
||
"transliteration": "",
|
||
"_padas": [],
|
||
"synonyms": [],
|
||
"translation": "",
|
||
"purport": [],
|
||
"_state": "VERSE_START",
|
||
"_ref_type": ref_type,
|
||
}
|
||
if verse_end is not None:
|
||
current_verse["verse_end"] = verse_end
|
||
state = "VERSE_START"
|
||
continue
|
||
|
||
# --- Summary state: just skip everything until next p20 ---
|
||
if state == "SUMMARY":
|
||
continue
|
||
|
||
# --- TEXT N header (p21) — authoritative verse number ---
|
||
# The p20 reference sometimes has wrong verse numbers (duplicates).
|
||
# The p21 TEXT header always has the correct verse number.
|
||
if cls == TEXT_HEADER_CLASS and current_verse:
|
||
m = re.match(r"TEXTS?\s+(\d+)(?:\s*[-–]\s*(\d+))?", text, re.IGNORECASE)
|
||
if m:
|
||
correct_verse = int(m.group(1))
|
||
correct_end = int(m.group(2)) if m.group(2) else None
|
||
|
||
# Fix the verse number if p20 was wrong
|
||
if current_verse["verse"] != correct_verse:
|
||
current_verse["verse"] = correct_verse
|
||
|
||
# Fix verse_end
|
||
if correct_end is not None:
|
||
current_verse["verse_end"] = correct_end
|
||
elif (
|
||
"verse_end" in current_verse
|
||
and current_verse.get("verse_end") != correct_end
|
||
):
|
||
if correct_end is None and "verse_end" in current_verse:
|
||
del current_verse["verse_end"]
|
||
|
||
# Rebuild the clean reference with corrected numbers
|
||
canto = current_verse["canto"]
|
||
chapter = current_verse["chapter"]
|
||
if correct_end is not None:
|
||
current_verse["reference"] = (
|
||
f"SB {canto}.{chapter}.{correct_verse}-{correct_end}"
|
||
)
|
||
else:
|
||
current_verse["reference"] = f"SB {canto}.{chapter}.{correct_verse}"
|
||
continue
|
||
|
||
# --- TEXT label (p22) ---
|
||
if cls == TEXT_LABEL_CLASS:
|
||
continue
|
||
|
||
# --- Devanagari (Indevr / RM Devanagari) — SKIP, derive from transliteration ---
|
||
if cls in DEVANAGARI_CLASSES and current_verse:
|
||
state = "DEVANAGARI"
|
||
current_verse["_state"] = state
|
||
continue
|
||
|
||
# --- Transliteration pādas (p24) ---
|
||
if (
|
||
cls in TRANSLITERATION_CLASSES
|
||
and current_verse
|
||
and state in ("VERSE_START", "DEVANAGARI", "TRANSLITERATION")
|
||
):
|
||
state = "TRANSLITERATION"
|
||
current_verse["_state"] = state
|
||
line = decode_text(p)
|
||
if line:
|
||
current_verse["_padas"].append(line)
|
||
continue
|
||
|
||
# --- Maybe-transliteration (p33) — only if we're in transliteration context ---
|
||
if (
|
||
cls in MAYBE_TRANSLITERATION_CLASSES
|
||
and current_verse
|
||
and state in ("DEVANAGARI", "TRANSLITERATION")
|
||
):
|
||
state = "TRANSLITERATION"
|
||
current_verse["_state"] = state
|
||
line = decode_text(p)
|
||
if line:
|
||
current_verse["_padas"].append(line)
|
||
continue
|
||
|
||
# --- Synonyms (p25, p124) ---
|
||
if cls in SYNONYM_CLASSES and current_verse:
|
||
state = "SYNONYMS"
|
||
current_verse["_state"] = state
|
||
current_verse["synonyms"] = parse_synonym_html(p)
|
||
continue
|
||
|
||
# --- Section headers: TRANSLATION / PURPORT (p26, p28) ---
|
||
if cls in SECTION_HEADER_CLASSES and current_verse:
|
||
header_text = text.strip().upper()
|
||
if "TRANSLATION" in header_text:
|
||
state = "TRANSLATION"
|
||
current_verse["_state"] = state
|
||
elif "PURPORT" in header_text:
|
||
state = "PURPORT"
|
||
current_verse["_state"] = state
|
||
continue
|
||
|
||
# --- Translation text (p27, p98) ---
|
||
# Also handle mislabeled headers: in some verses (e.g. SB 10.4.6, 10.4.7)
|
||
# the TRANSLATION header says "PURPORT", so state may be PURPORT but
|
||
# translation is still empty. If we see a p27 tag and translation is empty,
|
||
# capture it as translation regardless of state.
|
||
if (
|
||
cls in TRANSLATION_CLASSES
|
||
and current_verse
|
||
and (
|
||
state == "TRANSLATION"
|
||
or (state == "PURPORT" and not current_verse.get("translation"))
|
||
)
|
||
):
|
||
current_verse["translation"] = decode_html_content(p)
|
||
state = "POST_TRANSLATION"
|
||
current_verse["_state"] = state
|
||
continue
|
||
|
||
# --- p71 is blue-colored translation (miscolored — treat as translation) ---
|
||
if cls == "p71" and current_verse and state == "TRANSLATION":
|
||
current_verse["translation"] = decode_html_content(p)
|
||
state = "POST_TRANSLATION"
|
||
current_verse["_state"] = state
|
||
continue
|
||
|
||
# --- Purport paragraphs ---
|
||
if state == "PURPORT" and current_verse and cls in PURPORT_CLASSES:
|
||
content = decode_html_content(p)
|
||
if content and "Thus end the Bhaktivedanta" not in text:
|
||
current_verse["purport"].append(content)
|
||
continue
|
||
|
||
# --- Embedded verse quotations inside purport ---
|
||
if state == "PURPORT" and current_verse and cls in PURPORT_VERSE_CLASSES:
|
||
content = decode_html_content(p)
|
||
if content:
|
||
current_verse["purport"].append(content)
|
||
continue
|
||
|
||
# --- Citation / end-of-verse reference (p31) ---
|
||
if cls in CITATION_CLASSES and current_verse:
|
||
# These mark the end of a verse's content but we don't store them
|
||
# The verse continues to the next p20
|
||
continue
|
||
|
||
# --- Chapter end marker (p32) ---
|
||
if cls == CHAPTER_END_CLASS:
|
||
if current_verse and current_verse.get("_ref_type") not in ("summary",):
|
||
_finalize_verse(current_verse)
|
||
verses.append(current_verse)
|
||
current_verse = None
|
||
state = "IDLE"
|
||
continue
|
||
|
||
# Don't forget the last verse
|
||
if current_verse and current_verse.get("_ref_type") not in ("summary",):
|
||
_finalize_verse(current_verse)
|
||
verses.append(current_verse)
|
||
|
||
return verses, chapter_titles
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# JSON output
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def write_json(data, filepath):
|
||
"""Write JSON with proper Unicode and formatting."""
|
||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||
with open(filepath, "w", encoding="utf-8") as f:
|
||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||
print(f" Written: {filepath}")
|
||
|
||
|
||
def main():
|
||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
html_path = os.path.join(base_dir, "public", "html", "sb.html")
|
||
output_dir = os.path.join(base_dir, "lib", "jsons", "books", "sb")
|
||
|
||
print(f"Reading {html_path}...")
|
||
with open(html_path, "r", encoding="utf-8") as f:
|
||
soup = BeautifulSoup(f, "html.parser")
|
||
|
||
print("Extracting all SB verses...")
|
||
verses, chapter_titles = extract_verses(soup)
|
||
|
||
print(f"\nExtracted {len(verses)} verses across {len(chapter_titles)} chapters")
|
||
|
||
# Group by (canto, chapter) to build intro.json
|
||
cantos_info = {} # canto -> { chapter -> verse_count }
|
||
for verse in verses:
|
||
c = verse["canto"]
|
||
ch = verse["chapter"]
|
||
if c not in cantos_info:
|
||
cantos_info[c] = {}
|
||
if ch not in cantos_info[c]:
|
||
cantos_info[c][ch] = 0
|
||
cantos_info[c][ch] += 1
|
||
|
||
# Canto names
|
||
canto_names = {
|
||
1: "Creation",
|
||
2: "The Cosmic Manifestation",
|
||
3: "The Status Quo",
|
||
4: "The Creation of the Fourth Order",
|
||
5: "The Creative Impetus",
|
||
6: "Prescribed Duties for Mankind",
|
||
7: "The Science of God",
|
||
8: "Withdrawal of the Cosmic Creations",
|
||
9: "Liberation",
|
||
10: "The Summum Bonum",
|
||
}
|
||
|
||
# Build intro.json
|
||
total_cantos = max(cantos_info.keys()) if cantos_info else 0
|
||
cantos_list = []
|
||
for canto_num in sorted(cantos_info.keys()):
|
||
chapters_in_canto = cantos_info[canto_num]
|
||
chapters_list = []
|
||
for ch_num in sorted(chapters_in_canto.keys()):
|
||
title = chapter_titles.get((canto_num, ch_num), "")
|
||
chapters_list.append(
|
||
{
|
||
"number": ch_num,
|
||
"title": title,
|
||
"totalVerses": chapters_in_canto[ch_num],
|
||
}
|
||
)
|
||
cantos_list.append(
|
||
{
|
||
"number": canto_num,
|
||
"title": canto_names.get(canto_num, f"Canto {canto_num}"),
|
||
"totalChapters": len(chapters_list),
|
||
"chapters": chapters_list,
|
||
}
|
||
)
|
||
|
||
intro = {
|
||
"book": "sb",
|
||
"title": "Śrīmad-Bhāgavatam",
|
||
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
||
"totalCantos": total_cantos,
|
||
"cantos": cantos_list,
|
||
}
|
||
write_json(intro, os.path.join(output_dir, "intro.json"))
|
||
|
||
# Write verse JSON files:
|
||
# app/lib/books/sb/cantos/CC/chapters/CH/verses/VV.json
|
||
for verse in verses:
|
||
canto = str(verse["canto"]).zfill(2)
|
||
ch = str(verse["chapter"]).zfill(2)
|
||
vs = str(verse["verse"]).zfill(2)
|
||
filepath = os.path.join(
|
||
output_dir, "cantos", canto, "chapters", ch, "verses", f"{vs}.json"
|
||
)
|
||
write_json(verse, filepath)
|
||
|
||
# Print summary for verification
|
||
print("\n--- Verification ---")
|
||
for canto_num in sorted(cantos_info.keys()):
|
||
chapters_in_canto = cantos_info[canto_num]
|
||
total_verses_in_canto = sum(chapters_in_canto.values())
|
||
print(
|
||
f"\nCanto {canto_num}: {len(chapters_in_canto)} chapters, {total_verses_in_canto} verses"
|
||
)
|
||
for ch_num in sorted(chapters_in_canto.keys()):
|
||
ch_verses = [
|
||
v for v in verses if v["canto"] == canto_num and v["chapter"] == ch_num
|
||
]
|
||
if ch_verses:
|
||
first = ch_verses[0]
|
||
last = ch_verses[-1]
|
||
title = chapter_titles.get((canto_num, ch_num), "")
|
||
print(
|
||
f" Ch {ch_num:>2}: {len(ch_verses):>3} verses "
|
||
f"({first['reference']} → {last['reference']}) {title}"
|
||
)
|
||
|
||
# Quick quality check: show first verse details
|
||
if verses:
|
||
print("\n--- Sample (first verse) ---")
|
||
v = verses[0]
|
||
print(f"Reference: {v['reference']}")
|
||
print(f"Devanagari:\n{v['devanagari']}")
|
||
print(f"Transliteration:\n{v['transliteration']}")
|
||
print(f"Synonyms: {len(v['synonyms'])} entries")
|
||
if v["synonyms"]:
|
||
print(f" First: {v['synonyms'][0]}")
|
||
print(f"Translation: {v['translation'][:120]}...")
|
||
print(f"Purport: {len(v['purport'])} paragraphs")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|