#!/usr/bin/env python3 """Extract Caitanya-caritāmṛta verses from HTML into structured JSON files. Structure: Līlā (Ādi/Madhya/Antya) → Chapter → Verse Output goes to app/lib/books/cc/ The CC HTML uses a different class mapping than BG/SB but shares the same ScaGoudy font encoding and the same verse structure: Bengali (skip) → Transliteration pādas → Synonyms → Translation → Purport """ import json import os import re import sys from bs4 import BeautifulSoup, Tag from indic_transliteration import sanscript from indic_transliteration.sanscript import transliterate # --------------------------------------------------------------------------- # ScaGoudy → IAST Unicode mapping (shared with bg/sb scripts) # --------------------------------------------------------------------------- SCAGOUDY_TO_IAST = { "ä": "ā", "Ä": "Ā", "å": "ṛ", "Å": "Ṛ", "ñ": "ṣ", "Ñ": "Ṣ", "ö": "ṭ", "Ö": "Ṭ", "ë": "ṇ", "Ë": "Ṇ", "ò": "ḍ", "Ò": "Ḍ", "ç": "ś", "Ç": "Ś", "ì": "ṅ", "ï": "ñ", "ü": "ū", "Ü": "Ū", "é": "ī", "É": "Ī", "à": "ṁ", "ù": "ḥ", "è": "ṝ", # long vocalic r (U+1E5D) "û": "ṁl", # anusvara + l (ScaGoudy ligature glyph) "ÿ": "ḷ", # vocalic l (U+1E37) } def decode_scagoudy(text: str) -> str: """Convert ScaGoudy-encoded text to proper IAST Unicode.""" for src, dst in SCAGOUDY_TO_IAST.items(): text = text.replace(src, dst) return text def decode_html_content(tag: Tag) -> str: """Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags.""" inner = "".join(str(child) for child in tag.children) inner = decode_scagoudy(inner) inner = re.sub(r']*class="[^"]*"[^>]*>(.*?)', r"\1", inner) inner = re.sub(r"\s+", " ", inner).strip() return inner def decode_text(tag: Tag) -> str: """Extract plain text from a tag and decode ScaGoudy.""" text = tag.get_text() return decode_scagoudy(text).strip() # --------------------------------------------------------------------------- # Devanagari helpers (same as bg/sb scripts) # --------------------------------------------------------------------------- def normalize_devanagari_anusvara(text: str) -> str: """Replace explicit class nasals before same-class consonants with anusvāra.""" replacements = [ (r"ङ्([कखगघ])", r"ं\1"), (r"ञ्([चछजझ])", r"ं\1"), ] for pattern, repl in replacements: text = re.sub(pattern, repl, text) return text def join_virama_words(text: str) -> str: """Join Devanagari words where the previous word ends with virāma. Handles both virāma+consonant and virāma+vowel cases. Uses [ \\t]+ instead of \\s+ to avoid joining across newlines. """ text = re.sub(r"्[ \t]+([क-ह])", r"्\1", text) _VOWEL_TO_MATRA = { "अ": "", "आ": "ा", "इ": "ि", "ई": "ी", "उ": "ु", "ऊ": "ू", "ऋ": "ृ", "ॠ": "ॄ", "ऌ": "ॢ", "ॡ": "ॣ", "ए": "े", "ऐ": "ै", "ओ": "ो", "औ": "ौ", } def _virama_vowel_replace(m): vowel = m.group(1) return _VOWEL_TO_MATRA.get(vowel, vowel) text = re.sub(r"्[ \t]+([अ-औॠॡ])", _virama_vowel_replace, text) return text def iast_to_devanagari(iast_text: str) -> str: """Convert IAST transliteration to Devanagari script.""" lines = iast_text.split("\n") result_lines = [] for line in lines: clean = line.replace("-", "") clean = clean.replace("ṁ", "ṃ") deva = transliterate(clean, sanscript.IAST, sanscript.DEVANAGARI) deva = normalize_devanagari_anusvara(deva) deva = join_virama_words(deva) result_lines.append(deva) return "\n".join(result_lines) def iast_to_bengali(iast_text: str) -> str: """Convert IAST transliteration to Bengali script.""" lines = iast_text.split("\n") result_lines = [] for line in lines: clean = line.replace("-", "") clean = clean.replace("ṁ", "ṃ") beng = transliterate(clean, sanscript.IAST, sanscript.BENGALI) # Normalize anusvāra for Bengali (same classes as Devanagari) beng = re.sub(r"ঙ্([কখগঘ])", r"ং\1", beng) beng = re.sub(r"ঞ্([চছজঝ])", r"ং\1", beng) # Join virama words for Bengali (hasanta = ্) beng = re.sub(r"্[ \t]+([ক-হ])", r"্\1", beng) _BN_VOWEL_TO_MATRA = { "অ": "", "আ": "া", "ই": "ি", "ঈ": "ী", "উ": "ু", "ঊ": "ূ", "ঋ": "ৃ", "ৠ": "ৄ", "ঌ": "ৢ", "ৡ": "ৣ", "এ": "ে", "ঐ": "ৈ", "ও": "ো", "ঔ": "ৌ", } def _bn_virama_vowel_replace(m): vowel = m.group(1) return _BN_VOWEL_TO_MATRA.get(vowel, vowel) beng = re.sub(r"্[ \t]+([অ-ঔৠৡ])", _bn_virama_vowel_replace, beng) result_lines.append(beng) return "\n".join(result_lines) def pair_padas(padas: list) -> str: """Pair pādas into verse lines (2 pādas per line).""" if not padas: return "" lines = [] i = 0 if len(padas) % 2 == 1: lines.append(padas[0]) i = 1 while i < len(padas) - 1: lines.append(padas[i] + " " + padas[i + 1]) i += 2 if i < len(padas): lines.append(padas[i]) return "\n".join(lines) # --------------------------------------------------------------------------- # Synonym parser # --------------------------------------------------------------------------- def parse_synonyms(text: str) -> list: """Parse synonym text into structured array.""" result = [] entries = text.split(";") for entry in entries: entry = entry.strip() if not entry: continue if "—" in entry: parts = entry.split("—", 1) elif "--" in entry: parts = entry.split("--", 1) else: continue if len(parts) == 2: word = parts[0].strip().strip("., ") meaning = parts[1].strip() if word and meaning: result.append({"word": word, "meanings": [meaning]}) return result def parse_synonym_html(tag: Tag) -> list: text = decode_text(tag) return parse_synonyms(text) # --------------------------------------------------------------------------- # CSS class sets for CC # --------------------------------------------------------------------------- # Bengali font classes — content to skip (derive from transliteration) BENGALI_CLASSES = frozenset( [ "p23", "p29", "p35", "p61", "p62", "p63", "p64", "p65", "p66", "p67", "p68", "p69", "p70", "p72", "p73", "p75", "p83", "p84", "p87", "p88", ] ) # Transliteration pāda classes TRANSLITERATION_CLASSES = frozenset(["p24"]) # Maybe-transliteration: center-aligned italic that can be transliteration # or embedded verse quotes in purport depending on context MAYBE_TRANSLITERATION_CLASSES = frozenset(["p36", "p38", "p15"]) # Synonym classes (color #000080) SYNONYM_CLASSES = frozenset(["p26"]) # Section header classes — SYNONYMS, TRANSLATION, or PURPORT SECTION_HEADER_CLASSES = frozenset(["p25", "p27"]) # Translation text classes (color #800000) TRANSLATION_CLASSES = frozenset(["p28", "p34"]) # Purport paragraph classes PURPORT_CLASSES = frozenset( [ "p31", "p32", "p33", "p2", "p4", "p5", "p6", "p14", "p16", "p17", "p37", "p40", "p41", "p42", "p43", "p44", "p45", "p46", "p47", "p48", "p49", "p50", "p51", "p52", "p53", "p54", "p55", "p56", "p57", "p58", "p71", "p74", "p76", "p77", "p78", "p79", "p80", "p81", "p82", "p85", "p86", "p89", "p90", "p91", "p92", "p93", "p94", "p95", ] ) # Embedded verse quotations in purport (center-aligned) PURPORT_VERSE_CLASSES = frozenset(["p36", "p15", "p38", "p18", "p59", "p60"]) # Cross-reference notes (green, center) — skip CITATION_CLASSES = frozenset(["p19", "p30"]) # Chapter end markers ("Thus end the Bhaktivedanta purports…") CHAPTER_END_CLASSES = frozenset(["p37", "p93"]) # Chapter bookmark class CHAPTER_BOOKMARK_CLASS = "p11" # Verse reference marker VERSE_REF_CLASS = "p20" # TEXT N header TEXT_HEADER_CLASS = "p21" # TEXT label (maroon) TEXT_LABEL_CLASS = "p22" # Līlā banner classes — skip LILA_BANNER_CLASSES = frozenset(["p8", "p39"]) # Volume marker — skip VOLUME_MARKER_CLASS = "p7" # Chapter number/heading — skip CHAPTER_HEADING_CLASSES = frozenset(["p12", "p9", "p10", "p13"]) # Skip classes (decorative, front matter, etc.) SKIP_CLASSES = frozenset( [ "p1", # ScaOptima title "p3", # center indent text (front matter) ] ) def get_css_class(tag: Tag) -> str: """Get the primary CSS class of a tag.""" classes = tag.get("class", []) return classes[0] if classes else "" # --------------------------------------------------------------------------- # Līlā name mapping # --------------------------------------------------------------------------- LILA_NAMES = { "adi": "Ādi-līlā", "madhya": "Madhya-līlā", "antya": "Antya-līlā", } LILA_ENCODED_TO_KEY = { "ädi": "adi", "adi": "adi", "ādi": "adi", "madhya": "madhya", "antya": "antya", } def normalize_lila_name(raw: str) -> str: """Normalize a līlā name from ScaGoudy-encoded text to a key.""" lower = raw.strip().lower() # After ScaGoudy decoding, Ädi becomes Ādi decoded = decode_scagoudy(lower) for pattern, key in LILA_ENCODED_TO_KEY.items(): if decoded.startswith(pattern) or lower.startswith(pattern): return key return lower # --------------------------------------------------------------------------- # Reference parsing # --------------------------------------------------------------------------- # Matches "Ädi 1.1" or "Madhya 8.201" or "Antya 1.3-4" _VERSE_REF_RE = re.compile( r"(Ädi|Adi|Ādi|Madhya|Antya)\s+(\d+)\.(\d+)(?:\s*-\s*(\d+))?", re.IGNORECASE, ) def _parse_verse_ref(text: str): """Parse a verse reference string. Returns (lila_key, chapter, verse, verse_end, clean_ref) or None. """ text = decode_scagoudy(text.strip()) m = _VERSE_REF_RE.search(text) if not m: return None lila_raw = m.group(1) chapter = int(m.group(2)) verse = int(m.group(3)) verse_end = int(m.group(4)) if m.group(4) else None lila_key = normalize_lila_name(lila_raw) lila_display = {"adi": "Ādi", "madhya": "Madhya", "antya": "Antya"}.get( lila_key, lila_raw ) if verse_end is not None: clean_ref = f"Cc {lila_display} {chapter}.{verse}-{verse_end}" else: clean_ref = f"Cc {lila_display} {chapter}.{verse}" return (lila_key, chapter, verse, verse_end, clean_ref) def _parse_chapter_bookmark(text: str): """Parse a chapter bookmark like 'Ädi 1: The Spiritual Masters'. Returns (lila_key, chapter, title) or None. """ text = decode_scagoudy(text.strip()) # Match "Ādi 1: Title" or "Madhya 9b: Title" m = re.match(r"(Ādi|Adi|Madhya|Antya)\s+(\d+)b?\s*:\s*(.+)", text, re.IGNORECASE) if m: lila_key = normalize_lila_name(m.group(1)) chapter = int(m.group(2)) title = m.group(3).strip() return (lila_key, chapter, title) return None # --------------------------------------------------------------------------- # State machine # --------------------------------------------------------------------------- def _finalize_verse(verse: dict): """Build transliteration, Devanagari and Bengali from pādas.""" padas = verse.get("_padas", []) verse["transliteration"] = "\n".join(padas) if padas: paired = pair_padas(padas) verse["devanagari"] = iast_to_devanagari(paired) verse["bengali"] = iast_to_bengali(paired) else: verse["devanagari"] = "" verse["bengali"] = "" for key in ("_padas", "_state"): if key in verse: del verse[key] def extract_verses(soup: BeautifulSoup): """Extract verses from parsed CC HTML using a state machine. Returns (verses, chapter_titles) where chapter_titles is: { (lila_key, chapter): title_string } """ paragraphs = soup.find_all("p") verses = [] current_verse = None state = "IDLE" chapter_titles = {} # (lila_key, chapter) -> title for p in paragraphs: cls = get_css_class(p) text = p.get_text().strip() # --- Chapter bookmark (p11) --- if cls == CHAPTER_BOOKMARK_CLASS: parsed = _parse_chapter_bookmark(text) if parsed: lila_key, chapter, title = parsed chapter_titles[(lila_key, chapter)] = title continue # --- Volume/līlā markers — skip --- if cls == VOLUME_MARKER_CLASS or cls in LILA_BANNER_CLASSES: continue # --- Chapter heading/number — skip --- if cls in CHAPTER_HEADING_CLASSES or cls in SKIP_CLASSES: continue # --- Verse reference (p20) --- if cls == VERSE_REF_CLASS: # Finalize previous verse if current_verse: _finalize_verse(current_verse) verses.append(current_verse) current_verse = None state = "IDLE" ref = _parse_verse_ref(text) if ref is None: continue lila_key, chapter, verse_num, verse_end, clean_ref = ref current_verse = { "book_title": "Śrī Caitanya-caritāmṛta", "lila": lila_key, "chapter": chapter, "verse": verse_num, "reference": clean_ref, "devanagari": "", "bengali": "", "transliteration": "", "_padas": [], "synonyms": [], "translation": "", "purport": [], "_state": "VERSE_START", } if verse_end is not None: current_verse["verse_end"] = verse_end state = "VERSE_START" continue # --- TEXT N header (p21) --- if cls == TEXT_HEADER_CLASS and current_verse: continue # --- TEXT label (p22) --- if cls == TEXT_LABEL_CLASS: continue # --- Bengali (Inben fonts) — SKIP --- if cls in BENGALI_CLASSES and current_verse: state = "BENGALI" current_verse["_state"] = state continue # --- Transliteration pādas (p24) --- if ( cls in TRANSLITERATION_CLASSES and current_verse and state in ("VERSE_START", "BENGALI", "TRANSLITERATION") ): state = "TRANSLITERATION" current_verse["_state"] = state line = decode_text(p) if line: current_verse["_padas"].append(line) continue # --- Maybe-transliteration (context-dependent) --- if ( cls in MAYBE_TRANSLITERATION_CLASSES and current_verse and state in ("BENGALI", "TRANSLITERATION") ): state = "TRANSLITERATION" current_verse["_state"] = state line = decode_text(p) if line: current_verse["_padas"].append(line) continue # --- Synonyms (p26) --- if cls in SYNONYM_CLASSES and current_verse: state = "SYNONYMS" current_verse["_state"] = state current_verse["synonyms"] = parse_synonym_html(p) continue # --- Section headers: SYNONYMS / TRANSLATION / PURPORT (p25, p27) --- if cls in SECTION_HEADER_CLASSES and current_verse: header_text = text.strip().upper() if "SYNONYM" in header_text: state = "SYNONYMS_HEADER" current_verse["_state"] = state elif "TRANSLATION" in header_text: state = "TRANSLATION" current_verse["_state"] = state elif "PURPORT" in header_text: state = "PURPORT" current_verse["_state"] = state continue # --- Translation text (p28, p34) --- if cls in TRANSLATION_CLASSES and current_verse and state == "TRANSLATION": current_verse["translation"] = decode_html_content(p) state = "POST_TRANSLATION" current_verse["_state"] = state continue # --- Handle mislabeled TRANSLATION as PURPORT --- if ( cls in TRANSLATION_CLASSES and current_verse and state == "PURPORT" and not current_verse.get("translation") ): current_verse["translation"] = decode_html_content(p) state = "POST_TRANSLATION" current_verse["_state"] = state continue # --- Purport paragraphs --- if state == "PURPORT" and current_verse and cls in PURPORT_CLASSES: content = decode_html_content(p) if content and "Thus end the Bhaktivedanta" not in text: current_verse["purport"].append(content) continue # --- Embedded verse quotations in purport --- if state == "PURPORT" and current_verse and cls in PURPORT_VERSE_CLASSES: content = decode_html_content(p) if content: current_verse["purport"].append(content) continue # --- Cross-reference citations (p19, p30) — skip --- if cls in CITATION_CLASSES: continue # --- Chapter end marker --- if cls in CHAPTER_END_CLASSES: # Check if this is really a "Thus end" marker if "Thus end" in text or "thus end" in text: if current_verse: _finalize_verse(current_verse) verses.append(current_verse) current_verse = None state = "IDLE" continue # Otherwise it might be a purport paragraph with class p37 if state == "PURPORT" and current_verse: content = decode_html_content(p) if content: current_verse["purport"].append(content) continue # Don't forget the last verse if current_verse: _finalize_verse(current_verse) verses.append(current_verse) return verses, chapter_titles # --------------------------------------------------------------------------- # JSON output # --------------------------------------------------------------------------- def write_json(data, filepath): """Write JSON with proper Unicode and formatting.""" os.makedirs(os.path.dirname(filepath), exist_ok=True) with open(filepath, "w", encoding="utf-8") as f: json.dump(data, f, ensure_ascii=False, indent=2) print(f" Written: {filepath}") def main(): base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) html_path = os.path.join(base_dir, "public", "html", "cc.html") output_dir = os.path.join(base_dir, "lib", "jsons", "books", "cc") print(f"Reading {html_path}...") with open(html_path, "r", encoding="utf-8") as f: soup = BeautifulSoup(f, "html.parser") print("Extracting all CC verses...") verses, chapter_titles = extract_verses(soup) print(f"\nExtracted {len(verses)} verses across {len(chapter_titles)} chapters") # Group by (lila, chapter) lilas_info = {} # lila_key -> { chapter -> verse_count } for verse in verses: lk = verse["lila"] ch = verse["chapter"] if lk not in lilas_info: lilas_info[lk] = {} if ch not in lilas_info[lk]: lilas_info[lk][ch] = 0 lilas_info[lk][ch] += 1 # Build intro.json lilas_list = [] lila_order = ["adi", "madhya", "antya"] for lila_key in lila_order: if lila_key not in lilas_info: continue chapters_in_lila = lilas_info[lila_key] chapters_list = [] for ch_num in sorted(chapters_in_lila.keys()): title = chapter_titles.get((lila_key, ch_num), "") chapters_list.append( { "number": ch_num, "title": title, "totalVerses": chapters_in_lila[ch_num], } ) total_verses = sum(chapters_in_lila.values()) lilas_list.append( { "key": lila_key, "name": LILA_NAMES.get(lila_key, lila_key), "totalChapters": len(chapters_list), "totalVerses": total_verses, "chapters": chapters_list, } ) intro = { "book": "cc", "title": "Śrī Caitanya-caritāmṛta", "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda", "totalLilas": len(lilas_list), "lilas": lilas_list, } write_json(intro, os.path.join(output_dir, "intro.json")) # Write verse JSON files: # app/lib/books/cc/lilas/{lila}/chapters/{ch}/verses/{vs}.json for verse in verses: lila = verse["lila"] ch = str(verse["chapter"]).zfill(2) vs = str(verse["verse"]).zfill(3) filepath = os.path.join( output_dir, "lilas", lila, "chapters", ch, "verses", f"{vs}.json" ) write_json(verse, filepath) # Print summary print("\n--- Verification ---") for lila_key in lila_order: if lila_key not in lilas_info: continue chapters_in_lila = lilas_info[lila_key] total_verses_in_lila = sum(chapters_in_lila.values()) print( f"\n{LILA_NAMES.get(lila_key, lila_key)}: " f"{len(chapters_in_lila)} chapters, {total_verses_in_lila} verses" ) for ch_num in sorted(chapters_in_lila.keys()): ch_verses = [ v for v in verses if v["lila"] == lila_key and v["chapter"] == ch_num ] if ch_verses: first = ch_verses[0] last = ch_verses[-1] title = chapter_titles.get((lila_key, ch_num), "") print( f" Ch {ch_num:>2}: {len(ch_verses):>4} verses " f"({first['reference']} → {last['reference']}) {title}" ) # Quick quality check: show first verse if verses: print("\n--- Sample (first verse) ---") v = verses[0] print(f"Reference: {v['reference']}") print(f"Devanagari:\n{v['devanagari']}") print(f"Bengali:\n{v['bengali']}") print(f"Transliteration:\n{v['transliteration']}") print(f"Synonyms: {len(v['synonyms'])} entries") if v["synonyms"]: print(f" First: {v['synonyms'][0]}") print(f"Translation: {v['translation'][:120]}...") print(f"Purport: {len(v['purport'])} paragraphs") if __name__ == "__main__": main()