#!/usr/bin/env python3 """Extract Śrī Īśopaniṣad (1969 Edition) verses from other.html into structured JSON files.""" import json import os import re import sys from bs4 import BeautifulSoup, Tag from indic_transliteration import sanscript from indic_transliteration.sanscript import transliterate # --- ScaGoudy → IAST Unicode mapping --- SCAGOUDY_TO_IAST = { "ä": "ā", "Ä": "Ā", "å": "ṛ", "Å": "Ṛ", "ñ": "ṣ", "Ñ": "Ṣ", "ö": "ṭ", "Ö": "Ṭ", "ë": "ṇ", "Ë": "Ṇ", "ò": "ḍ", "Ò": "Ḍ", "ç": "ś", "Ç": "Ś", "ì": "ṅ", "ï": "ñ", "ü": "ū", "Ü": "Ū", "é": "ī", "É": "Ī", "à": "ṁ", "ù": "ḥ", "è": "ṝ", # long vocalic r (U+1E5D) "û": "ṁl", # anusvara + l (ScaGoudy ligature glyph) "ÿ": "ḷ", # vocalic l (U+1E37) } # --- Mantra number to title mapping --- MANTRA_TITLES = { 0: "Invocation", 1: "Mantra One", 2: "Mantra Two", 3: "Mantra Three", 4: "Mantra Four", 5: "Mantra Five", 6: "Mantra Six", 7: "Mantra Seven", 8: "Mantra Eight", 9: "Mantra Nine", 10: "Mantra Ten", 11: "Mantra Eleven", 12: "Mantra Twelve", 13: "Mantra Thirteen", 14: "Mantra Fourteen", 15: "Mantra Fifteen", 16: "Mantra Sixteen", 17: "Mantra Seventeen", 18: "Mantra Eighteen", } # --- CSS class constants --- BOOK_HEADER_CLASS = "p12" BOOKMARK_CLASS = "p13" VERSE_BOOKMARK_CLASS = "p49" VERSE_HEADING_CLASS = "p50" DEVANAGARI_CLASS = "p51" # RM Devanagari — SKIP TEXT_LABEL_CLASS = "p52" INVOCATION_TRANSLIT_CLASS = "p53" PADA_TRANSLIT_CLASS = "p5" SYNONYM_CLASSES = ("p54", "p59") TRANSLATION_LABEL_CLASS = "p55" TRANSLATION_TEXT_CLASS = "p56" PURPORT_LABEL_CLASS = "p57" PURPORT_CLASSES = ("p48", "p58", "p23", "p60", "p63", "p46", "p47", "p61", "p62") SECTION_HEADING_CLASS = "p31" BODY_CLASSES = ("p6", "p45") SKIP_CLASSES = ("p9", "p15", "p19") def decode_scagoudy(text: str) -> str: """Convert ScaGoudy-encoded text to proper IAST Unicode.""" for src, dst in SCAGOUDY_TO_IAST.items(): text = text.replace(src, dst) return text def decode_html_content(tag: Tag) -> str: """Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags.""" inner = "".join(str(child) for child in tag.children) inner = decode_scagoudy(inner) # Remove span tags with CSS classes (artifacts from RTF conversion) inner = re.sub( r']*class="[^"]*"[^>]*>(.*?)', r"\1", inner, flags=re.DOTALL ) # Clean up Apple-tab-span leftovers (tab characters) inner = inner.replace("\t", " ") # Normalize whitespace inner = re.sub(r"\s+", " ", inner).strip() return inner def decode_text(tag: Tag) -> str: """Extract plain text from a tag and decode ScaGoudy.""" text = tag.get_text() return decode_scagoudy(text).strip() def normalize_devanagari_anusvara(text: str) -> str: """Replace explicit class nasals before same-class consonants with anusvāra. Only normalize ङ् and ञ् — these are almost always written with anusvāra. Keep ण्ड, न्त, म्ब etc. as explicit conjuncts (standard spelling). """ replacements = [ # ङ् before velars (क, ख, ग, घ) (r"ङ्([कखगघ])", r"ं\1"), # ञ् before palatals (च, छ, ज, झ) (r"ञ्([चछजझ])", r"ं\1"), ] for pattern, repl in replacements: text = re.sub(pattern, repl, text) return text def join_virama_words(text: str) -> str: """Join Devanagari words where the previous word ends with virāma (्). Handles both virāma+consonant and virāma+vowel cases. Uses [ \\t]+ instead of \\s+ to avoid joining across newlines. """ # Join virāma + space + consonant (क U+0915 through ह U+0939) text = re.sub(r"्[ \t]+([क-ह])", r"्\1", text) # Join virāma + space + independent vowel _VOWEL_TO_MATRA = { "अ": "", # inherent vowel — just drop virāma and the अ "आ": "ा", "इ": "ि", "ई": "ी", "उ": "ु", "ऊ": "ू", "ऋ": "ृ", "ॠ": "ॄ", "ऌ": "ॢ", "ॡ": "ॣ", "ए": "े", "ऐ": "ै", "ओ": "ो", "औ": "ौ", } def _virama_vowel_replace(m: re.Match[str]) -> str: vowel = m.group(1) return _VOWEL_TO_MATRA.get(vowel, vowel) text = re.sub(r"्[ \t]+([अ-औॠॡ])", _virama_vowel_replace, text) return text def iast_to_devanagari(iast_text: str) -> str: """Convert IAST transliteration to Devanagari script.""" lines = iast_text.split("\n") result_lines = [] for line in lines: # Remove hyphens between words for Devanagari clean = line.replace("-", "") # Normalize ṁ (U+1E41) to ṃ (U+1E43) for proper anusvāra clean = clean.replace("ṁ", "ṃ") deva = transliterate(clean, sanscript.IAST, sanscript.DEVANAGARI) deva = normalize_devanagari_anusvara(deva) deva = join_virama_words(deva) result_lines.append(deva) return "\n".join(result_lines) def parse_synonyms(text: str) -> list: """Parse synonym text into structured array. Input format: "word—meaning; word—meaning; word—meaning" """ result = [] entries = text.split(";") for entry in entries: entry = entry.strip() if not entry: continue if "—" in entry: parts = entry.split("—", 1) elif "--" in entry: parts = entry.split("--", 1) else: continue if len(parts) == 2: word = parts[0].strip() meaning = parts[1].strip() # Clean word: remove trailing/leading punctuation word = word.strip("., ") if word and meaning: result.append({"word": word, "meanings": [meaning]}) return result def pair_padas(padas: list) -> str: """Pair pādas into verse lines (2 pādas per line). A śloka has 4 pādas displayed as 2 lines. If there's an odd number, the first pāda stands alone (usually a speaker attribution). """ if not padas: return "" lines = [] i = 0 # If odd number of pādas, first one is standalone if len(padas) % 2 == 1: lines.append(padas[0]) i = 1 # Pair remaining pādas while i < len(padas) - 1: lines.append(padas[i] + " " + padas[i + 1]) i += 2 # Handle any trailing single pāda if i < len(padas): lines.append(padas[i]) return "\n".join(lines) def get_css_class(tag: Tag) -> str: """Get the primary CSS class of a tag.""" classes = tag.get("class", []) return classes[0] if classes else "" def is_empty(tag: Tag) -> bool: """Check if a tag is empty or whitespace-only.""" text = tag.get_text().strip() return not text or text == "\xa0" def is_dashed_line(text: str) -> bool: """Check if text is a dashed-line separator.""" clean = text.strip().replace("-", "").replace("—", "").strip() return clean == "" def parse_verse_bookmark(text: str) -> int | None: """Parse a verse bookmark like 'Iso Invocation-1969' or 'Iso 1-1969'. Returns mantra number (0 for Invocation, 1-18 for mantras), or None. """ text = text.strip() if text == "Iso Invocation-1969": return 0 m = re.match(r"Iso\s+(\d+)-1969", text) if m: return int(m.group(1)) return None # --------------------------------------------------------------------------- # Boundary detection # --------------------------------------------------------------------------- def find_iso_boundaries(all_paragraphs: list) -> tuple[int, int]: """Find the start and end paragraph indices of the Iso content. Start:

containing "Īśopaniṣad" (ScaGoudy encoded) End: next

after start (start of next book), or end of file """ start_idx = None end_idx = len(all_paragraphs) for i, p in enumerate(all_paragraphs): cls = get_css_class(p) if cls != BOOK_HEADER_CLASS: continue text = p.get_text().strip() # The ScaGoudy-encoded title contains "Éçopaniñad" if "opani" in text.lower() and "1969" in text: start_idx = i elif start_idx is not None: # Next p12 after Iso start = end of Iso end_idx = i break if start_idx is None: print( "ERROR: Could not find 'Śrī Īśopaniṣad - 1969 Edition' header in other.html!" ) sys.exit(1) return start_idx, end_idx def find_section_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> dict: """Find the key p13 bookmarks for Iso section boundaries. Returns dict with bookmark text -> paragraph index. """ bookmarks = {} for i in range(start_idx, end_idx): p = all_paragraphs[i] cls = get_css_class(p) if cls == BOOKMARK_CLASS: text = decode_text(p) if "Iso" in text and "1969" in text: bookmarks[text] = i return bookmarks def find_verse_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list: """Find all p49 verse bookmark tags in the Iso range. Returns list of (paragraph_index, mantra_number). """ bookmarks = [] for i in range(start_idx, end_idx): p = all_paragraphs[i] cls = get_css_class(p) if cls == VERSE_BOOKMARK_CLASS: text = decode_text(p) mantra_num = parse_verse_bookmark(text) if mantra_num is not None: bookmarks.append((i, mantra_num)) return bookmarks # --------------------------------------------------------------------------- # Intro segment extraction # --------------------------------------------------------------------------- def extract_intro_segment( all_paragraphs: list, start_idx: int, end_idx: int ) -> list[str]: """Extract paragraphs for an intro segment (front pages or about the author). Decodes ScaGoudy, preserves HTML formatting, skips dashed lines and empty paragraphs. """ result = [] for i in range(start_idx, end_idx): p = all_paragraphs[i] cls = get_css_class(p) if is_empty(p): continue # Skip bookmark and header classes if cls in (BOOKMARK_CLASS, BOOK_HEADER_CLASS, VERSE_BOOKMARK_CLASS): continue # Skip noise classes if cls in SKIP_CLASSES: continue html = decode_html_content(p) if not html: continue # Skip dashed-line separators plain = re.sub(r"<[^>]+>", "", html).strip() if is_dashed_line(plain): continue result.append(html) return result # --------------------------------------------------------------------------- # Verse extraction state machine # --------------------------------------------------------------------------- def extract_verses(all_paragraphs: list, verses_start: int, verses_end: int) -> list: """Extract verses from the Iso Texts section using a state machine. Returns list of verse dicts. """ verses = [] current_verse = None state = "IDLE" for i in range(verses_start, verses_end): p = all_paragraphs[i] cls = get_css_class(p) # --- Verse bookmark (p49) --- if cls == VERSE_BOOKMARK_CLASS: # Finalize previous verse if any if current_verse and state != "IDLE": _finalize_verse(current_verse) verses.append(current_verse) text = decode_text(p) mantra_num = parse_verse_bookmark(text) if mantra_num is not None: title = MANTRA_TITLES.get(mantra_num, f"Mantra {mantra_num}") if mantra_num == 0: reference = "Invocation" else: reference = f"Īśo {mantra_num}" current_verse = { "book": "iso", "book_title": "Śrī Īśopaniṣad", "mantra": mantra_num, "title": title, "reference": reference, "devanagari": "", "transliteration": "", "_padas": [], "synonyms": [], "translation": "", "purport": [], } state = "VERSE_START" continue if not current_verse: continue # --- Verse heading (p50) --- if cls == VERSE_HEADING_CLASS: # Just confirms the mantra title; skip continue # --- RM Devanagari (p51) — SKIP --- if cls == DEVANAGARI_CLASS: continue # --- TEXT label (p52) — skip --- if cls == TEXT_LABEL_CLASS: state = "TRANSLITERATION" continue # --- Invocation transliteration (p53) — single

with
--- if cls == INVOCATION_TRANSLIT_CLASS and current_verse["mantra"] == 0: state = "TRANSLITERATION" # Split on
tags to get individual pādas # Get inner HTML and split on
variants inner_html = "".join(str(child) for child in p.children) # Remove tags inner_html = re.sub(r"", "", inner_html) # Split on
or
or
raw_padas = re.split(r"", inner_html) for pada in raw_padas: # Decode ScaGoudy on plain text pada_text = BeautifulSoup(pada, "html.parser").get_text() pada_text = decode_scagoudy(pada_text).strip() # Remove trailing period pada_text = pada_text.rstrip(".") if pada_text: current_verse["_padas"].append(pada_text) continue # --- Pāda transliteration lines (p5) --- if cls == PADA_TRANSLIT_CLASS and state in ( "VERSE_START", "TRANSLITERATION", ): state = "TRANSLITERATION" line = decode_text(p) # Remove trailing period line = line.rstrip(".") # Strip leading/trailing whitespace line = line.strip() if line: current_verse["_padas"].append(line) continue # --- Synonyms (p54, p59) --- if cls in SYNONYM_CLASSES and current_verse: state = "SYNONYMS" syn_text = decode_text(p) # Remove leading/trailing italic markers that might come through syn_text = syn_text.strip() parsed = parse_synonyms(syn_text) current_verse["synonyms"].extend(parsed) continue # --- TRANSLATION label (p55) --- if cls == TRANSLATION_LABEL_CLASS and current_verse: state = "TRANSLATION" continue # --- Translation text (p56) --- if cls == TRANSLATION_TEXT_CLASS and current_verse and state == "TRANSLATION": html = decode_html_content(p) # Remove bold wrapper if present (the whole translation is bold) html = re.sub(r"^(.*)$", r"\1", html, flags=re.DOTALL) # Remove trailing source references like [& Bṛhad-āraṇyaka Upaniṣad 5.1] # These are in span.s4 tags — they may have survived decode_html_content html = html.strip() current_verse["translation"] = html state = "POST_TRANSLATION" continue # --- PURPORT label (p57) --- if cls == PURPORT_LABEL_CLASS and current_verse: state = "PURPORT" continue # --- Purport paragraphs --- if state == "PURPORT" and current_verse and cls in PURPORT_CLASSES: html = decode_html_content(p) if html: # Skip dashed-line separators plain = re.sub(r"<[^>]+>", "", html).strip() if not is_dashed_line(plain): current_verse["purport"].append(html) continue # --- Section heading within purport (p31) --- if state == "PURPORT" and current_verse and cls == SECTION_HEADING_CLASS: html = decode_html_content(p) if html: current_verse["purport"].append(html) continue # --- Body text classes that might appear in purport (p6) --- if state == "PURPORT" and current_verse and cls in BODY_CLASSES: html = decode_html_content(p) if html: plain = re.sub(r"<[^>]+>", "", html).strip() if not is_dashed_line(plain): current_verse["purport"].append(html) continue # Finalize last verse if current_verse and state != "IDLE": _finalize_verse(current_verse) verses.append(current_verse) return verses def _finalize_verse(verse: dict): """Build transliteration and Devanagari from pādas. Remove internal fields.""" padas = verse.get("_padas", []) # Transliteration: one pāda per line verse["transliteration"] = "\n".join(padas) # Devanagari: pair pādas into 2-line verse structure, then convert if padas: paired = pair_padas(padas) verse["devanagari"] = iast_to_devanagari(paired) # Clean up internal field if "_padas" in verse: del verse["_padas"] # --------------------------------------------------------------------------- # JSON writing # --------------------------------------------------------------------------- def write_json(data, filepath): """Write JSON with proper Unicode and formatting.""" os.makedirs(os.path.dirname(filepath), exist_ok=True) with open(filepath, "w", encoding="utf-8") as f: json.dump(data, f, ensure_ascii=False, indent=2) print(f" Written: {filepath}") # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- def main(): base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) html_path = os.path.join(base_dir, "public", "html", "other.html") output_base = os.path.join(base_dir, "lib", "jsons", "books", "iso") if not os.path.exists(html_path): print(f"ERROR: {html_path} not found!") sys.exit(1) print(f"Reading {html_path}...") with open(html_path, "r", encoding="utf-8") as f: soup = BeautifulSoup(f, "html.parser") all_paragraphs = soup.find_all("p") print(f"Total paragraphs in file: {len(all_paragraphs)}") # --- Find Iso boundaries --- iso_start, iso_end = find_iso_boundaries(all_paragraphs) print(f"Iso content: paragraphs {iso_start} to {iso_end}") # --- Find section bookmarks (p13) --- section_bookmarks = find_section_bookmarks(all_paragraphs, iso_start, iso_end) print(f"\nSection bookmarks found:") for bm_text, bm_idx in sorted(section_bookmarks.items(), key=lambda x: x[1]): print(f" [{bm_idx}] {bm_text}") # Key bookmark indices front_pages_idx = section_bookmarks.get("Iso back cover & front pages-1969") introduction_idx = section_bookmarks.get("Iso Introduction-1969") texts_idx = section_bookmarks.get("Iso Texts-1969") about_idx = section_bookmarks.get("Iso About the author-1969") glossary_idx = section_bookmarks.get("Iso Glossary-1969") if texts_idx is None: print("ERROR: Could not find 'Iso Texts-1969' bookmark!") sys.exit(1) # Use iso_end as fallback for glossary boundary end_boundary = glossary_idx if glossary_idx is not None else iso_end # --- Extract intro segments --- print("\n--- Extracting Intro Segments ---") intro_segments = [] # 1. Front pages segment (up to the Introduction bookmark, or texts if no intro) front_end = introduction_idx if introduction_idx is not None else texts_idx if front_pages_idx is not None and front_end is not None: front_paras = extract_intro_segment(all_paragraphs, front_pages_idx, front_end) front_segment = { "slug": "front-pages", "title": "Back Cover & Front Pages", "type": "intro-segment", "book": "iso", "order": 1, "paragraphs": front_paras, } intro_segments.append(front_segment) print(f" Front Pages: {len(front_paras)} paragraphs") # 2. Introduction (Teachings of the Vedas) if introduction_idx is not None and texts_idx is not None: intro_paras = extract_intro_segment(all_paragraphs, introduction_idx, texts_idx) intro_segment = { "slug": "introduction", "title": "Introduction (Teachings of the Vedas)", "type": "intro-segment", "book": "iso", "order": 2, "paragraphs": intro_paras, } intro_segments.append(intro_segment) print(f" Introduction: {len(intro_paras)} paragraphs") # 3. About the Author segment if about_idx is not None: about_paras = extract_intro_segment(all_paragraphs, about_idx, end_boundary) about_segment = { "slug": "about-the-author", "title": "About the Author", "type": "intro-segment", "book": "iso", "order": 3, "paragraphs": about_paras, } intro_segments.append(about_segment) print(f" About the Author: {len(about_paras)} paragraphs") # --- Extract verses --- print("\n--- Extracting Verses ---") # Verse section runs from texts_idx to about_idx (or end_boundary) verse_end = about_idx if about_idx is not None else end_boundary verses = extract_verses(all_paragraphs, texts_idx, verse_end) print(f"Extracted {len(verses)} verses (expected 19: Invocation + Mantras 1-18)") # --- Write intro.json --- mantras_listing = [] for num in range(0, 19): title = MANTRA_TITLES[num] if num == 0: slug = "invocation" else: slug = str(num).zfill(2) mantras_listing.append({"number": num, "title": title, "slug": slug}) intro_meta = { "book": "iso", "title": "Śrī Īśopaniṣad", "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda", "year": 1969, "description": "The foremost of the 108 Upaniṣads, containing the essence of all Vedic knowledge.", "totalMantras": 18, "mantras": mantras_listing, } write_json(intro_meta, os.path.join(output_base, "intro.json")) # --- Write intro segments --- print("\n--- Writing Intro Segments ---") for seg in intro_segments: filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json") write_json(seg, filepath) # --- Write verse JSON files --- print("\n--- Writing Verse Files ---") for verse in verses: mantra_num = verse["mantra"] if mantra_num == 0: filename = "invocation.json" else: filename = f"{str(mantra_num).zfill(2)}.json" filepath = os.path.join(output_base, "verses", filename) write_json(verse, filepath) # --- Summary --- print(f"\n{'=' * 60}") print(f" EXTRACTION SUMMARY") print(f"{'=' * 60}") print(f" Book: Śrī Īśopaniṣad (1969 Edition)") print(f" Intro segments: {len(intro_segments)}") for seg in intro_segments: print( f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs" ) print(f" Verses extracted: {len(verses)}") print( f" Invocation: {'Yes' if any(v['mantra'] == 0 for v in verses) else 'No'}" ) mantras_found = sorted([v["mantra"] for v in verses if v["mantra"] > 0]) print( f" Mantras: {mantras_found[0]}-{mantras_found[-1]} ({len(mantras_found)} total)" if mantras_found else " Mantras: None" ) total_purport_paras = sum(len(v["purport"]) for v in verses) total_synonyms = sum(len(v["synonyms"]) for v in verses) print(f" Total purport ¶s: {total_purport_paras}") print(f" Total synonyms: {total_synonyms}") print(f" Output directory: {output_base}/") # --- Quality checks --- print(f"\n--- Quality Checks ---") # Check all verses have expected fields issues = [] for verse in verses: ref = verse["reference"] if not verse["transliteration"]: issues.append(f" {ref}: missing transliteration") if not verse["devanagari"]: issues.append(f" {ref}: missing devanagari") if not verse["synonyms"]: issues.append(f" {ref}: missing synonyms") if not verse["translation"]: issues.append(f" {ref}: missing translation") if not verse["purport"]: issues.append(f" {ref}: missing purport") if issues: print(" ISSUES FOUND:") for issue in issues: print(issue) else: print( " All verses have transliteration, devanagari, synonyms, translation, and purport ✓" ) # --- Samples --- print(f"\n--- Sample: Invocation ---") inv = [v for v in verses if v["mantra"] == 0] if inv: v = inv[0] print(f" Reference: {v['reference']}") print(f" Transliteration: {v['transliteration'][:80]}...") print(f" Devanagari: {v['devanagari'][:80]}...") print(f" Synonyms: {len(v['synonyms'])} entries") if v["synonyms"]: print(f" First: {v['synonyms'][0]}") trans_preview = re.sub(r"<[^>]+>", "", v["translation"])[:100] print(f" Translation: {trans_preview}...") print(f" Purport ¶s: {len(v['purport'])}") print(f"\n--- Sample: Mantra One ---") m1 = [v for v in verses if v["mantra"] == 1] if m1: v = m1[0] print(f" Reference: {v['reference']}") print(f" Transliteration: {v['transliteration'][:80]}...") print(f" Devanagari: {v['devanagari'][:80]}...") print(f" Synonyms: {len(v['synonyms'])} entries") if v["synonyms"]: print(f" First: {v['synonyms'][0]}") print(f" Last: {v['synonyms'][-1]}") trans_preview = re.sub(r"<[^>]+>", "", v["translation"])[:100] print(f" Translation: {trans_preview}...") print(f" Purport ¶s: {len(v['purport'])}") print(f"\n--- Sample: Mantra Eighteen (last) ---") m18 = [v for v in verses if v["mantra"] == 18] if m18: v = m18[0] print(f" Reference: {v['reference']}") print(f" Transliteration: {v['transliteration'][:80]}...") print(f" Devanagari: {v['devanagari'][:80]}...") print(f" Synonyms: {len(v['synonyms'])} entries") trans_preview = re.sub(r"<[^>]+>", "", v["translation"])[:100] print(f" Translation: {trans_preview}...") print(f" Purport ¶s: {len(v['purport'])}") if __name__ == "__main__": main()