#!/usr/bin/env python3 """Extract The Nectar of Instruction (1976 Edition) verses from other.html into structured JSON files.""" import json import os import re import sys from bs4 import BeautifulSoup, Tag from indic_transliteration import sanscript from indic_transliteration.sanscript import transliterate # --- ScaGoudy → IAST Unicode mapping --- SCAGOUDY_TO_IAST = { "ä": "ā", "Ä": "Ā", "å": "ṛ", "Å": "Ṛ", "ñ": "ṣ", "Ñ": "Ṣ", "ö": "ṭ", "Ö": "Ṭ", "ë": "ṇ", "Ë": "Ṇ", "ò": "ḍ", "Ò": "Ḍ", "ç": "ś", "Ç": "Ś", "ì": "ṅ", "ï": "ñ", "ü": "ū", "Ü": "Ū", "é": "ī", "É": "Ī", "à": "ṁ", "ù": "ḥ", "è": "ṝ", # long vocalic r (U+1E5D) "û": "ṁl", # anusvara + l (ScaGoudy ligature glyph) "ÿ": "ḷ", # vocalic l (U+1E37) } # --- Text number to title mapping --- TEXT_TITLES = { 1: "Text One", 2: "Text Two", 3: "Text Three", 4: "Text Four", 5: "Text Five", 6: "Text Six", 7: "Text Seven", 8: "Text Eight", 9: "Text Nine", 10: "Text Ten", 11: "Text Eleven", } # --- CSS class constants --- BOOK_HEADER_CLASS = "p12" SECTION_BOOKMARK_CLASS = "p64" # NoI section bookmarks (Front pages, Preface) VERSE_BOOKMARK_CLASS = "p49" # NoI verse bookmarks (NoI: verse N) AND appendixes VERSE_HEADING_CLASS = "p50" DEVANAGARI_CLASS = "p51" # RM Devanagari — SKIP TEXT_LABEL_CLASS = "p52" PADA_TRANSLIT_CLASS = "p5" SYNONYM_CLASS = "p54" TRANSLATION_LABEL_CLASS = "p55" TRANSLATION_TEXT_CLASS = "p56" PURPORT_LABEL_CLASS = "p57" PURPORT_BODY_CLASS = "p48" QUOTED_VERSE_CLASSES = ( "p33", "p53", "p66", ) # Quoted Sanskrit/Bengali verses in purport BODY_TEXT_CLASS = "p44" # Verse translations within purport MISC_TEXT_CLASS = "p67" # Miscellaneous text (1 occurrence in NoI) SECTION_HEADING_CLASS = "p31" BODY_CLASSES = ("p6", "p45") SKIP_CLASSES = ("p9", "p15", "p19", "p25", "p32", "p37", "p65") def decode_scagoudy(text: str) -> str: """Convert ScaGoudy-encoded text to proper IAST Unicode.""" for src, dst in SCAGOUDY_TO_IAST.items(): text = text.replace(src, dst) return text def decode_html_content(tag: Tag) -> str: """Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags.""" inner = "".join(str(child) for child in tag.children) inner = decode_scagoudy(inner) # Remove span tags with CSS classes (artifacts from RTF conversion) inner = re.sub( r']*class="[^"]*"[^>]*>(.*?)', r"\1", inner, flags=re.DOTALL ) # Clean up Apple-tab-span leftovers (tab characters) inner = inner.replace("\t", " ") # Normalize whitespace inner = re.sub(r"\s+", " ", inner).strip() return inner def decode_text(tag: Tag) -> str: """Extract plain text from a tag and decode ScaGoudy.""" text = tag.get_text() return decode_scagoudy(text).strip() def normalize_devanagari_anusvara(text: str) -> str: """Replace explicit class nasals before same-class consonants with anusvāra. Only normalize ङ् and ञ् — these are almost always written with anusvāra. Keep ण्ड, न्त, म्ब etc. as explicit conjuncts (standard spelling). """ replacements = [ # ङ् before velars (क, ख, ग, घ) (r"ङ्([कखगघ])", r"ं\1"), # ञ् before palatals (च, छ, ज, झ) (r"ञ्([चछजझ])", r"ं\1"), ] for pattern, repl in replacements: text = re.sub(pattern, repl, text) return text def join_virama_words(text: str) -> str: """Join Devanagari words where the previous word ends with virāma (्). Handles both virāma+consonant and virāma+vowel cases. Uses [ \\t]+ instead of \\s+ to avoid joining across newlines. """ # Join virāma + space + consonant (क U+0915 through ह U+0939) text = re.sub(r"्[ \t]+([क-ह])", r"्\1", text) # Join virāma + space + independent vowel _VOWEL_TO_MATRA = { "अ": "", # inherent vowel — just drop virāma and the अ "आ": "ा", "इ": "ि", "ई": "ी", "उ": "ु", "ऊ": "ू", "ऋ": "ृ", "ॠ": "ॄ", "ऌ": "ॢ", "ॡ": "ॣ", "ए": "े", "ऐ": "ै", "ओ": "ो", "औ": "ौ", } def _virama_vowel_replace(m: re.Match[str]) -> str: vowel = m.group(1) return _VOWEL_TO_MATRA.get(vowel, vowel) text = re.sub(r"्[ \t]+([अ-औॠॡ])", _virama_vowel_replace, text) return text def iast_to_devanagari(iast_text: str) -> str: """Convert IAST transliteration to Devanagari script.""" lines = iast_text.split("\n") result_lines = [] for line in lines: # Remove hyphens between words for Devanagari clean = line.replace("-", "") # Normalize ṁ (U+1E41) to ṃ (U+1E43) for proper anusvāra clean = clean.replace("ṁ", "ṃ") deva = transliterate(clean, sanscript.IAST, sanscript.DEVANAGARI) deva = normalize_devanagari_anusvara(deva) deva = join_virama_words(deva) result_lines.append(deva) return "\n".join(result_lines) def parse_synonyms(text: str) -> list: """Parse synonym text into structured array. Input format: "word—meaning; word—meaning; word—meaning" """ result = [] entries = text.split(";") for entry in entries: entry = entry.strip() if not entry: continue if "—" in entry: parts = entry.split("—", 1) elif "--" in entry: parts = entry.split("--", 1) else: continue if len(parts) == 2: word = parts[0].strip() meaning = parts[1].strip() # Clean word: remove trailing/leading punctuation word = word.strip("., ") if word and meaning: result.append({"word": word, "meanings": [meaning]}) return result def pair_padas(padas: list) -> str: """Pair pādas into verse lines (2 pādas per line). A śloka has 4 pādas displayed as 2 lines. If there's an odd number, the first pāda stands alone (usually a speaker attribution). """ if not padas: return "" lines = [] i = 0 # If odd number of pādas, first one is standalone if len(padas) % 2 == 1: lines.append(padas[0]) i = 1 # Pair remaining pādas while i < len(padas) - 1: lines.append(padas[i] + " " + padas[i + 1]) i += 2 # Handle any trailing single pāda if i < len(padas): lines.append(padas[i]) return "\n".join(lines) def get_css_class(tag: Tag) -> str: """Get the primary CSS class of a tag.""" classes = tag.get("class", []) return classes[0] if classes else "" def is_empty(tag: Tag) -> bool: """Check if a tag is empty or whitespace-only.""" text = tag.get_text().strip() return not text or text == "\xa0" def is_dashed_line(text: str) -> bool: """Check if text is a dashed-line separator.""" clean = text.strip().replace("-", "").replace("—", "").strip() return clean == "" def parse_verse_bookmark(text: str) -> int | None: """Parse a verse bookmark like 'NoI: verse 1' through 'NoI: verse 11'. Returns text number (1-11), or None if not a verse bookmark. """ text = text.strip() m = re.match(r"NoI:\s*verse\s+(\d+)", text) if m: return int(m.group(1)) return None # --------------------------------------------------------------------------- # Boundary detection # --------------------------------------------------------------------------- def find_noi_boundaries(all_paragraphs: list) -> tuple[int, int]: """Find the start and end paragraph indices of the NoI content. Start:
containing "Nectar of Instruction" and "1976" End: next
after start (start of next book), or end of file
"""
start_idx = None
end_idx = len(all_paragraphs)
for i, p in enumerate(all_paragraphs):
cls = get_css_class(p)
if cls != BOOK_HEADER_CLASS:
continue
text = p.get_text().strip()
if "Nectar of Instruction" in text and "1976" in text:
start_idx = i
elif start_idx is not None:
# Next p12 after NoI start = end of NoI
end_idx = i
break
if start_idx is None:
print(
"ERROR: Could not find 'The Nectar of Instruction - 1976 Edition' header in other.html!"
)
sys.exit(1)
return start_idx, end_idx
def find_section_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> dict:
"""Find the key p64 bookmarks for NoI section boundaries.
Returns dict with bookmark text -> paragraph index.
"""
bookmarks = {}
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
if cls == SECTION_BOOKMARK_CLASS:
text = decode_text(p)
if "NoI" in text:
bookmarks[text] = i
return bookmarks
def find_verse_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
"""Find all p49 verse bookmark tags in the NoI range.
Returns list of (paragraph_index, text_number) for actual verse bookmarks.
Also returns appendixes bookmark index separately.
"""
verse_bookmarks = []
appendixes_idx = None
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
if cls == VERSE_BOOKMARK_CLASS:
text = decode_text(p)
text_num = parse_verse_bookmark(text)
if text_num is not None:
verse_bookmarks.append((i, text_num))
elif text.strip() == "NoI Appendixes":
appendixes_idx = i
return verse_bookmarks, appendixes_idx
# ---------------------------------------------------------------------------
# Intro segment extraction
# ---------------------------------------------------------------------------
def extract_intro_segment(
all_paragraphs: list, start_idx: int, end_idx: int
) -> list[str]:
"""Extract paragraphs for an intro segment (front pages, preface, or appendixes).
Decodes ScaGoudy, preserves HTML formatting, skips dashed lines and empty paragraphs.
"""
result = []
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
if is_empty(p):
continue
# Skip bookmark and header classes
if cls in (SECTION_BOOKMARK_CLASS, BOOK_HEADER_CLASS, VERSE_BOOKMARK_CLASS):
continue
# Skip noise classes
if cls in SKIP_CLASSES:
continue
html = decode_html_content(p)
if not html:
continue
# Skip dashed-line separators
plain = re.sub(r"<[^>]+>", "", html).strip()
if is_dashed_line(plain):
continue
result.append(html)
return result
# ---------------------------------------------------------------------------
# Verse extraction state machine
# ---------------------------------------------------------------------------
def extract_verses(all_paragraphs: list, verses_start: int, verses_end: int) -> list:
"""Extract verses from the NoI section using a state machine.
Returns list of verse dicts.
"""
verses = []
current_verse = None
state = "IDLE"
for i in range(verses_start, verses_end):
p = all_paragraphs[i]
cls = get_css_class(p)
# --- Verse bookmark (p49) ---
if cls == VERSE_BOOKMARK_CLASS:
# Finalize previous verse if any
if current_verse and state != "IDLE":
_finalize_verse(current_verse)
verses.append(current_verse)
text = decode_text(p)
text_num = parse_verse_bookmark(text)
if text_num is not None:
title = TEXT_TITLES.get(text_num, f"Text {text_num}")
reference = f"NoI {text_num}"
current_verse = {
"book": "noi",
"book_title": "The Nectar of Instruction",
"text": text_num,
"title": title,
"reference": reference,
"devanagari": "",
"transliteration": "",
"_padas": [],
"synonyms": [],
"translation": "",
"purport": [],
}
state = "VERSE_START"
else:
# Non-verse bookmark (e.g., "NoI Appendixes") — stop processing
current_verse = None
state = "IDLE"
continue
if not current_verse:
continue
# --- Verse heading (p50) ---
if cls == VERSE_HEADING_CLASS:
# Just confirms the text title; skip
continue
# --- RM Devanagari (p51) — SKIP ---
if cls == DEVANAGARI_CLASS:
continue
# --- TEXT label (p52) — skip ---
if cls == TEXT_LABEL_CLASS:
state = "TRANSLITERATION"
continue
# --- Pāda transliteration lines (p5) ---
if cls == PADA_TRANSLIT_CLASS and state in (
"VERSE_START",
"TRANSLITERATION",
):
state = "TRANSLITERATION"
line = decode_text(p)
# Remove trailing period
line = line.rstrip(".")
# Strip leading/trailing whitespace
line = line.strip()
if line:
current_verse["_padas"].append(line)
continue
# --- Synonyms (p54) ---
if cls == SYNONYM_CLASS and current_verse:
state = "SYNONYMS"
syn_text = decode_text(p)
syn_text = syn_text.strip()
parsed = parse_synonyms(syn_text)
current_verse["synonyms"].extend(parsed)
continue
# --- TRANSLATION / PURPORT label (p55) ---
if cls == TRANSLATION_LABEL_CLASS and current_verse:
label_text = decode_text(p).strip().upper()
if "TRANSLATION" in label_text:
state = "TRANSLATION"
elif "PURPORT" in label_text:
state = "PURPORT"
else:
# Other labels within purport (e.g., "The Author", "Glossary")
if state == "PURPORT":
html = decode_html_content(p)
if html:
current_verse["purport"].append(html)
continue
# --- Translation text (p56) ---
if cls == TRANSLATION_TEXT_CLASS and current_verse and state == "TRANSLATION":
html = decode_html_content(p)
# Remove bold wrapper if present (the whole translation is bold)
html = re.sub(r"^(.*)$", r"\1", html, flags=re.DOTALL)
html = html.strip()
current_verse["translation"] = html
state = "POST_TRANSLATION"
continue
# --- PURPORT label (p57) ---
if cls == PURPORT_LABEL_CLASS and current_verse:
state = "PURPORT"
continue
# --- Purport body paragraphs (p48) ---
if state == "PURPORT" and current_verse and cls == PURPORT_BODY_CLASS:
html = decode_html_content(p)
if html:
plain = re.sub(r"<[^>]+>", "", html).strip()
if not is_dashed_line(plain):
current_verse["purport"].append(html)
continue
# --- Quoted Sanskrit/Bengali verses within purport (p33, p53, p66) ---
if state == "PURPORT" and current_verse and cls in QUOTED_VERSE_CLASSES:
html = decode_html_content(p)
if html:
# Strip inner /<\/i> tags to avoid double wrapping
stripped = re.sub(r"?i>", "", html)
current_verse["purport"].append(
f'
Glossary
or noi_end end_boundary = noi_end # Look for the Glossary marker just before the next book for i in range(noi_start, noi_end): p = all_paragraphs[i] cls = get_css_class(p) if cls == TRANSLATION_LABEL_CLASS: text = decode_text(p).strip() if text.upper() == "GLOSSARY": end_boundary = i break # --- Extract intro segments --- print("\n--- Extracting Intro Segments ---") intro_segments = [] # 1. Front pages segment — from NoI Front pages (p64) to NoI Preface (p64) front_end = preface_idx if preface_idx is not None else first_verse_idx if front_pages_idx is not None and front_end is not None: front_paras = extract_intro_segment(all_paragraphs, front_pages_idx, front_end) front_segment = { "slug": "front-pages", "title": "Front Pages", "type": "intro-segment", "book": "noi", "order": 1, "paragraphs": front_paras, } intro_segments.append(front_segment) print(f" Front Pages: {len(front_paras)} paragraphs") # 2. Preface — from NoI Preface (p64) to first verse bookmark if preface_idx is not None and first_verse_idx is not None: preface_paras = extract_intro_segment( all_paragraphs, preface_idx, first_verse_idx ) preface_segment = { "slug": "preface", "title": "Preface", "type": "intro-segment", "book": "noi", "order": 2, "paragraphs": preface_paras, } intro_segments.append(preface_segment) print(f" Preface: {len(preface_paras)} paragraphs") # 3. Appendixes — from NoI Appendixes (p49) to end boundary if appendixes_idx is not None: appendixes_paras = extract_intro_segment( all_paragraphs, appendixes_idx, end_boundary ) appendixes_segment = { "slug": "appendixes", "title": "Appendixes", "type": "intro-segment", "book": "noi", "order": 3, "paragraphs": appendixes_paras, } intro_segments.append(appendixes_segment) print(f" Appendixes: {len(appendixes_paras)} paragraphs") # --- Extract verses --- print("\n--- Extracting Verses ---") # Verse section runs from first verse bookmark to appendixes (or end boundary) verse_end = appendixes_idx if appendixes_idx is not None else end_boundary verses = extract_verses(all_paragraphs, first_verse_idx, verse_end) print(f"Extracted {len(verses)} texts (expected 11: Texts 1-11)") # --- Write intro.json --- texts_listing = [] for num in range(1, 12): title = TEXT_TITLES[num] slug = str(num).zfill(2) texts_listing.append({"number": num, "title": title, "slug": slug}) intro_meta = { "book": "noi", "title": "The Nectar of Instruction", "subtitle": "Śrī Upadeśāmṛta", "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda", "year": 1976, "description": "Eleven essential instructions on the science of bhakti-yoga by Śrīla Rūpa Gosvāmī.", "totalTexts": 11, "texts": texts_listing, } write_json(intro_meta, os.path.join(output_base, "intro.json")) # --- Write intro segments --- print("\n--- Writing Intro Segments ---") for seg in intro_segments: filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json") write_json(seg, filepath) # --- Write verse JSON files --- print("\n--- Writing Verse Files ---") for verse in verses: text_num = verse["text"] filename = f"{str(text_num).zfill(2)}.json" filepath = os.path.join(output_base, "verses", filename) write_json(verse, filepath) # --- Summary --- print(f"\n{'=' * 60}") print(f" EXTRACTION SUMMARY") print(f"{'=' * 60}") print(f" Book: The Nectar of Instruction (1976 Edition)") print(f" Intro segments: {len(intro_segments)}") for seg in intro_segments: print( f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs" ) print(f" Texts extracted: {len(verses)}") texts_found = sorted([v["text"] for v in verses]) if texts_found: print( f" Texts: {texts_found[0]}-{texts_found[-1]} ({len(texts_found)} total)" ) else: print(" Texts: None") total_purport_paras = sum(len(v["purport"]) for v in verses) total_synonyms = sum(len(v["synonyms"]) for v in verses) print(f" Total purport ¶s: {total_purport_paras}") print(f" Total synonyms: {total_synonyms}") print(f" Output directory: {output_base}/") # --- Quality checks --- print(f"\n--- Quality Checks ---") # Check all verses have expected fields issues = [] for verse in verses: ref = verse["reference"] if not verse["transliteration"]: issues.append(f" {ref}: missing transliteration") if not verse["devanagari"]: issues.append(f" {ref}: missing devanagari") if not verse["synonyms"]: issues.append(f" {ref}: missing synonyms") if not verse["translation"]: issues.append(f" {ref}: missing translation") if not verse["purport"]: issues.append(f" {ref}: missing purport") if issues: print(" ISSUES FOUND:") for issue in issues: print(issue) else: print( " All texts have transliteration, devanagari, synonyms, translation, and purport ✓" ) # Check expected text numbers are present expected = set(range(1, 12)) found = set(texts_found) missing = expected - found extra = found - expected if missing: print(f" MISSING texts: {sorted(missing)}") if extra: print(f" UNEXPECTED texts: {sorted(extra)}") if not missing and not extra: print(f" All 11 texts present ✓") # --- Samples --- print(f"\n--- Sample: Text One ---") t1 = [v for v in verses if v["text"] == 1] if t1: v = t1[0] print(f" Reference: {v['reference']}") print(f" Title: {v['title']}") print(f" Transliteration: {v['transliteration'][:80]}...") print(f" Devanagari: {v['devanagari'][:80]}...") print(f" Synonyms: {len(v['synonyms'])} entries") if v["synonyms"]: print(f" First: {v['synonyms'][0]}") print(f" Last: {v['synonyms'][-1]}") trans_preview = re.sub(r"<[^>]+>", "", v["translation"])[:100] print(f" Translation: {trans_preview}...") print(f" Purport ¶s: {len(v['purport'])}") print(f"\n--- Sample: Text Six (middle) ---") t6 = [v for v in verses if v["text"] == 6] if t6: v = t6[0] print(f" Reference: {v['reference']}") print(f" Title: {v['title']}") print(f" Transliteration: {v['transliteration'][:80]}...") print(f" Devanagari: {v['devanagari'][:80]}...") print(f" Synonyms: {len(v['synonyms'])} entries") trans_preview = re.sub(r"<[^>]+>", "", v["translation"])[:100] print(f" Translation: {trans_preview}...") print(f" Purport ¶s: {len(v['purport'])}") print(f"\n--- Sample: Text Eleven (last) ---") t11 = [v for v in verses if v["text"] == 11] if t11: v = t11[0] print(f" Reference: {v['reference']}") print(f" Title: {v['title']}") print(f" Transliteration: {v['transliteration'][:80]}...") print(f" Devanagari: {v['devanagari'][:80]}...") print(f" Synonyms: {len(v['synonyms'])} entries") trans_preview = re.sub(r"<[^>]+>", "", v["translation"])[:100] print(f" Translation: {trans_preview}...") print(f" Purport ¶s: {len(v['purport'])}") if __name__ == "__main__": main()