807 lines
27 KiB
Python
807 lines
27 KiB
Python
#!/usr/bin/env python3
|
||
"""Extract Mukunda-mālā-stotra verses from other.html into structured JSON files."""
|
||
|
||
import json
|
||
import os
|
||
import re
|
||
import sys
|
||
|
||
from bs4 import BeautifulSoup, Tag
|
||
from indic_transliteration import sanscript
|
||
from indic_transliteration.sanscript import transliterate
|
||
|
||
# --- ScaGoudy → IAST Unicode mapping ---
|
||
SCAGOUDY_TO_IAST = {
|
||
"ä": "ā",
|
||
"Ä": "Ā",
|
||
"å": "ṛ",
|
||
"Å": "Ṛ",
|
||
"ñ": "ṣ",
|
||
"Ñ": "Ṣ",
|
||
"ö": "ṭ",
|
||
"Ö": "Ṭ",
|
||
"ë": "ṇ",
|
||
"Ë": "Ṇ",
|
||
"ò": "ḍ",
|
||
"Ò": "Ḍ",
|
||
"ç": "ś",
|
||
"Ç": "Ś",
|
||
"ì": "ṅ",
|
||
"ï": "ñ",
|
||
"ü": "ū",
|
||
"Ü": "Ū",
|
||
"é": "ī",
|
||
"É": "Ī",
|
||
"à": "ṁ",
|
||
"ù": "ḥ",
|
||
"è": "ṝ",
|
||
"û": "ṁl",
|
||
"ÿ": "ḷ",
|
||
}
|
||
|
||
# --- CSS class constants ---
|
||
BOOK_HEADER_CLASS = "p12" # Book title
|
||
AUTHOR_SUBTITLE_CLASS = "p28" # Author subtitle
|
||
INTRO_BOOKMARK_CLASS = "p64" # "MM Introduction" bookmark
|
||
SECTION_HEADING_CLASS = "p31" # "Introduction" heading (bold)
|
||
SUB_HEADING_CLASS = "p109" # "About the Present Edition" sub-heading
|
||
BODY_CLASS = "p48" # Body/purport paragraphs (MAIN content class)
|
||
ATTRIBUTION_CLASS = "p47" # "—The Publishers"
|
||
VERSE_BOOKMARK_CLASS = "p49" # Verse bookmarks ("MM mantra N")
|
||
VERSE_HEADING_V1_CLASS = "p6" # "TEXT 1" heading (variant for text 1)
|
||
VERSE_HEADING_CLASS = "p57" # "TEXT N" heading for texts 2-6 (bold)
|
||
TEXT_LABEL_CLASS = "p52" # "TEXT" label (skip)
|
||
PADA_TRANSLIT_CLASS = "p5" # Transliteration pādas (italic)
|
||
SYNONYMS_HEADER_CLASS = "p165" # "SYNONYMS" header
|
||
SYNONYM_TEXT_CLASS = "p54" # Synonyms text
|
||
TRANSLATION_LABEL_CLASS = "p55" # "TRANSLATION" / "PURPORT" headers
|
||
TRANSLATION_TEXT_CLASS = "p56" # Translation text (bold)
|
||
QUOTED_VERSE_CLASS = "p53" # Embedded Sanskrit verse in purport
|
||
COMMENTATOR_CLASS = "p44" # Verse translation within purport
|
||
|
||
|
||
def decode_scagoudy(text: str) -> str:
|
||
"""Convert ScaGoudy-encoded text to proper IAST Unicode."""
|
||
for src, dst in SCAGOUDY_TO_IAST.items():
|
||
text = text.replace(src, dst)
|
||
return text
|
||
|
||
|
||
def decode_html_content(tag: Tag) -> str:
|
||
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
|
||
inner = "".join(str(child) for child in tag.children)
|
||
inner = decode_scagoudy(inner)
|
||
# Remove span tags with CSS classes (artifacts from RTF conversion)
|
||
inner = re.sub(
|
||
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
|
||
)
|
||
# Clean up Apple-tab-span leftovers (tab characters)
|
||
inner = inner.replace("\t", " ")
|
||
# Normalize whitespace
|
||
inner = re.sub(r"\s+", " ", inner).strip()
|
||
return inner
|
||
|
||
|
||
def decode_text(tag: Tag) -> str:
|
||
"""Extract plain text from a tag and decode ScaGoudy."""
|
||
text = tag.get_text()
|
||
return decode_scagoudy(text).strip()
|
||
|
||
|
||
def normalize_devanagari_anusvara(text: str) -> str:
|
||
"""Replace explicit class nasals before same-class consonants with anusvāra.
|
||
|
||
Only normalize ङ् and ञ् — these are almost always written with anusvāra.
|
||
Keep ण्ड, न्त, म्ब etc. as explicit conjuncts (standard spelling).
|
||
"""
|
||
replacements = [
|
||
# ङ् before velars (क, ख, ग, घ)
|
||
(r"ङ्([कखगघ])", r"ं\1"),
|
||
# ञ् before palatals (च, छ, ज, झ)
|
||
(r"ञ्([चछजझ])", r"ं\1"),
|
||
]
|
||
for pattern, repl in replacements:
|
||
text = re.sub(pattern, repl, text)
|
||
return text
|
||
|
||
|
||
def join_virama_words(text: str) -> str:
|
||
"""Join Devanagari words where the previous word ends with virāma (्).
|
||
|
||
Handles both virāma+consonant and virāma+vowel cases.
|
||
Uses [ \\t]+ instead of \\s+ to avoid joining across newlines.
|
||
"""
|
||
# Join virāma + space + consonant (क U+0915 through ह U+0939)
|
||
text = re.sub(r"्[ \t]+([क-ह])", r"्\1", text)
|
||
|
||
# Join virāma + space + independent vowel
|
||
_VOWEL_TO_MATRA = {
|
||
"अ": "", # inherent vowel — just drop virāma and the अ
|
||
"आ": "ा",
|
||
"इ": "ि",
|
||
"ई": "ी",
|
||
"उ": "ु",
|
||
"ऊ": "ू",
|
||
"ऋ": "ृ",
|
||
"ॠ": "ॄ",
|
||
"ऌ": "ॢ",
|
||
"ॡ": "ॣ",
|
||
"ए": "े",
|
||
"ऐ": "ै",
|
||
"ओ": "ो",
|
||
"औ": "ौ",
|
||
}
|
||
|
||
def _virama_vowel_replace(m: re.Match[str]) -> str:
|
||
vowel = m.group(1)
|
||
return _VOWEL_TO_MATRA.get(vowel, vowel)
|
||
|
||
text = re.sub(r"्[ \t]+([अ-औॠॡ])", _virama_vowel_replace, text)
|
||
return text
|
||
|
||
|
||
def iast_to_devanagari(iast_text: str) -> str:
|
||
"""Convert IAST transliteration to Devanagari script."""
|
||
lines = iast_text.split("\n")
|
||
result_lines = []
|
||
for line in lines:
|
||
# Remove hyphens between words for Devanagari
|
||
clean = line.replace("-", "")
|
||
# Normalize ṁ (U+1E41) to ṃ (U+1E43) for proper anusvāra
|
||
clean = clean.replace("ṁ", "ṃ")
|
||
deva = transliterate(clean, sanscript.IAST, sanscript.DEVANAGARI)
|
||
deva = normalize_devanagari_anusvara(deva)
|
||
deva = join_virama_words(deva)
|
||
result_lines.append(deva)
|
||
return "\n".join(result_lines)
|
||
|
||
|
||
def parse_synonyms(text: str) -> list:
|
||
"""Parse synonym text into structured array.
|
||
|
||
Input format: "word—meaning; word—meaning; word—meaning"
|
||
"""
|
||
result = []
|
||
entries = text.split(";")
|
||
for entry in entries:
|
||
entry = entry.strip()
|
||
if not entry:
|
||
continue
|
||
if "—" in entry:
|
||
parts = entry.split("—", 1)
|
||
elif "--" in entry:
|
||
parts = entry.split("--", 1)
|
||
else:
|
||
continue
|
||
|
||
if len(parts) == 2:
|
||
word = parts[0].strip()
|
||
meaning = parts[1].strip()
|
||
# Clean word: remove trailing/leading punctuation
|
||
word = word.strip("., ")
|
||
if word and meaning:
|
||
result.append({"word": word, "meanings": [meaning]})
|
||
|
||
return result
|
||
|
||
|
||
def pair_padas(padas: list) -> str:
|
||
"""Combine pādas into 2-line verse format.
|
||
|
||
If the number of pādas is odd, the first one is treated as a
|
||
standalone line (e.g. speaker attribution like 'uvāca').
|
||
|
||
Empty strings in the list act as stanza-break sentinels and are
|
||
rendered as blank lines in the output.
|
||
"""
|
||
if not padas:
|
||
return ""
|
||
|
||
# Split pādas into stanza groups separated by "" sentinels
|
||
groups: list[list[str]] = [[]]
|
||
for p in padas:
|
||
if p == "":
|
||
groups.append([])
|
||
else:
|
||
groups[-1].append(p)
|
||
|
||
result_lines = []
|
||
for group in groups:
|
||
if not group:
|
||
continue
|
||
|
||
lines = []
|
||
i = 0
|
||
|
||
# If odd number of pādas, first one is standalone (speaker attribution)
|
||
if len(group) % 2 == 1:
|
||
lines.append(group[0])
|
||
i = 1
|
||
|
||
# Pair remaining pādas
|
||
while i < len(group) - 1:
|
||
lines.append(group[i] + " " + group[i + 1])
|
||
i += 2
|
||
|
||
# Handle any trailing single pāda
|
||
if i < len(group):
|
||
lines.append(group[i])
|
||
|
||
result_lines.append("\n".join(lines))
|
||
|
||
return "\n\n".join(result_lines)
|
||
|
||
|
||
def get_css_class(tag: Tag) -> str:
|
||
"""Get the primary CSS class of a tag."""
|
||
classes = tag.get("class", [])
|
||
return classes[0] if classes else ""
|
||
|
||
|
||
def is_empty(tag: Tag) -> bool:
|
||
"""Check if a tag is empty or whitespace-only."""
|
||
text = tag.get_text().strip()
|
||
return not text or text == "\xa0"
|
||
|
||
|
||
def write_json(data, filepath):
|
||
"""Write JSON with proper Unicode and formatting."""
|
||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||
with open(filepath, "w", encoding="utf-8") as f:
|
||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||
print(f" Written: {filepath}")
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Boundary detection
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def find_mm_boundaries(all_paragraphs: list) -> tuple[int, int]:
|
||
"""Find the start and end paragraph indices of the Mukunda-mālā-stotra content.
|
||
|
||
Start: <p class="p12"> containing "Mukunda-mälä-stotra" (ScaGoudy encoded)
|
||
End: next <p class="p12"> after start (start of next book — Nārada-bhakti-sūtra)
|
||
"""
|
||
start_idx = None
|
||
end_idx = len(all_paragraphs)
|
||
|
||
for i, p in enumerate(all_paragraphs):
|
||
cls = get_css_class(p)
|
||
if cls == BOOK_HEADER_CLASS:
|
||
text = p.get_text().strip()
|
||
if start_idx is None and "Mukunda" in text:
|
||
start_idx = i
|
||
continue
|
||
# Next p12 after our start → end boundary
|
||
if start_idx is not None:
|
||
end_idx = i
|
||
break
|
||
|
||
if start_idx is None:
|
||
print("ERROR: Could not find 'Mukunda-mālā-stotra' header in other.html!")
|
||
sys.exit(1)
|
||
|
||
return start_idx, end_idx
|
||
|
||
|
||
def parse_verse_bookmark(text: str) -> int | None:
|
||
"""Parse a verse bookmark like 'MM mantra N'.
|
||
|
||
Returns mantra number (1-6), or None.
|
||
"""
|
||
text = text.strip()
|
||
m = re.match(r"MM\s+mantra\s+(\d+)", text)
|
||
if m:
|
||
return int(m.group(1))
|
||
return None
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Section boundary detection within MM range
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def find_section_boundaries(all_paragraphs: list, start_idx: int, end_idx: int) -> dict:
|
||
"""Find key section boundaries within the MM content.
|
||
|
||
Returns dict with section names and their start indices.
|
||
"""
|
||
sections = {}
|
||
|
||
for i in range(start_idx, end_idx):
|
||
p = all_paragraphs[i]
|
||
cls = get_css_class(p)
|
||
text = p.get_text().strip()
|
||
|
||
# Introduction bookmark (p64 with "MM Introduction")
|
||
if cls == INTRO_BOOKMARK_CLASS and "MM Introduction" in text:
|
||
sections["introduction_bookmark"] = i
|
||
|
||
# First verse bookmark (MM mantra 1)
|
||
elif cls == VERSE_BOOKMARK_CLASS:
|
||
decoded = decode_text(p)
|
||
vnum = parse_verse_bookmark(decoded)
|
||
if vnum == 1 and "first_verse" not in sections:
|
||
sections["first_verse"] = i
|
||
break
|
||
|
||
return sections
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Intro segment extraction
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def extract_intro_segment(
|
||
all_paragraphs: list, start_idx: int, end_idx: int
|
||
) -> list[str]:
|
||
"""Extract paragraphs for the introduction segment.
|
||
|
||
Decodes ScaGoudy, preserves HTML formatting, skips empty paragraphs.
|
||
Includes p48 (body), p47 (attribution), p109 (sub-heading) content.
|
||
"""
|
||
result = []
|
||
|
||
for i in range(start_idx, end_idx):
|
||
p = all_paragraphs[i]
|
||
cls = get_css_class(p)
|
||
|
||
if is_empty(p):
|
||
continue
|
||
|
||
# Skip bookmark and header classes
|
||
if cls in (
|
||
INTRO_BOOKMARK_CLASS,
|
||
BOOK_HEADER_CLASS,
|
||
SECTION_HEADING_CLASS,
|
||
AUTHOR_SUBTITLE_CLASS,
|
||
):
|
||
continue
|
||
|
||
# Include body, attribution, and sub-heading classes
|
||
if cls in (BODY_CLASS, ATTRIBUTION_CLASS, SUB_HEADING_CLASS):
|
||
html = decode_html_content(p)
|
||
if html:
|
||
result.append(html)
|
||
|
||
return result
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Verse extraction state machine
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def extract_verses(all_paragraphs: list, verses_start: int, verses_end: int) -> list:
|
||
"""Extract verses from the MM content using a state machine.
|
||
|
||
States: IDLE → VERSE_START → TRANSLITERATION → SYNONYMS →
|
||
TRANSLATION → POST_TRANSLATION → PURPORT → (back to IDLE)
|
||
|
||
Returns list of verse dicts.
|
||
"""
|
||
verses = []
|
||
current_verse = None
|
||
state = "IDLE"
|
||
|
||
for i in range(verses_start, verses_end):
|
||
p = all_paragraphs[i]
|
||
cls = get_css_class(p)
|
||
|
||
# --- Verse bookmark (p49) ---
|
||
if cls == VERSE_BOOKMARK_CLASS:
|
||
# Finalize previous verse if any
|
||
if current_verse and state != "IDLE":
|
||
_finalize_verse(current_verse)
|
||
verses.append(current_verse)
|
||
|
||
text = decode_text(p)
|
||
verse_num = parse_verse_bookmark(text)
|
||
if verse_num is not None:
|
||
reference = f"MM {verse_num}"
|
||
|
||
current_verse = {
|
||
"book_title": "Mukunda-mālā-stotra",
|
||
"chapter": 1,
|
||
"verse": verse_num,
|
||
"reference": reference,
|
||
"devanagari": "",
|
||
"transliteration": "",
|
||
"_padas": [],
|
||
"synonyms": [],
|
||
"translation": "",
|
||
"purport": [],
|
||
}
|
||
state = "VERSE_START"
|
||
continue
|
||
|
||
if not current_verse:
|
||
continue
|
||
|
||
# --- Verse heading (p6 for text 1, p57 for texts 2-6) - confirms text number ---
|
||
if cls in (VERSE_HEADING_V1_CLASS, VERSE_HEADING_CLASS):
|
||
inner_text = p.get_text().strip().upper()
|
||
# If this says "TEXT N" it's a verse heading — skip it
|
||
if "TEXT" in inner_text and state == "VERSE_START":
|
||
continue
|
||
# p57 can also be "PURPORT" header
|
||
if "PURPORT" in inner_text:
|
||
state = "PURPORT"
|
||
continue
|
||
# p57 can also be "TRANSLATION" header
|
||
if "TRANSLATION" in inner_text:
|
||
state = "TRANSLATION"
|
||
continue
|
||
# Otherwise skip if we're in VERSE_START
|
||
if state == "VERSE_START":
|
||
continue
|
||
|
||
# --- TEXT label (p52) - transition to transliteration ---
|
||
if cls == TEXT_LABEL_CLASS:
|
||
state = "TRANSLITERATION"
|
||
continue
|
||
|
||
# --- Pāda transliteration lines (p5) ---
|
||
if cls == PADA_TRANSLIT_CLASS and state in ("VERSE_START", "TRANSLITERATION"):
|
||
state = "TRANSLITERATION"
|
||
line = decode_text(p)
|
||
# Remove trailing period
|
||
line = line.rstrip(".")
|
||
line = line.strip()
|
||
if line:
|
||
current_verse["_padas"].append(line)
|
||
continue
|
||
|
||
# --- SYNONYMS header (p165) ---
|
||
if cls == SYNONYMS_HEADER_CLASS:
|
||
state = "SYNONYMS"
|
||
continue
|
||
|
||
# --- Synonyms text (p54) ---
|
||
if cls == SYNONYM_TEXT_CLASS and current_verse:
|
||
state = "SYNONYMS"
|
||
syn_text = decode_text(p)
|
||
syn_text = syn_text.strip()
|
||
parsed = parse_synonyms(syn_text)
|
||
current_verse["synonyms"].extend(parsed)
|
||
continue
|
||
|
||
# --- TRANSLATION label (p55) — check text content to distinguish from PURPORT ---
|
||
if cls == TRANSLATION_LABEL_CLASS and current_verse:
|
||
inner_text = p.get_text().strip().upper()
|
||
if "TRANSLATION" in inner_text:
|
||
state = "TRANSLATION"
|
||
elif "PURPORT" in inner_text:
|
||
state = "PURPORT"
|
||
continue
|
||
|
||
# --- PURPORT label via p57 (alt class) ---
|
||
if cls == VERSE_HEADING_CLASS and current_verse:
|
||
inner_text = p.get_text().strip().upper()
|
||
if "PURPORT" in inner_text:
|
||
state = "PURPORT"
|
||
continue
|
||
elif "TRANSLATION" in inner_text:
|
||
state = "TRANSLATION"
|
||
continue
|
||
|
||
# --- Translation text (p56) ---
|
||
if cls == TRANSLATION_TEXT_CLASS and current_verse and state == "TRANSLATION":
|
||
html = decode_html_content(p)
|
||
# Remove bold wrapper if present (the whole translation is bold)
|
||
html = re.sub(r"^<b>(.*)</b>$", r"\1", html, flags=re.DOTALL)
|
||
html = html.strip()
|
||
current_verse["translation"] = html
|
||
state = "POST_TRANSLATION"
|
||
continue
|
||
|
||
# --- Purport paragraphs (p48) ---
|
||
if (
|
||
cls == BODY_CLASS
|
||
and current_verse
|
||
and state in ("PURPORT", "POST_TRANSLATION")
|
||
):
|
||
state = "PURPORT"
|
||
html = decode_html_content(p)
|
||
if html:
|
||
current_verse["purport"].append(html)
|
||
continue
|
||
|
||
# --- Embedded Sanskrit verse in purport (p53) ---
|
||
if cls == QUOTED_VERSE_CLASS and current_verse and state == "PURPORT":
|
||
html = decode_html_content(p)
|
||
if html:
|
||
# Wrap as centered italic div for display
|
||
current_verse["purport"].append(
|
||
f'<div class="quoted-verse">{html}</div>'
|
||
)
|
||
continue
|
||
|
||
# --- Verse translation within purport (p44) ---
|
||
if cls == COMMENTATOR_CLASS and current_verse and state == "PURPORT":
|
||
html = decode_html_content(p)
|
||
if html:
|
||
current_verse["purport"].append(html)
|
||
continue
|
||
|
||
# --- p6 paragraphs in purport context (can appear as body text) ---
|
||
if (
|
||
cls == VERSE_HEADING_V1_CLASS
|
||
and current_verse
|
||
and state in ("PURPORT", "POST_TRANSLATION")
|
||
):
|
||
state = "PURPORT"
|
||
html = decode_html_content(p)
|
||
if html:
|
||
current_verse["purport"].append(html)
|
||
continue
|
||
|
||
# Finalize last verse
|
||
if current_verse and state != "IDLE":
|
||
_finalize_verse(current_verse)
|
||
verses.append(current_verse)
|
||
|
||
return verses
|
||
|
||
|
||
def _finalize_verse(verse: dict):
|
||
"""Build transliteration and Devanagari from pādas. Remove internal fields."""
|
||
padas = verse.get("_padas", [])
|
||
|
||
# Transliteration: one pāda per line, blank sentinels become blank lines
|
||
trans_lines = []
|
||
for p in padas:
|
||
if p == "":
|
||
trans_lines.append("") # blank line between stanza groups
|
||
else:
|
||
trans_lines.append(p)
|
||
verse["transliteration"] = "\n".join(trans_lines)
|
||
|
||
# Devanagari: pair pādas into 2-line verse structure, then convert
|
||
if padas:
|
||
paired = pair_padas(padas)
|
||
verse["devanagari"] = iast_to_devanagari(paired)
|
||
|
||
# Clean up internal field
|
||
if "_padas" in verse:
|
||
del verse["_padas"]
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Main
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def main():
|
||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
html_path = os.path.join(base_dir, "public", "html", "other.html")
|
||
output_base = os.path.join(base_dir, "lib", "jsons", "books", "mm")
|
||
|
||
if not os.path.exists(html_path):
|
||
print(f"ERROR: {html_path} not found!")
|
||
sys.exit(1)
|
||
|
||
print(f"Reading {html_path}...")
|
||
with open(html_path, "r", encoding="utf-8") as f:
|
||
soup = BeautifulSoup(f, "html.parser")
|
||
|
||
all_paragraphs = soup.find_all("p")
|
||
print(f"Total paragraphs in file: {len(all_paragraphs)}")
|
||
|
||
# --- Find MM boundaries ---
|
||
mm_start, mm_end = find_mm_boundaries(all_paragraphs)
|
||
print(f"MM content: paragraphs {mm_start} to {mm_end}")
|
||
|
||
# --- Find section boundaries ---
|
||
sections = find_section_boundaries(all_paragraphs, mm_start, mm_end)
|
||
print(f"\nSection boundaries found:")
|
||
for name, idx in sorted(sections.items(), key=lambda x: x[1]):
|
||
print(f" [{idx}] {name}")
|
||
|
||
intro_bm = sections.get("introduction_bookmark")
|
||
first_verse = sections.get("first_verse")
|
||
|
||
if first_verse is None:
|
||
print("ERROR: Could not find first verse bookmark (MM mantra 1)!")
|
||
sys.exit(1)
|
||
|
||
# --- Extract intro segment ---
|
||
print("\n--- Extracting Intro Segment ---")
|
||
|
||
intro_segments = []
|
||
|
||
if intro_bm is not None:
|
||
intro_end = first_verse
|
||
intro_paras = extract_intro_segment(all_paragraphs, intro_bm, intro_end)
|
||
intro_segment = {
|
||
"slug": "introduction",
|
||
"title": "Introduction",
|
||
"type": "intro-segment",
|
||
"book": "mm",
|
||
"order": 1,
|
||
"paragraphs": intro_paras,
|
||
}
|
||
intro_segments.append(intro_segment)
|
||
print(f" Introduction: {len(intro_paras)} paragraphs")
|
||
|
||
# --- Extract verses ---
|
||
print("\n--- Extracting Verses ---")
|
||
|
||
verses = extract_verses(all_paragraphs, first_verse, mm_end)
|
||
print(f"Extracted {len(verses)} verses (expected 6: Texts 1-6)")
|
||
|
||
# --- Write intro.json ---
|
||
intro_meta = {
|
||
"book": "mm",
|
||
"title": "Mukunda-mālā-stotra",
|
||
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
||
"year": 1989,
|
||
"description": "King Kulaśekhara's prayers to Lord Mukunda — translations and purports for verses 1–6.",
|
||
"totalChapters": 1,
|
||
"chapters": [
|
||
{
|
||
"number": 1,
|
||
"title": "Prayers of King Kulaśekhara",
|
||
"totalVerses": 6,
|
||
}
|
||
],
|
||
}
|
||
write_json(intro_meta, os.path.join(output_base, "intro.json"))
|
||
|
||
# --- Write intro segments ---
|
||
print("\n--- Writing Intro Segments ---")
|
||
for seg in intro_segments:
|
||
filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
|
||
write_json(seg, filepath)
|
||
|
||
# --- Write verse JSON files ---
|
||
print("\n--- Writing Verse Files ---")
|
||
for verse in verses:
|
||
text_num = verse["verse"]
|
||
filename = f"{str(text_num).zfill(2)}.json"
|
||
filepath = os.path.join(output_base, "chapters", "01", "verses", filename)
|
||
write_json(verse, filepath)
|
||
|
||
# --- Summary ---
|
||
print(f"\n{'=' * 60}")
|
||
print(f" EXTRACTION SUMMARY")
|
||
print(f"{'=' * 60}")
|
||
print(f" Book: Mukunda-mālā-stotra")
|
||
print(f" Intro segments: {len(intro_segments)}")
|
||
for seg in intro_segments:
|
||
print(
|
||
f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs"
|
||
)
|
||
print(f" Verses extracted: {len(verses)}")
|
||
texts_found = sorted([v["verse"] for v in verses])
|
||
if texts_found:
|
||
print(
|
||
f" Texts: {texts_found[0]}-{texts_found[-1]} ({len(texts_found)} total)"
|
||
)
|
||
else:
|
||
print(" Texts: None")
|
||
|
||
total_purport_paras = sum(len(v["purport"]) for v in verses)
|
||
total_synonyms = sum(len(v["synonyms"]) for v in verses)
|
||
print(f" Total purport ¶s: {total_purport_paras}")
|
||
print(f" Total synonyms: {total_synonyms}")
|
||
print(f" Output directory: {output_base}/")
|
||
|
||
# --- Quality checks ---
|
||
print(f"\n--- Quality Checks ---")
|
||
|
||
# Check all verses have expected fields
|
||
issues = []
|
||
for verse in verses:
|
||
ref = verse["reference"]
|
||
if not verse["transliteration"]:
|
||
issues.append(f" {ref}: missing transliteration")
|
||
if not verse["devanagari"]:
|
||
issues.append(f" {ref}: missing devanagari")
|
||
if not verse["synonyms"]:
|
||
issues.append(f" {ref}: missing synonyms")
|
||
if not verse["translation"]:
|
||
issues.append(f" {ref}: missing translation")
|
||
if not verse["purport"]:
|
||
issues.append(f" {ref}: missing purport (may be intentional)")
|
||
|
||
if issues:
|
||
print(" ISSUES FOUND:")
|
||
for issue in issues:
|
||
print(issue)
|
||
else:
|
||
print(
|
||
" All verses have transliteration, devanagari, synonyms, and translation ✓"
|
||
)
|
||
|
||
# Check expected verse count
|
||
expected_texts = set(range(1, 7))
|
||
found_texts = set(v["verse"] for v in verses)
|
||
missing = expected_texts - found_texts
|
||
extra = found_texts - expected_texts
|
||
if missing:
|
||
print(f" MISSING texts: {sorted(missing)}")
|
||
if extra:
|
||
print(f" UNEXPECTED texts: {sorted(extra)}")
|
||
if not missing and not extra:
|
||
print(f" All 6 texts found ✓")
|
||
|
||
# --- ScaGoudy spot checks ---
|
||
print(f"\n--- ScaGoudy Spot Checks ---")
|
||
for verse in verses:
|
||
ref = verse["reference"]
|
||
trans = verse["transliteration"]
|
||
# Check for leftover ScaGoudy characters
|
||
leftover = set()
|
||
for ch in trans:
|
||
if ch in SCAGOUDY_TO_IAST:
|
||
leftover.add(ch)
|
||
if leftover:
|
||
print(
|
||
f" WARNING: {ref} transliteration has leftover ScaGoudy chars: {leftover}"
|
||
)
|
||
else:
|
||
print(f" {ref}: transliteration clean ✓")
|
||
|
||
# --- Samples ---
|
||
print(f"\n--- Sample: Text 1 (first) ---")
|
||
t1 = [v for v in verses if v["verse"] == 1]
|
||
if t1:
|
||
v = t1[0]
|
||
print(f" Reference: {v['reference']}")
|
||
print(f" Transliteration: {v['transliteration'][:100]}...")
|
||
print(f" Devanagari: {v['devanagari'][:100]}...")
|
||
print(f" Synonyms: {len(v['synonyms'])} entries")
|
||
if v["synonyms"]:
|
||
print(f" First: {v['synonyms'][0]}")
|
||
print(f" Last: {v['synonyms'][-1]}")
|
||
trans_preview = re.sub(r"<[^>]+>", "", v["translation"])[:120]
|
||
print(f" Translation: {trans_preview}...")
|
||
print(f" Purport ¶s: {len(v['purport'])}")
|
||
|
||
print(f"\n--- Sample: Text 3 (middle) ---")
|
||
t3 = [v for v in verses if v["verse"] == 3]
|
||
if t3:
|
||
v = t3[0]
|
||
print(f" Reference: {v['reference']}")
|
||
print(f" Transliteration: {v['transliteration'][:100]}...")
|
||
print(f" Devanagari: {v['devanagari'][:100]}...")
|
||
print(f" Synonyms: {len(v['synonyms'])} entries")
|
||
if v["synonyms"]:
|
||
print(f" First: {v['synonyms'][0]}")
|
||
print(f" Last: {v['synonyms'][-1]}")
|
||
trans_preview = re.sub(r"<[^>]+>", "", v["translation"])[:120]
|
||
print(f" Translation: {trans_preview}...")
|
||
print(f" Purport ¶s: {len(v['purport'])}")
|
||
|
||
print(f"\n--- Sample: Text 6 (last) ---")
|
||
t6 = [v for v in verses if v["verse"] == 6]
|
||
if t6:
|
||
v = t6[0]
|
||
print(f" Reference: {v['reference']}")
|
||
print(f" Transliteration: {v['transliteration'][:100]}...")
|
||
print(f" Devanagari: {v['devanagari'][:100]}...")
|
||
print(f" Synonyms: {len(v['synonyms'])} entries")
|
||
if v["synonyms"]:
|
||
print(f" First: {v['synonyms'][0]}")
|
||
print(f" Last: {v['synonyms'][-1]}")
|
||
trans_preview = re.sub(r"<[^>]+>", "", v["translation"])[:120]
|
||
print(f" Translation: {trans_preview}...")
|
||
print(f" Purport ¶s: {len(v['purport'])}")
|
||
|
||
# --- Intro segment details ---
|
||
print(f"\n--- Intro Segment Details ---")
|
||
for seg in intro_segments:
|
||
print(f" {seg['title']}: {len(seg['paragraphs'])} paragraphs")
|
||
if seg["paragraphs"]:
|
||
first_preview = re.sub(r"<[^>]+>", "", seg["paragraphs"][0])[:120]
|
||
print(f" First ¶: {first_preview}...")
|
||
last_preview = re.sub(r"<[^>]+>", "", seg["paragraphs"][-1])[:120]
|
||
print(f" Last ¶: {last_preview}...")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|