444 lines
15 KiB
Python
444 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""Extract Bhagavad Gita verses from HTML into structured JSON files."""
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
from bs4 import BeautifulSoup, Tag
|
|
from indic_transliteration import sanscript
|
|
from indic_transliteration.sanscript import transliterate
|
|
|
|
# --- ScaGoudy → IAST Unicode mapping ---
|
|
SCAGOUDY_TO_IAST = {
|
|
"ä": "ā",
|
|
"Ä": "Ā",
|
|
"å": "ṛ",
|
|
"Å": "Ṛ",
|
|
"ñ": "ṣ",
|
|
"Ñ": "Ṣ",
|
|
"ö": "ṭ",
|
|
"Ö": "Ṭ",
|
|
"ë": "ṇ",
|
|
"Ë": "Ṇ",
|
|
"ò": "ḍ",
|
|
"Ò": "Ḍ",
|
|
"ç": "ś",
|
|
"Ç": "Ś",
|
|
"ì": "ṅ",
|
|
"ï": "ñ",
|
|
"ü": "ū",
|
|
"Ü": "Ū",
|
|
"é": "ī",
|
|
"É": "Ī",
|
|
"à": "ṁ",
|
|
"ù": "ḥ",
|
|
"è": "ṝ", # long vocalic r (U+1E5D)
|
|
"û": "ṁl", # anusvara + l (ScaGoudy ligature glyph)
|
|
"ÿ": "ḷ", # vocalic l (U+1E37)
|
|
}
|
|
|
|
|
|
def decode_scagoudy(text: str) -> str:
|
|
"""Convert ScaGoudy-encoded text to proper IAST Unicode."""
|
|
for src, dst in SCAGOUDY_TO_IAST.items():
|
|
text = text.replace(src, dst)
|
|
return text
|
|
|
|
|
|
def decode_html_content(tag: Tag) -> str:
|
|
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
|
|
# Get inner HTML as string
|
|
inner = "".join(str(child) for child in tag.children)
|
|
# Decode ScaGoudy characters
|
|
inner = decode_scagoudy(inner)
|
|
# Remove span tags with CSS classes (artifacts from RTF conversion)
|
|
inner = re.sub(r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner)
|
|
# Clean up whitespace
|
|
inner = re.sub(r"\s+", " ", inner).strip()
|
|
return inner
|
|
|
|
|
|
def decode_text(tag: Tag) -> str:
|
|
"""Extract plain text from a tag and decode ScaGoudy."""
|
|
text = tag.get_text()
|
|
return decode_scagoudy(text).strip()
|
|
|
|
|
|
def normalize_devanagari_anusvara(text: str) -> str:
|
|
"""Replace explicit class nasals before same-class consonants with anusvāra.
|
|
|
|
In modern Devanagari convention, anusvāra (ं) is preferred over explicit
|
|
nasal consonants (ङ्, ञ्, ण्, न्, म्) when they appear before consonants
|
|
of their own class.
|
|
|
|
e.g., सङ्गम्य → संगम्य, सञ्जय → संजय
|
|
"""
|
|
# Only normalize ङ् and ञ् — these are almost always written with anusvāra.
|
|
# Keep ण्ड, न्त, म्ब etc. as explicit conjuncts (standard spelling).
|
|
replacements = [
|
|
# ङ् before velars (क, ख, ग, घ)
|
|
(r"ङ्([कखगघ])", r"ं\1"),
|
|
# ञ् before palatals (च, छ, ज, झ)
|
|
(r"ञ्([चछजझ])", r"ं\1"),
|
|
]
|
|
for pattern, repl in replacements:
|
|
text = re.sub(pattern, repl, text)
|
|
return text
|
|
|
|
|
|
def join_virama_words(text: str) -> str:
|
|
"""Join Devanagari words where the previous word ends with virāma (्).
|
|
|
|
Handles both virāma+consonant and virāma+vowel cases.
|
|
In Devanagari script, a halant (virāma) consonant always absorbs the
|
|
following letter — whether consonant or vowel — without a space.
|
|
|
|
virāma + consonant: श्रीर् विजयो → श्रीर्विजयो
|
|
virāma + vowel: किम् अकुर्वत → किमकुर्वत
|
|
तैर् अलातायुधैः → तैरलातायुधैः
|
|
माम् एकं → मामेकं
|
|
|
|
Uses [ \\t]+ instead of \\s+ to avoid joining across newlines (line breaks
|
|
separate paired pādas and must be preserved).
|
|
"""
|
|
# Join virāma + space + consonant (क U+0915 through ह U+0939)
|
|
text = re.sub(r"्[ \t]+([क-ह])", r"्\1", text)
|
|
|
|
# Join virāma + space + independent vowel (अ U+0905 through औ U+0914, plus ॠ ॡ)
|
|
# Drop the virāma and replace the independent vowel with its dependent mātrā form.
|
|
_VOWEL_TO_MATRA = {
|
|
"अ": "", # inherent vowel — just drop virāma and the अ
|
|
"आ": "ा",
|
|
"इ": "ि",
|
|
"ई": "ी",
|
|
"उ": "ु",
|
|
"ऊ": "ू",
|
|
"ऋ": "ृ",
|
|
"ॠ": "ॄ",
|
|
"ऌ": "ॢ",
|
|
"ॡ": "ॣ",
|
|
"ए": "े",
|
|
"ऐ": "ै",
|
|
"ओ": "ो",
|
|
"औ": "ौ",
|
|
}
|
|
|
|
def _virama_vowel_replace(m):
|
|
vowel = m.group(1)
|
|
matra = _VOWEL_TO_MATRA.get(vowel, vowel)
|
|
return matra
|
|
|
|
text = re.sub(r"्[ \t]+([अ-औॠॡ])", _virama_vowel_replace, text)
|
|
return text
|
|
|
|
|
|
def iast_to_devanagari(iast_text: str) -> str:
|
|
"""Convert IAST transliteration to Devanagari script."""
|
|
lines = iast_text.split("\n")
|
|
result_lines = []
|
|
for line in lines:
|
|
# Remove hyphens between words for Devanagari
|
|
clean = line.replace("-", "")
|
|
# Normalize ṁ (U+1E41) to ṃ (U+1E43) for proper anusvāra
|
|
clean = clean.replace("ṁ", "ṃ")
|
|
deva = transliterate(clean, sanscript.IAST, sanscript.DEVANAGARI)
|
|
# Normalize class nasals to anusvāra
|
|
deva = normalize_devanagari_anusvara(deva)
|
|
# Join words where previous word ends with virāma (्)
|
|
deva = join_virama_words(deva)
|
|
result_lines.append(deva)
|
|
return "\n".join(result_lines)
|
|
|
|
|
|
def parse_synonyms(text: str) -> list:
|
|
"""Parse synonym text into structured array.
|
|
|
|
Input format: "word—meaning; word—meaning; word—meaning"
|
|
Some entries use em-dash (—) and some use double-hyphen.
|
|
"""
|
|
result = []
|
|
# Split by semicolons
|
|
entries = text.split(";")
|
|
for entry in entries:
|
|
entry = entry.strip()
|
|
if not entry:
|
|
continue
|
|
# Split on em-dash (—) or double-hyphen
|
|
if "—" in entry:
|
|
parts = entry.split("—", 1)
|
|
elif "--" in entry:
|
|
parts = entry.split("--", 1)
|
|
else:
|
|
# Sometimes just "word-meaning" with single hyphen at boundary
|
|
# Skip entries without clear separator
|
|
continue
|
|
|
|
if len(parts) == 2:
|
|
word = parts[0].strip()
|
|
meaning = parts[1].strip()
|
|
# Clean word: remove trailing/leading punctuation
|
|
word = word.strip("., ")
|
|
if word and meaning:
|
|
result.append({"word": word, "meanings": [meaning]})
|
|
|
|
return result
|
|
|
|
|
|
def parse_synonym_html(tag: Tag) -> list:
|
|
"""Parse synonyms from HTML tag, preserving italic markup for words."""
|
|
text = decode_text(tag)
|
|
return parse_synonyms(text)
|
|
|
|
|
|
def pair_padas(padas: list) -> str:
|
|
"""Pair pādas into verse lines (2 pādas per line).
|
|
|
|
A śloka has 4 pādas displayed as 2 lines. If there's an odd number,
|
|
the first pāda stands alone (usually a speaker attribution like 'arjuna uvāca').
|
|
"""
|
|
if not padas:
|
|
return ""
|
|
|
|
lines = []
|
|
i = 0
|
|
|
|
# If odd number of pādas, first one is standalone (speaker attribution)
|
|
if len(padas) % 2 == 1:
|
|
lines.append(padas[0])
|
|
i = 1
|
|
|
|
# Pair remaining pādas
|
|
while i < len(padas) - 1:
|
|
lines.append(padas[i] + " " + padas[i + 1])
|
|
i += 2
|
|
|
|
# Handle any trailing single pāda
|
|
if i < len(padas):
|
|
lines.append(padas[i])
|
|
|
|
return "\n".join(lines)
|
|
|
|
|
|
def get_css_class(tag: Tag) -> str:
|
|
"""Get the primary CSS class of a tag."""
|
|
classes = tag.get("class", [])
|
|
return classes[0] if classes else ""
|
|
|
|
|
|
def _finalize_verse(verse: dict):
|
|
"""Build transliteration and Devanagari from pādas. Remove internal fields."""
|
|
padas = verse.get("_padas", [])
|
|
# Transliteration: one pāda per line (original structure)
|
|
verse["transliteration"] = "\n".join(padas)
|
|
# Devanagari: pair pādas into 2-line verse structure
|
|
if padas:
|
|
verse["devanagari"] = iast_to_devanagari(pair_padas(padas))
|
|
del verse["_padas"]
|
|
|
|
|
|
def extract_verses(soup: BeautifulSoup, max_chapter=1, max_verse=5):
|
|
"""Extract verses from parsed HTML using state machine."""
|
|
paragraphs = soup.find_all("p")
|
|
|
|
verses = []
|
|
current_verse = None
|
|
state = "IDLE" # IDLE, TRANSLITERATION, SYNONYMS, TRANSLATION, PURPORT
|
|
current_chapter = 0
|
|
chapter_titles = {}
|
|
|
|
for p in paragraphs:
|
|
cls = get_css_class(p)
|
|
text = p.get_text().strip()
|
|
|
|
# Chapter title
|
|
if cls == "p11":
|
|
decoded = decode_text(p)
|
|
match = re.match(r"^(\d+)\.\s+(.+)$", decoded, re.DOTALL)
|
|
if match:
|
|
current_chapter = int(match.group(1))
|
|
chapter_titles[current_chapter] = (
|
|
match.group(2).replace("\n", " ").strip()
|
|
)
|
|
if current_chapter > max_chapter:
|
|
break
|
|
|
|
# Verse reference (e.g., "Bg 1.1")
|
|
elif cls == "p25":
|
|
# Finalize previous verse
|
|
if current_verse and state != "IDLE":
|
|
_finalize_verse(current_verse)
|
|
verses.append(current_verse)
|
|
current_verse = None
|
|
state = "IDLE"
|
|
|
|
match = re.search(r"Bg\s+(\d+)\.(\d+)", text)
|
|
if match:
|
|
ch = int(match.group(1))
|
|
vs = int(match.group(2))
|
|
|
|
if ch > max_chapter or (ch == max_chapter and vs > max_verse):
|
|
break
|
|
|
|
raw_ref = decode_scagoudy(text.strip())
|
|
# For multi-verse entries like "Bg 1.16, Bg 1.17, Bg 1.18, Bg 1.16-18",
|
|
# use only the last part: "Bg 1.16-18"
|
|
if "," in raw_ref:
|
|
clean_ref = raw_ref.rsplit(",", 1)[-1].strip()
|
|
else:
|
|
clean_ref = raw_ref
|
|
|
|
# Detect verse range (e.g., "Bg 1.16-18")
|
|
verse_end = None
|
|
range_match = re.search(r"Bg\s+\d+\.(\d+)-(\d+)", clean_ref)
|
|
if range_match:
|
|
verse_end = int(range_match.group(2))
|
|
|
|
current_verse = {
|
|
"book_title": "Bhagavad-gītā As It Is",
|
|
"chapter": ch,
|
|
"verse": vs,
|
|
"reference": clean_ref,
|
|
"devanagari": "",
|
|
"transliteration": "",
|
|
"_padas": [], # raw pādas, paired into lines later
|
|
"synonyms": [],
|
|
"translation": "",
|
|
"purport": [],
|
|
}
|
|
if verse_end is not None:
|
|
current_verse["verse_end"] = verse_end
|
|
state = "VERSE_START"
|
|
|
|
# TEXT header (e.g., "TEXT 1")
|
|
elif cls == "p26" and current_verse:
|
|
pass # Just confirms we're in a verse
|
|
|
|
# TEXT label
|
|
elif cls == "p27" and current_verse:
|
|
pass # Skip
|
|
|
|
# Devanagari in Indevr font — SKIP (will derive from transliteration)
|
|
elif cls == "p28" and current_verse:
|
|
state = "TRANSLITERATION"
|
|
|
|
# Transliteration lines (individual pādas)
|
|
# p29 is the primary class; p35 is an alternate class used for some pādas
|
|
elif (
|
|
cls in ("p29", "p35")
|
|
and current_verse
|
|
and state in ("TRANSLITERATION", "VERSE_START")
|
|
):
|
|
state = "TRANSLITERATION"
|
|
line = decode_text(p)
|
|
current_verse["_padas"].append(line)
|
|
|
|
# Synonyms
|
|
elif cls == "p30" and current_verse:
|
|
state = "SYNONYMS"
|
|
current_verse["synonyms"] = parse_synonym_html(p)
|
|
|
|
# Section headers (TRANSLATION / PURPORT)
|
|
# p31 and p33 are primary; p38 is an alternate class used in some verses
|
|
elif cls in ("p31", "p33", "p38") and current_verse:
|
|
header_text = text.strip().upper()
|
|
if "TRANSLATION" in header_text:
|
|
state = "TRANSLATION"
|
|
elif "PURPORT" in header_text:
|
|
state = "PURPORT"
|
|
|
|
# Translation text
|
|
elif cls == "p32" and current_verse and state == "TRANSLATION":
|
|
current_verse["translation"] = decode_html_content(p)
|
|
state = "POST_TRANSLATION"
|
|
|
|
# Purport paragraphs (exclude Collier markers)
|
|
elif (
|
|
state == "PURPORT"
|
|
and current_verse
|
|
and cls in ("p6", "p12", "p13", "p4", "p3", "p36", "p19", "p20")
|
|
):
|
|
content = decode_html_content(p)
|
|
if content and "Collier" not in text:
|
|
current_verse["purport"].append(content)
|
|
|
|
# Collier end marker (can be p24 or p34)
|
|
elif cls in ("p24", "p34") and current_verse:
|
|
_finalize_verse(current_verse)
|
|
verses.append(current_verse)
|
|
current_verse = None
|
|
state = "IDLE"
|
|
|
|
# Don't forget the last verse if no Collier marker
|
|
if current_verse:
|
|
_finalize_verse(current_verse)
|
|
verses.append(current_verse)
|
|
|
|
return verses, chapter_titles
|
|
|
|
|
|
def write_json(data, filepath):
|
|
"""Write JSON with proper Unicode and formatting."""
|
|
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
|
with open(filepath, "w", encoding="utf-8") as f:
|
|
json.dump(data, f, ensure_ascii=False, indent=2)
|
|
print(f" Written: {filepath}")
|
|
|
|
|
|
def main():
|
|
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
html_path = os.path.join(base_dir, "public", "html", "bg.html")
|
|
output_dir = os.path.join(base_dir, "lib", "jsons", "books", "bg")
|
|
|
|
print(f"Reading {html_path}...")
|
|
with open(html_path, "r", encoding="utf-8") as f:
|
|
soup = BeautifulSoup(f, "html.parser")
|
|
|
|
print("Extracting verses (all 18 chapters)...")
|
|
verses, chapter_titles = extract_verses(soup, max_chapter=18, max_verse=999)
|
|
|
|
print(f"\nExtracted {len(verses)} verses")
|
|
|
|
# Group verses by chapter to build intro.json
|
|
chapters_info = {}
|
|
for verse in verses:
|
|
ch = verse["chapter"]
|
|
if ch not in chapters_info:
|
|
chapters_info[ch] = 0
|
|
chapters_info[ch] += 1
|
|
|
|
# Write intro.json
|
|
intro = {
|
|
"book": "bg",
|
|
"title": "Bhagavad-gītā As It Is",
|
|
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
|
"totalChapters": 18,
|
|
"chapters": [
|
|
{"number": ch, "title": chapter_titles.get(ch, ""), "totalVerses": count}
|
|
for ch, count in sorted(chapters_info.items())
|
|
],
|
|
}
|
|
write_json(intro, os.path.join(output_dir, "intro.json"))
|
|
|
|
# Write verse JSON files
|
|
for verse in verses:
|
|
ch = str(verse["chapter"]).zfill(2)
|
|
vs = str(verse["verse"]).zfill(2)
|
|
filepath = os.path.join(output_dir, "chapters", ch, "verses", f"{vs}.json")
|
|
write_json(verse, filepath)
|
|
|
|
# Print summary for verification
|
|
print("\n--- Verification ---")
|
|
for ch in sorted(chapters_info.keys()):
|
|
ch_verses = [v for v in verses if v["chapter"] == ch]
|
|
print(f"\nChapter {ch}: {len(ch_verses)} verses")
|
|
first = ch_verses[0]
|
|
last = ch_verses[-1]
|
|
print(f" First: {first['reference']}")
|
|
print(f" Last: {last['reference']}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|