351 lines
12 KiB
Python
351 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""Extract Bhagavad Gita verses from HTML into structured JSON files."""
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
from bs4 import BeautifulSoup, Tag
|
|
from indic_transliteration import sanscript
|
|
from indic_transliteration.sanscript import transliterate
|
|
|
|
# --- ScaGoudy → IAST Unicode mapping ---
|
|
SCAGOUDY_TO_IAST = {
|
|
'ä': 'ā', 'Ä': 'Ā',
|
|
'å': 'ṛ', 'Å': 'Ṛ',
|
|
'ñ': 'ṣ', 'Ñ': 'Ṣ',
|
|
'ö': 'ṭ', 'Ö': 'Ṭ',
|
|
'ë': 'ṇ', 'Ë': 'Ṇ',
|
|
'ò': 'ḍ', 'Ò': 'Ḍ',
|
|
'ç': 'ś', 'Ç': 'Ś',
|
|
'ì': 'ṅ',
|
|
'ï': 'ñ',
|
|
'ü': 'ū', 'Ü': 'Ū',
|
|
'é': 'ī', 'É': 'Ī',
|
|
'à': 'ṁ',
|
|
'ù': 'ḥ',
|
|
}
|
|
|
|
|
|
def decode_scagoudy(text: str) -> str:
|
|
"""Convert ScaGoudy-encoded text to proper IAST Unicode."""
|
|
for src, dst in SCAGOUDY_TO_IAST.items():
|
|
text = text.replace(src, dst)
|
|
return text
|
|
|
|
|
|
def decode_html_content(tag: Tag) -> str:
|
|
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
|
|
# Get inner HTML as string
|
|
inner = ''.join(str(child) for child in tag.children)
|
|
# Decode ScaGoudy characters
|
|
inner = decode_scagoudy(inner)
|
|
# Remove span tags with CSS classes (artifacts from RTF conversion)
|
|
inner = re.sub(r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r'\1', inner)
|
|
# Clean up whitespace
|
|
inner = re.sub(r'\s+', ' ', inner).strip()
|
|
return inner
|
|
|
|
|
|
def decode_text(tag: Tag) -> str:
|
|
"""Extract plain text from a tag and decode ScaGoudy."""
|
|
text = tag.get_text()
|
|
return decode_scagoudy(text).strip()
|
|
|
|
|
|
def normalize_devanagari_anusvara(text: str) -> str:
|
|
"""Replace explicit class nasals before same-class consonants with anusvāra.
|
|
|
|
In modern Devanagari convention, anusvāra (ं) is preferred over explicit
|
|
nasal consonants (ङ्, ञ्, ण्, न्, म्) when they appear before consonants
|
|
of their own class.
|
|
|
|
e.g., सङ्गम्य → संगम्य, सञ्जय → संजय
|
|
"""
|
|
# Only normalize ङ् and ञ् — these are almost always written with anusvāra.
|
|
# Keep ण्ड, न्त, म्ब etc. as explicit conjuncts (standard spelling).
|
|
replacements = [
|
|
# ङ् before velars (क, ख, ग, घ)
|
|
(r'ङ्([कखगघ])', r'ं\1'),
|
|
# ञ् before palatals (च, छ, ज, झ)
|
|
(r'ञ्([चछजझ])', r'ं\1'),
|
|
]
|
|
for pattern, repl in replacements:
|
|
text = re.sub(pattern, repl, text)
|
|
return text
|
|
|
|
|
|
def iast_to_devanagari(iast_text: str) -> str:
|
|
"""Convert IAST transliteration to Devanagari script."""
|
|
lines = iast_text.split('\n')
|
|
result_lines = []
|
|
for line in lines:
|
|
# Remove hyphens between words for Devanagari
|
|
clean = line.replace('-', '')
|
|
# Normalize ṁ (U+1E41) to ṃ (U+1E43) for proper anusvāra
|
|
clean = clean.replace('ṁ', 'ṃ')
|
|
deva = transliterate(clean, sanscript.IAST, sanscript.DEVANAGARI)
|
|
# Normalize class nasals to anusvāra
|
|
deva = normalize_devanagari_anusvara(deva)
|
|
result_lines.append(deva)
|
|
return '\n'.join(result_lines)
|
|
|
|
|
|
def parse_synonyms(text: str) -> list:
|
|
"""Parse synonym text into structured array.
|
|
|
|
Input format: "word—meaning; word—meaning; word—meaning"
|
|
Some entries use em-dash (—) and some use double-hyphen.
|
|
"""
|
|
result = []
|
|
# Split by semicolons
|
|
entries = text.split(';')
|
|
for entry in entries:
|
|
entry = entry.strip()
|
|
if not entry:
|
|
continue
|
|
# Split on em-dash (—) or double-hyphen
|
|
if '—' in entry:
|
|
parts = entry.split('—', 1)
|
|
elif '--' in entry:
|
|
parts = entry.split('--', 1)
|
|
else:
|
|
# Sometimes just "word-meaning" with single hyphen at boundary
|
|
# Skip entries without clear separator
|
|
continue
|
|
|
|
if len(parts) == 2:
|
|
word = parts[0].strip()
|
|
meaning = parts[1].strip()
|
|
# Clean word: remove trailing/leading punctuation
|
|
word = word.strip('., ')
|
|
if word and meaning:
|
|
result.append({
|
|
"word": word,
|
|
"meanings": [meaning]
|
|
})
|
|
|
|
return result
|
|
|
|
|
|
def parse_synonym_html(tag: Tag) -> list:
|
|
"""Parse synonyms from HTML tag, preserving italic markup for words."""
|
|
text = decode_text(tag)
|
|
return parse_synonyms(text)
|
|
|
|
|
|
def pair_padas(padas: list) -> str:
|
|
"""Pair pādas into verse lines (2 pādas per line).
|
|
|
|
A śloka has 4 pādas displayed as 2 lines. If there's an odd number,
|
|
the first pāda stands alone (usually a speaker attribution like 'arjuna uvāca').
|
|
"""
|
|
if not padas:
|
|
return ''
|
|
|
|
lines = []
|
|
i = 0
|
|
|
|
# If odd number of pādas, first one is standalone (speaker attribution)
|
|
if len(padas) % 2 == 1:
|
|
lines.append(padas[0])
|
|
i = 1
|
|
|
|
# Pair remaining pādas
|
|
while i < len(padas) - 1:
|
|
lines.append(padas[i] + ' ' + padas[i + 1])
|
|
i += 2
|
|
|
|
# Handle any trailing single pāda
|
|
if i < len(padas):
|
|
lines.append(padas[i])
|
|
|
|
return '\n'.join(lines)
|
|
|
|
|
|
def get_css_class(tag: Tag) -> str:
|
|
"""Get the primary CSS class of a tag."""
|
|
classes = tag.get('class', [])
|
|
return classes[0] if classes else ''
|
|
|
|
|
|
def _finalize_verse(verse: dict):
|
|
"""Build transliteration and Devanagari from pādas. Remove internal fields."""
|
|
padas = verse.get('_padas', [])
|
|
# Transliteration: one pāda per line (original structure)
|
|
verse['transliteration'] = '\n'.join(padas)
|
|
# Devanagari: pair pādas into 2-line verse structure
|
|
if padas:
|
|
verse['devanagari'] = iast_to_devanagari(pair_padas(padas))
|
|
del verse['_padas']
|
|
|
|
|
|
def extract_verses(soup: BeautifulSoup, max_chapter=1, max_verse=5):
|
|
"""Extract verses from parsed HTML using state machine."""
|
|
paragraphs = soup.find_all('p')
|
|
|
|
verses = []
|
|
current_verse = None
|
|
state = 'IDLE' # IDLE, TRANSLITERATION, SYNONYMS, TRANSLATION, PURPORT
|
|
current_chapter = 0
|
|
chapter_title = ""
|
|
|
|
for p in paragraphs:
|
|
cls = get_css_class(p)
|
|
text = p.get_text().strip()
|
|
|
|
# Chapter title
|
|
if cls == 'p11':
|
|
decoded = decode_text(p)
|
|
match = re.match(r'^(\d+)\.\s+(.+)$', decoded, re.DOTALL)
|
|
if match:
|
|
current_chapter = int(match.group(1))
|
|
chapter_title = match.group(2).replace('\n', ' ').strip()
|
|
if current_chapter > max_chapter:
|
|
break
|
|
|
|
# Verse reference (e.g., "Bg 1.1")
|
|
elif cls == 'p25':
|
|
# Finalize previous verse
|
|
if current_verse and state != 'IDLE':
|
|
verses.append(current_verse)
|
|
current_verse = None
|
|
state = 'IDLE'
|
|
|
|
match = re.search(r'Bg\s+(\d+)\.(\d+)', text)
|
|
if match:
|
|
ch = int(match.group(1))
|
|
vs = int(match.group(2))
|
|
|
|
if ch > max_chapter or (ch == max_chapter and vs > max_verse):
|
|
break
|
|
|
|
current_verse = {
|
|
'chapter': ch,
|
|
'verse': vs,
|
|
'reference': decode_scagoudy(text.strip()),
|
|
'devanagari': '',
|
|
'transliteration': '',
|
|
'_padas': [], # raw pādas, paired into lines later
|
|
'synonyms': [],
|
|
'translation': '',
|
|
'purport': []
|
|
}
|
|
state = 'VERSE_START'
|
|
|
|
# TEXT header (e.g., "TEXT 1")
|
|
elif cls == 'p26' and current_verse:
|
|
pass # Just confirms we're in a verse
|
|
|
|
# TEXT label
|
|
elif cls == 'p27' and current_verse:
|
|
pass # Skip
|
|
|
|
# Devanagari in Indevr font — SKIP (will derive from transliteration)
|
|
elif cls == 'p28' and current_verse:
|
|
state = 'TRANSLITERATION'
|
|
|
|
# Transliteration lines (individual pādas)
|
|
elif cls == 'p29' and current_verse and state in ('TRANSLITERATION', 'VERSE_START'):
|
|
state = 'TRANSLITERATION'
|
|
line = decode_text(p)
|
|
current_verse['_padas'].append(line)
|
|
|
|
# Synonyms
|
|
elif cls == 'p30' and current_verse:
|
|
state = 'SYNONYMS'
|
|
current_verse['synonyms'] = parse_synonym_html(p)
|
|
|
|
# Section headers (TRANSLATION / PURPORT)
|
|
elif cls in ('p31', 'p33') and current_verse:
|
|
header_text = text.strip().upper()
|
|
if 'TRANSLATION' in header_text:
|
|
state = 'TRANSLATION'
|
|
elif 'PURPORT' in header_text:
|
|
state = 'PURPORT'
|
|
|
|
# Translation text
|
|
elif cls == 'p32' and current_verse and state == 'TRANSLATION':
|
|
current_verse['translation'] = decode_html_content(p)
|
|
state = 'POST_TRANSLATION'
|
|
|
|
# Purport paragraphs (exclude Collier markers)
|
|
elif state == 'PURPORT' and current_verse and cls in ('p6', 'p12', 'p13', 'p4', 'p3', 'p36', 'p19', 'p20'):
|
|
content = decode_html_content(p)
|
|
if content and 'Collier' not in text:
|
|
current_verse['purport'].append(content)
|
|
|
|
# Collier end marker (can be p24 or p34)
|
|
elif cls in ('p24', 'p34') and current_verse:
|
|
_finalize_verse(current_verse)
|
|
verses.append(current_verse)
|
|
current_verse = None
|
|
state = 'IDLE'
|
|
|
|
# Don't forget the last verse if no Collier marker
|
|
if current_verse:
|
|
_finalize_verse(current_verse)
|
|
verses.append(current_verse)
|
|
|
|
return verses, current_chapter, chapter_title
|
|
|
|
|
|
def write_json(data, filepath):
|
|
"""Write JSON with proper Unicode and formatting."""
|
|
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
|
with open(filepath, 'w', encoding='utf-8') as f:
|
|
json.dump(data, f, ensure_ascii=False, indent=2)
|
|
print(f" Written: {filepath}")
|
|
|
|
|
|
def main():
|
|
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
html_path = os.path.join(base_dir, 'public', 'html', 'bg.html')
|
|
output_dir = os.path.join(base_dir, 'app', 'lib', 'books', 'bg')
|
|
|
|
print(f"Reading {html_path}...")
|
|
with open(html_path, 'r', encoding='utf-8') as f:
|
|
soup = BeautifulSoup(f, 'html.parser')
|
|
|
|
print("Extracting verses (Chapter 1, all verses)...")
|
|
verses, chapter_num, chapter_title = extract_verses(soup, max_chapter=1, max_verse=999)
|
|
|
|
print(f"\nExtracted {len(verses)} verses")
|
|
|
|
# Write intro.json
|
|
intro = {
|
|
"book": "bg",
|
|
"title": "Bhagavad-gītā As It Is",
|
|
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
|
"totalChapters": 18,
|
|
"chapters": [
|
|
{
|
|
"number": 1,
|
|
"title": chapter_title or "Observing the Armies on the Battlefield of Kurukṣetra",
|
|
"totalVerses": 46
|
|
}
|
|
]
|
|
}
|
|
write_json(intro, os.path.join(output_dir, 'intro.json'))
|
|
|
|
# Write verse JSON files
|
|
for verse in verses:
|
|
ch = str(verse['chapter']).zfill(2)
|
|
vs = str(verse['verse']).zfill(2)
|
|
filepath = os.path.join(output_dir, 'chapters', ch, 'verses', f'{vs}.json')
|
|
write_json(verse, filepath)
|
|
|
|
# Print summary for verification
|
|
print("\n--- Verification ---")
|
|
for verse in verses:
|
|
print(f"\n=== {verse['reference']} ===")
|
|
print(f"Devanagari: {verse['devanagari'][:80]}...")
|
|
print(f"IAST: {verse['transliteration'][:80]}...")
|
|
print(f"Synonyms: {len(verse['synonyms'])} entries")
|
|
print(f"Translation: {verse['translation'][:80]}...")
|
|
print(f"Purport: {len(verse['purport'])} paragraphs")
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|