Initial commit
This commit is contained in:
gosangam
@@ -0,0 +1,350 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract Bhagavad Gita verses from HTML into structured JSON files."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
from indic_transliteration import sanscript
|
||||
from indic_transliteration.sanscript import transliterate
|
||||
|
||||
# --- ScaGoudy → IAST Unicode mapping ---
|
||||
SCAGOUDY_TO_IAST = {
|
||||
'ä': 'ā', 'Ä': 'Ā',
|
||||
'å': 'ṛ', 'Å': 'Ṛ',
|
||||
'ñ': 'ṣ', 'Ñ': 'Ṣ',
|
||||
'ö': 'ṭ', 'Ö': 'Ṭ',
|
||||
'ë': 'ṇ', 'Ë': 'Ṇ',
|
||||
'ò': 'ḍ', 'Ò': 'Ḍ',
|
||||
'ç': 'ś', 'Ç': 'Ś',
|
||||
'ì': 'ṅ',
|
||||
'ï': 'ñ',
|
||||
'ü': 'ū', 'Ü': 'Ū',
|
||||
'é': 'ī', 'É': 'Ī',
|
||||
'à': 'ṁ',
|
||||
'ù': 'ḥ',
|
||||
}
|
||||
|
||||
|
||||
def decode_scagoudy(text: str) -> str:
|
||||
"""Convert ScaGoudy-encoded text to proper IAST Unicode."""
|
||||
for src, dst in SCAGOUDY_TO_IAST.items():
|
||||
text = text.replace(src, dst)
|
||||
return text
|
||||
|
||||
|
||||
def decode_html_content(tag: Tag) -> str:
|
||||
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
|
||||
# Get inner HTML as string
|
||||
inner = ''.join(str(child) for child in tag.children)
|
||||
# Decode ScaGoudy characters
|
||||
inner = decode_scagoudy(inner)
|
||||
# Remove span tags with CSS classes (artifacts from RTF conversion)
|
||||
inner = re.sub(r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r'\1', inner)
|
||||
# Clean up whitespace
|
||||
inner = re.sub(r'\s+', ' ', inner).strip()
|
||||
return inner
|
||||
|
||||
|
||||
def decode_text(tag: Tag) -> str:
|
||||
"""Extract plain text from a tag and decode ScaGoudy."""
|
||||
text = tag.get_text()
|
||||
return decode_scagoudy(text).strip()
|
||||
|
||||
|
||||
def normalize_devanagari_anusvara(text: str) -> str:
|
||||
"""Replace explicit class nasals before same-class consonants with anusvāra.
|
||||
|
||||
In modern Devanagari convention, anusvāra (ं) is preferred over explicit
|
||||
nasal consonants (ङ्, ञ्, ण्, न्, म्) when they appear before consonants
|
||||
of their own class.
|
||||
|
||||
e.g., सङ्गम्य → संगम्य, सञ्जय → संजय
|
||||
"""
|
||||
# Only normalize ङ् and ञ् — these are almost always written with anusvāra.
|
||||
# Keep ण्ड, न्त, म्ब etc. as explicit conjuncts (standard spelling).
|
||||
replacements = [
|
||||
# ङ् before velars (क, ख, ग, घ)
|
||||
(r'ङ्([कखगघ])', r'ं\1'),
|
||||
# ञ् before palatals (च, छ, ज, झ)
|
||||
(r'ञ्([चछजझ])', r'ं\1'),
|
||||
]
|
||||
for pattern, repl in replacements:
|
||||
text = re.sub(pattern, repl, text)
|
||||
return text
|
||||
|
||||
|
||||
def iast_to_devanagari(iast_text: str) -> str:
|
||||
"""Convert IAST transliteration to Devanagari script."""
|
||||
lines = iast_text.split('\n')
|
||||
result_lines = []
|
||||
for line in lines:
|
||||
# Remove hyphens between words for Devanagari
|
||||
clean = line.replace('-', '')
|
||||
# Normalize ṁ (U+1E41) to ṃ (U+1E43) for proper anusvāra
|
||||
clean = clean.replace('ṁ', 'ṃ')
|
||||
deva = transliterate(clean, sanscript.IAST, sanscript.DEVANAGARI)
|
||||
# Normalize class nasals to anusvāra
|
||||
deva = normalize_devanagari_anusvara(deva)
|
||||
result_lines.append(deva)
|
||||
return '\n'.join(result_lines)
|
||||
|
||||
|
||||
def parse_synonyms(text: str) -> list:
|
||||
"""Parse synonym text into structured array.
|
||||
|
||||
Input format: "word—meaning; word—meaning; word—meaning"
|
||||
Some entries use em-dash (—) and some use double-hyphen.
|
||||
"""
|
||||
result = []
|
||||
# Split by semicolons
|
||||
entries = text.split(';')
|
||||
for entry in entries:
|
||||
entry = entry.strip()
|
||||
if not entry:
|
||||
continue
|
||||
# Split on em-dash (—) or double-hyphen
|
||||
if '—' in entry:
|
||||
parts = entry.split('—', 1)
|
||||
elif '--' in entry:
|
||||
parts = entry.split('--', 1)
|
||||
else:
|
||||
# Sometimes just "word-meaning" with single hyphen at boundary
|
||||
# Skip entries without clear separator
|
||||
continue
|
||||
|
||||
if len(parts) == 2:
|
||||
word = parts[0].strip()
|
||||
meaning = parts[1].strip()
|
||||
# Clean word: remove trailing/leading punctuation
|
||||
word = word.strip('., ')
|
||||
if word and meaning:
|
||||
result.append({
|
||||
"word": word,
|
||||
"meanings": [meaning]
|
||||
})
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def parse_synonym_html(tag: Tag) -> list:
|
||||
"""Parse synonyms from HTML tag, preserving italic markup for words."""
|
||||
text = decode_text(tag)
|
||||
return parse_synonyms(text)
|
||||
|
||||
|
||||
def pair_padas(padas: list) -> str:
|
||||
"""Pair pādas into verse lines (2 pādas per line).
|
||||
|
||||
A śloka has 4 pādas displayed as 2 lines. If there's an odd number,
|
||||
the first pāda stands alone (usually a speaker attribution like 'arjuna uvāca').
|
||||
"""
|
||||
if not padas:
|
||||
return ''
|
||||
|
||||
lines = []
|
||||
i = 0
|
||||
|
||||
# If odd number of pādas, first one is standalone (speaker attribution)
|
||||
if len(padas) % 2 == 1:
|
||||
lines.append(padas[0])
|
||||
i = 1
|
||||
|
||||
# Pair remaining pādas
|
||||
while i < len(padas) - 1:
|
||||
lines.append(padas[i] + ' ' + padas[i + 1])
|
||||
i += 2
|
||||
|
||||
# Handle any trailing single pāda
|
||||
if i < len(padas):
|
||||
lines.append(padas[i])
|
||||
|
||||
return '\n'.join(lines)
|
||||
|
||||
|
||||
def get_css_class(tag: Tag) -> str:
|
||||
"""Get the primary CSS class of a tag."""
|
||||
classes = tag.get('class', [])
|
||||
return classes[0] if classes else ''
|
||||
|
||||
|
||||
def _finalize_verse(verse: dict):
|
||||
"""Build transliteration and Devanagari from pādas. Remove internal fields."""
|
||||
padas = verse.get('_padas', [])
|
||||
# Transliteration: one pāda per line (original structure)
|
||||
verse['transliteration'] = '\n'.join(padas)
|
||||
# Devanagari: pair pādas into 2-line verse structure
|
||||
if padas:
|
||||
verse['devanagari'] = iast_to_devanagari(pair_padas(padas))
|
||||
del verse['_padas']
|
||||
|
||||
|
||||
def extract_verses(soup: BeautifulSoup, max_chapter=1, max_verse=5):
|
||||
"""Extract verses from parsed HTML using state machine."""
|
||||
paragraphs = soup.find_all('p')
|
||||
|
||||
verses = []
|
||||
current_verse = None
|
||||
state = 'IDLE' # IDLE, TRANSLITERATION, SYNONYMS, TRANSLATION, PURPORT
|
||||
current_chapter = 0
|
||||
chapter_title = ""
|
||||
|
||||
for p in paragraphs:
|
||||
cls = get_css_class(p)
|
||||
text = p.get_text().strip()
|
||||
|
||||
# Chapter title
|
||||
if cls == 'p11':
|
||||
decoded = decode_text(p)
|
||||
match = re.match(r'^(\d+)\.\s+(.+)$', decoded, re.DOTALL)
|
||||
if match:
|
||||
current_chapter = int(match.group(1))
|
||||
chapter_title = match.group(2).replace('\n', ' ').strip()
|
||||
if current_chapter > max_chapter:
|
||||
break
|
||||
|
||||
# Verse reference (e.g., "Bg 1.1")
|
||||
elif cls == 'p25':
|
||||
# Finalize previous verse
|
||||
if current_verse and state != 'IDLE':
|
||||
verses.append(current_verse)
|
||||
current_verse = None
|
||||
state = 'IDLE'
|
||||
|
||||
match = re.search(r'Bg\s+(\d+)\.(\d+)', text)
|
||||
if match:
|
||||
ch = int(match.group(1))
|
||||
vs = int(match.group(2))
|
||||
|
||||
if ch > max_chapter or (ch == max_chapter and vs > max_verse):
|
||||
break
|
||||
|
||||
current_verse = {
|
||||
'chapter': ch,
|
||||
'verse': vs,
|
||||
'reference': decode_scagoudy(text.strip()),
|
||||
'devanagari': '',
|
||||
'transliteration': '',
|
||||
'_padas': [], # raw pādas, paired into lines later
|
||||
'synonyms': [],
|
||||
'translation': '',
|
||||
'purport': []
|
||||
}
|
||||
state = 'VERSE_START'
|
||||
|
||||
# TEXT header (e.g., "TEXT 1")
|
||||
elif cls == 'p26' and current_verse:
|
||||
pass # Just confirms we're in a verse
|
||||
|
||||
# TEXT label
|
||||
elif cls == 'p27' and current_verse:
|
||||
pass # Skip
|
||||
|
||||
# Devanagari in Indevr font — SKIP (will derive from transliteration)
|
||||
elif cls == 'p28' and current_verse:
|
||||
state = 'TRANSLITERATION'
|
||||
|
||||
# Transliteration lines (individual pādas)
|
||||
elif cls == 'p29' and current_verse and state in ('TRANSLITERATION', 'VERSE_START'):
|
||||
state = 'TRANSLITERATION'
|
||||
line = decode_text(p)
|
||||
current_verse['_padas'].append(line)
|
||||
|
||||
# Synonyms
|
||||
elif cls == 'p30' and current_verse:
|
||||
state = 'SYNONYMS'
|
||||
current_verse['synonyms'] = parse_synonym_html(p)
|
||||
|
||||
# Section headers (TRANSLATION / PURPORT)
|
||||
elif cls in ('p31', 'p33') and current_verse:
|
||||
header_text = text.strip().upper()
|
||||
if 'TRANSLATION' in header_text:
|
||||
state = 'TRANSLATION'
|
||||
elif 'PURPORT' in header_text:
|
||||
state = 'PURPORT'
|
||||
|
||||
# Translation text
|
||||
elif cls == 'p32' and current_verse and state == 'TRANSLATION':
|
||||
current_verse['translation'] = decode_html_content(p)
|
||||
state = 'POST_TRANSLATION'
|
||||
|
||||
# Purport paragraphs (exclude Collier markers)
|
||||
elif state == 'PURPORT' and current_verse and cls in ('p6', 'p12', 'p13', 'p4', 'p3', 'p36', 'p19', 'p20'):
|
||||
content = decode_html_content(p)
|
||||
if content and 'Collier' not in text:
|
||||
current_verse['purport'].append(content)
|
||||
|
||||
# Collier end marker (can be p24 or p34)
|
||||
elif cls in ('p24', 'p34') and current_verse:
|
||||
_finalize_verse(current_verse)
|
||||
verses.append(current_verse)
|
||||
current_verse = None
|
||||
state = 'IDLE'
|
||||
|
||||
# Don't forget the last verse if no Collier marker
|
||||
if current_verse:
|
||||
_finalize_verse(current_verse)
|
||||
verses.append(current_verse)
|
||||
|
||||
return verses, current_chapter, chapter_title
|
||||
|
||||
|
||||
def write_json(data, filepath):
|
||||
"""Write JSON with proper Unicode and formatting."""
|
||||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||||
with open(filepath, 'w', encoding='utf-8') as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
print(f" Written: {filepath}")
|
||||
|
||||
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, 'public', 'html', 'bg.html')
|
||||
output_dir = os.path.join(base_dir, 'app', 'lib', 'books', 'bg')
|
||||
|
||||
print(f"Reading {html_path}...")
|
||||
with open(html_path, 'r', encoding='utf-8') as f:
|
||||
soup = BeautifulSoup(f, 'html.parser')
|
||||
|
||||
print("Extracting verses (Chapter 1, all verses)...")
|
||||
verses, chapter_num, chapter_title = extract_verses(soup, max_chapter=1, max_verse=999)
|
||||
|
||||
print(f"\nExtracted {len(verses)} verses")
|
||||
|
||||
# Write intro.json
|
||||
intro = {
|
||||
"book": "bg",
|
||||
"title": "Bhagavad-gītā As It Is",
|
||||
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
||||
"totalChapters": 18,
|
||||
"chapters": [
|
||||
{
|
||||
"number": 1,
|
||||
"title": chapter_title or "Observing the Armies on the Battlefield of Kurukṣetra",
|
||||
"totalVerses": 46
|
||||
}
|
||||
]
|
||||
}
|
||||
write_json(intro, os.path.join(output_dir, 'intro.json'))
|
||||
|
||||
# Write verse JSON files
|
||||
for verse in verses:
|
||||
ch = str(verse['chapter']).zfill(2)
|
||||
vs = str(verse['verse']).zfill(2)
|
||||
filepath = os.path.join(output_dir, 'chapters', ch, 'verses', f'{vs}.json')
|
||||
write_json(verse, filepath)
|
||||
|
||||
# Print summary for verification
|
||||
print("\n--- Verification ---")
|
||||
for verse in verses:
|
||||
print(f"\n=== {verse['reference']} ===")
|
||||
print(f"Devanagari: {verse['devanagari'][:80]}...")
|
||||
print(f"IAST: {verse['transliteration'][:80]}...")
|
||||
print(f"Synonyms: {len(verse['synonyms'])} entries")
|
||||
print(f"Translation: {verse['translation'][:80]}...")
|
||||
print(f"Purport: {len(verse['purport'])} paragraphs")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user