krispire/spsays-training
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: bg extracted

Browse Source
This commit is contained in:
gosangam
2026-04-14 14:36:05 +05:30
parent 9bf1e1125f
commit 82b6685c97
655 changed files with 62835 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/bg_extract.py
+151 -108
View File
@@ -5,25 +5,35 @@ import json
import os
import re
import sys
from bs4 import BeautifulSoup, Tag
from indic_transliteration import sanscript
from indic_transliteration.sanscript import transliterate
# --- ScaGoudy → IAST Unicode mapping ---
SCAGOUDY_TO_IAST = {
'ä': 'ā', 'Ä': 'Ā',
'å': 'ṛ', 'Å': 'Ṛ',
'ñ': 'ṣ', 'Ñ': 'Ṣ',
'ö': 'ṭ', 'Ö': 'Ṭ',
'ë': 'ṇ', 'Ë': 'Ṇ',
'ò': 'ḍ', 'Ò': 'Ḍ',
'ç': 'ś', 'Ç': 'Ś',
'ì': 'ṅ',
'ï': 'ñ',
'ü': 'ū', 'Ü': 'Ū',
'é': 'ī', 'É': 'Ī',
'à': 'ṁ',
'ù': 'ḥ',
"ä": "ā",
"Ä": "Ā",
"å": "ṛ",
"Å": "Ṛ",
"ñ": "ṣ",
"Ñ": "Ṣ",
"ö": "ṭ",
"Ö": "Ṭ",
"ë": "ṇ",
"Ë": "Ṇ",
"ò": "ḍ",
"Ò": "Ḍ",
"ç": "ś",
"Ç": "Ś",
"ì": "ṅ",
"ï": "ñ",
"ü": "ū",
"Ü": "Ū",
"é": "ī",
"É": "Ī",
"à": "ṁ",
"ù": "ḥ",
}
@@ -37,13 +47,13 @@ def decode_scagoudy(text: str) -> str:
def decode_html_content(tag: Tag) -> str:
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
# Get inner HTML as string
inner = ''.join(str(child) for child in tag.children)
inner = "".join(str(child) for child in tag.children)
# Decode ScaGoudy characters
inner = decode_scagoudy(inner)
# Remove span tags with CSS classes (artifacts from RTF conversion)
inner = re.sub(r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r'\1', inner)
inner = re.sub(r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner)
# Clean up whitespace
inner = re.sub(r'\s+', ' ', inner).strip()
inner = re.sub(r"\s+", " ", inner).strip()
return inner
@@ -66,29 +76,46 @@ def normalize_devanagari_anusvara(text: str) -> str:
# Keep ण्ड, न्त, म्ब etc. as explicit conjuncts (standard spelling).
replacements = [
# ङ् before velars (क, ख, ग, घ)
(r'ङ्([कखगघ])', r'ं\1'),
(r"ङ्([कखगघ])", r"ं\1"),
# ञ् before palatals (च, छ, ज, झ)
(r'ञ्([चछजझ])', r'ं\1'),
(r"ञ्([चछजझ])", r"ं\1"),
]
for pattern, repl in replacements:
text = re.sub(pattern, repl, text)
return text
def join_virama_words(text: str) -> str:
"""Join Devanagari words where the previous word ends with virāma (्)
and the next word starts with a consonant.
In standard Devanagari verse rendering, a word ending in a halant
(virāma) connects to the following consonant-initial word without a space.
e.g., श्रीर् विजयो → श्रीर्विजयो
But when followed by a vowel, the space is preserved:
e.g., किम् अकुर्वत → किम् अकुर्वत (stays separate)
"""
# Devanagari consonants: क (U+0915) through ह (U+0939)
return re.sub(r"्\s+([क-ह])", r"्\1", text)
def iast_to_devanagari(iast_text: str) -> str:
"""Convert IAST transliteration to Devanagari script."""
lines = iast_text.split('\n')
lines = iast_text.split("\n")
result_lines = []
for line in lines:
# Remove hyphens between words for Devanagari
clean = line.replace('-', '')
clean = line.replace("-", "")
# Normalize ṁ (U+1E41) to ṃ (U+1E43) for proper anusvāra
clean = clean.replace('ṁ', 'ṃ')
clean = clean.replace("ṁ", "ṃ")
deva = transliterate(clean, sanscript.IAST, sanscript.DEVANAGARI)
# Normalize class nasals to anusvāra
deva = normalize_devanagari_anusvara(deva)
# Join words where previous word ends with virāma (्)
deva = join_virama_words(deva)
result_lines.append(deva)
return '\n'.join(result_lines)
return "\n".join(result_lines)
def parse_synonyms(text: str) -> list:
@@ -99,16 +126,16 @@ def parse_synonyms(text: str) -> list:
"""
result = []
# Split by semicolons
entries = text.split(';')
entries = text.split(";")
for entry in entries:
entry = entry.strip()
if not entry:
continue
# Split on em-dash (—) or double-hyphen
if '—' in entry:
parts = entry.split('—', 1)
elif '--' in entry:
parts = entry.split('--', 1)
if "—" in entry:
parts = entry.split("—", 1)
elif "--" in entry:
parts = entry.split("--", 1)
else:
# Sometimes just "word-meaning" with single hyphen at boundary
# Skip entries without clear separator
@@ -118,12 +145,9 @@ def parse_synonyms(text: str) -> list:
word = parts[0].strip()
meaning = parts[1].strip()
# Clean word: remove trailing/leading punctuation
word = word.strip('., ')
word = word.strip("., ")
if word and meaning:
result.append({
"word": word,
"meanings": [meaning]
})
result.append({"word": word, "meanings": [meaning]})
return result
@@ -141,7 +165,7 @@ def pair_padas(padas: list) -> str:
the first pāda stands alone (usually a speaker attribution like 'arjuna uvāca').
"""
if not padas:
return ''
return ""
lines = []
i = 0
@@ -153,40 +177,40 @@ def pair_padas(padas: list) -> str:
# Pair remaining pādas
while i < len(padas) - 1:
lines.append(padas[i] + ' ' + padas[i + 1])
lines.append(padas[i] + " " + padas[i + 1])
i += 2
# Handle any trailing single pāda
if i < len(padas):
lines.append(padas[i])
return '\n'.join(lines)
return "\n".join(lines)
def get_css_class(tag: Tag) -> str:
"""Get the primary CSS class of a tag."""
classes = tag.get('class', [])
return classes[0] if classes else ''
classes = tag.get("class", [])
return classes[0] if classes else ""
def _finalize_verse(verse: dict):
"""Build transliteration and Devanagari from pādas. Remove internal fields."""
padas = verse.get('_padas', [])
padas = verse.get("_padas", [])
# Transliteration: one pāda per line (original structure)
verse['transliteration'] = '\n'.join(padas)
verse["transliteration"] = "\n".join(padas)
# Devanagari: pair pādas into 2-line verse structure
if padas:
verse['devanagari'] = iast_to_devanagari(pair_padas(padas))
del verse['_padas']
verse["devanagari"] = iast_to_devanagari(pair_padas(padas))
del verse["_padas"]
def extract_verses(soup: BeautifulSoup, max_chapter=1, max_verse=5):
"""Extract verses from parsed HTML using state machine."""
paragraphs = soup.find_all('p')
paragraphs = soup.find_all("p")
verses = []
current_verse = None
state = 'IDLE' # IDLE, TRANSLITERATION, SYNONYMS, TRANSLATION, PURPORT
state = "IDLE" # IDLE, TRANSLITERATION, SYNONYMS, TRANSLATION, PURPORT
current_chapter = 0
chapter_title = ""
@@ -195,24 +219,25 @@ def extract_verses(soup: BeautifulSoup, max_chapter=1, max_verse=5):
text = p.get_text().strip()
# Chapter title
if cls == 'p11':
if cls == "p11":
decoded = decode_text(p)
match = re.match(r'^(\d+)\.\s+(.+)$', decoded, re.DOTALL)
match = re.match(r"^(\d+)\.\s+(.+)$", decoded, re.DOTALL)
if match:
current_chapter = int(match.group(1))
chapter_title = match.group(2).replace('\n', ' ').strip()
chapter_title = match.group(2).replace("\n", " ").strip()
if current_chapter > max_chapter:
break
# Verse reference (e.g., "Bg 1.1")
elif cls == 'p25':
elif cls == "p25":
# Finalize previous verse
if current_verse and state != 'IDLE':
if current_verse and state != "IDLE":
_finalize_verse(current_verse)
verses.append(current_verse)
current_verse = None
state = 'IDLE'
state = "IDLE"
match = re.search(r'Bg\s+(\d+)\.(\d+)', text)
match = re.search(r"Bg\s+(\d+)\.(\d+)", text)
if match:
ch = int(match.group(1))
vs = int(match.group(2))
@@ -221,66 +246,77 @@ def extract_verses(soup: BeautifulSoup, max_chapter=1, max_verse=5):
break
current_verse = {
'chapter': ch,
'verse': vs,
'reference': decode_scagoudy(text.strip()),
'devanagari': '',
'transliteration': '',
'_padas': [], # raw pādas, paired into lines later
'synonyms': [],
'translation': '',
'purport': []
"book_title": "Bhagavad-gītā As It Is",
"chapter": ch,
"verse": vs,
"reference": decode_scagoudy(text.strip()),
"devanagari": "",
"transliteration": "",
"_padas": [], # raw pādas, paired into lines later
"synonyms": [],
"translation": "",
"purport": [],
}
state = 'VERSE_START'
state = "VERSE_START"
# TEXT header (e.g., "TEXT 1")
elif cls == 'p26' and current_verse:
elif cls == "p26" and current_verse:
pass # Just confirms we're in a verse
# TEXT label
elif cls == 'p27' and current_verse:
elif cls == "p27" and current_verse:
pass # Skip
# Devanagari in Indevr font — SKIP (will derive from transliteration)
elif cls == 'p28' and current_verse:
state = 'TRANSLITERATION'
elif cls == "p28" and current_verse:
state = "TRANSLITERATION"
# Transliteration lines (individual pādas)
elif cls == 'p29' and current_verse and state in ('TRANSLITERATION', 'VERSE_START'):
state = 'TRANSLITERATION'
# p29 is the primary class; p35 is an alternate class used for some pādas
elif (
cls in ("p29", "p35")
and current_verse
and state in ("TRANSLITERATION", "VERSE_START")
):
state = "TRANSLITERATION"
line = decode_text(p)
current_verse['_padas'].append(line)
current_verse["_padas"].append(line)
# Synonyms
elif cls == 'p30' and current_verse:
state = 'SYNONYMS'
current_verse['synonyms'] = parse_synonym_html(p)
elif cls == "p30" and current_verse:
state = "SYNONYMS"
current_verse["synonyms"] = parse_synonym_html(p)
# Section headers (TRANSLATION / PURPORT)
elif cls in ('p31', 'p33') and current_verse:
# p31 and p33 are primary; p38 is an alternate class used in some verses
elif cls in ("p31", "p33", "p38") and current_verse:
header_text = text.strip().upper()
if 'TRANSLATION' in header_text:
state = 'TRANSLATION'
elif 'PURPORT' in header_text:
state = 'PURPORT'
if "TRANSLATION" in header_text:
state = "TRANSLATION"
elif "PURPORT" in header_text:
state = "PURPORT"
# Translation text
elif cls == 'p32' and current_verse and state == 'TRANSLATION':
current_verse['translation'] = decode_html_content(p)
state = 'POST_TRANSLATION'
elif cls == "p32" and current_verse and state == "TRANSLATION":
current_verse["translation"] = decode_html_content(p)
state = "POST_TRANSLATION"
# Purport paragraphs (exclude Collier markers)
elif state == 'PURPORT' and current_verse and cls in ('p6', 'p12', 'p13', 'p4', 'p3', 'p36', 'p19', 'p20'):
elif (
state == "PURPORT"
and current_verse
and cls in ("p6", "p12", "p13", "p4", "p3", "p36", "p19", "p20")
):
content = decode_html_content(p)
if content and 'Collier' not in text:
current_verse['purport'].append(content)
if content and "Collier" not in text:
current_verse["purport"].append(content)
# Collier end marker (can be p24 or p34)
elif cls in ('p24', 'p34') and current_verse:
elif cls in ("p24", "p34") and current_verse:
_finalize_verse(current_verse)
verses.append(current_verse)
current_verse = None
state = 'IDLE'
state = "IDLE"
# Don't forget the last verse if no Collier marker
if current_verse:
@@ -293,25 +329,35 @@ def extract_verses(soup: BeautifulSoup, max_chapter=1, max_verse=5):
def write_json(data, filepath):
"""Write JSON with proper Unicode and formatting."""
os.makedirs(os.path.dirname(filepath), exist_ok=True)
with open(filepath, 'w', encoding='utf-8') as f:
with open(filepath, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False, indent=2)
print(f" Written: {filepath}")
def main():
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
html_path = os.path.join(base_dir, 'public', 'html', 'bg.html')
output_dir = os.path.join(base_dir, 'app', 'lib', 'books', 'bg')
html_path = os.path.join(base_dir, "public", "html", "bg.html")
output_dir = os.path.join(base_dir, "app", "lib", "books", "bg")
print(f"Reading {html_path}...")
with open(html_path, 'r', encoding='utf-8') as f:
soup = BeautifulSoup(f, 'html.parser')
with open(html_path, "r", encoding="utf-8") as f:
soup = BeautifulSoup(f, "html.parser")
print("Extracting verses (Chapter 1, all verses)...")
verses, chapter_num, chapter_title = extract_verses(soup, max_chapter=1, max_verse=999)
print("Extracting verses (all 18 chapters)...")
verses, chapter_num, chapter_title = extract_verses(
soup, max_chapter=18, max_verse=999
)
print(f"\nExtracted {len(verses)} verses")
# Group verses by chapter to build intro.json
chapters_info = {}
for verse in verses:
ch = verse["chapter"]
if ch not in chapters_info:
chapters_info[ch] = 0
chapters_info[ch] += 1
# Write intro.json
intro = {
"book": "bg",
@@ -319,32 +365,29 @@ def main():
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
"totalChapters": 18,
"chapters": [
{
"number": 1,
"title": chapter_title or "Observing the Armies on the Battlefield of Kurukṣetra",
"totalVerses": 46
}
]
{"number": ch, "totalVerses": count}
for ch, count in sorted(chapters_info.items())
],
}
write_json(intro, os.path.join(output_dir, 'intro.json'))
write_json(intro, os.path.join(output_dir, "intro.json"))
# Write verse JSON files
for verse in verses:
ch = str(verse['chapter']).zfill(2)
vs = str(verse['verse']).zfill(2)
filepath = os.path.join(output_dir, 'chapters', ch, 'verses', f'{vs}.json')
ch = str(verse["chapter"]).zfill(2)
vs = str(verse["verse"]).zfill(2)
filepath = os.path.join(output_dir, "chapters", ch, "verses", f"{vs}.json")
write_json(verse, filepath)
# Print summary for verification
print("\n--- Verification ---")
for verse in verses:
print(f"\n=== {verse['reference']} ===")
print(f"Devanagari: {verse['devanagari'][:80]}...")
print(f"IAST: {verse['transliteration'][:80]}...")
print(f"Synonyms: {len(verse['synonyms'])} entries")
print(f"Translation: {verse['translation'][:80]}...")
print(f"Purport: {len(verse['purport'])} paragraphs")
for ch in sorted(chapters_info.keys()):
ch_verses = [v for v in verses if v["chapter"] == ch]
print(f"\nChapter {ch}: {len(ch_verses)} verses")
first = ch_verses[0]
last = ch_verses[-1]
print(f" First: {first['reference']}")
print(f" Last: {last['reference']}")
if __name__ == '__main__':
if __name__ == "__main__":
main()