Initial commit

scripts/bg_extract.py

@@ -0,0 +1,350 @@
+#!/usr/bin/env python3
+"""Extract Bhagavad Gita verses from HTML into structured JSON files."""
+
+import json
+import os
+import re
+import sys
+from bs4 import BeautifulSoup, Tag
+from indic_transliteration import sanscript
+from indic_transliteration.sanscript import transliterate
+
+# --- ScaGoudy → IAST Unicode mapping ---
+SCAGOUDY_TO_IAST = {
+    'ä': 'ā', 'Ä': 'Ā',
+    'å': 'ṛ', 'Å': 'Ṛ',
+    'ñ': 'ṣ', 'Ñ': 'Ṣ',
+    'ö': 'ṭ', 'Ö': 'Ṭ',
+    'ë': 'ṇ', 'Ë': 'Ṇ',
+    'ò': 'ḍ', 'Ò': 'Ḍ',
+    'ç': 'ś', 'Ç': 'Ś',
+    'ì': 'ṅ',
+    'ï': 'ñ',
+    'ü': 'ū', 'Ü': 'Ū',
+    'é': 'ī', 'É': 'Ī',
+    'à': 'ṁ',
+    'ù': 'ḥ',
+}
+
+
+def decode_scagoudy(text: str) -> str:
+    """Convert ScaGoudy-encoded text to proper IAST Unicode."""
+    for src, dst in SCAGOUDY_TO_IAST.items():
+        text = text.replace(src, dst)
+    return text
+
+
+def decode_html_content(tag: Tag) -> str:
+    """Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
+    # Get inner HTML as string
+    inner = ''.join(str(child) for child in tag.children)
+    # Decode ScaGoudy characters
+    inner = decode_scagoudy(inner)
+    # Remove span tags with CSS classes (artifacts from RTF conversion)
+    inner = re.sub(r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r'\1', inner)
+    # Clean up whitespace
+    inner = re.sub(r'\s+', ' ', inner).strip()
+    return inner
+
+
+def decode_text(tag: Tag) -> str:
+    """Extract plain text from a tag and decode ScaGoudy."""
+    text = tag.get_text()
+    return decode_scagoudy(text).strip()
+
+
+def normalize_devanagari_anusvara(text: str) -> str:
+    """Replace explicit class nasals before same-class consonants with anusvāra.
+
+    In modern Devanagari convention, anusvāra (ं) is preferred over explicit
+    nasal consonants (ङ्, ञ्, ण्, न्, म्) when they appear before consonants
+    of their own class.
+
+    e.g., सङ्गम्य → संगम्य, सञ्जय → संजय
+    """
+    # Only normalize ङ् and ञ् — these are almost always written with anusvāra.
+    # Keep ण्ड, न्त, म्ब etc. as explicit conjuncts (standard spelling).
+    replacements = [
+        # ङ् before velars (क, ख, ग, घ)
+        (r'ङ्([कखगघ])', r'ं\1'),
+        # ञ् before palatals (च, छ, ज, झ)
+        (r'ञ्([चछजझ])', r'ं\1'),
+    ]
+    for pattern, repl in replacements:
+        text = re.sub(pattern, repl, text)
+    return text
+
+
+def iast_to_devanagari(iast_text: str) -> str:
+    """Convert IAST transliteration to Devanagari script."""
+    lines = iast_text.split('\n')
+    result_lines = []
+    for line in lines:
+        # Remove hyphens between words for Devanagari
+        clean = line.replace('-', '')
+        # Normalize ṁ (U+1E41) to ṃ (U+1E43) for proper anusvāra
+        clean = clean.replace('ṁ', 'ṃ')
+        deva = transliterate(clean, sanscript.IAST, sanscript.DEVANAGARI)
+        # Normalize class nasals to anusvāra
+        deva = normalize_devanagari_anusvara(deva)
+        result_lines.append(deva)
+    return '\n'.join(result_lines)
+
+
+def parse_synonyms(text: str) -> list:
+    """Parse synonym text into structured array.
+
+    Input format: "word—meaning; word—meaning; word—meaning"
+    Some entries use em-dash (—) and some use double-hyphen.
+    """
+    result = []
+    # Split by semicolons
+    entries = text.split(';')
+    for entry in entries:
+        entry = entry.strip()
+        if not entry:
+            continue
+        # Split on em-dash (—) or double-hyphen
+        if '—' in entry:
+            parts = entry.split('—', 1)
+        elif '--' in entry:
+            parts = entry.split('--', 1)
+        else:
+            # Sometimes just "word-meaning" with single hyphen at boundary
+            # Skip entries without clear separator
+            continue
+
+        if len(parts) == 2:
+            word = parts[0].strip()
+            meaning = parts[1].strip()
+            # Clean word: remove trailing/leading punctuation
+            word = word.strip('., ')
+            if word and meaning:
+                result.append({
+                    "word": word,
+                    "meanings": [meaning]
+                })
+
+    return result
+
+
+def parse_synonym_html(tag: Tag) -> list:
+    """Parse synonyms from HTML tag, preserving italic markup for words."""
+    text = decode_text(tag)
+    return parse_synonyms(text)
+
+
+def pair_padas(padas: list) -> str:
+    """Pair pādas into verse lines (2 pādas per line).
+
+    A śloka has 4 pādas displayed as 2 lines. If there's an odd number,
+    the first pāda stands alone (usually a speaker attribution like 'arjuna uvāca').
+    """
+    if not padas:
+        return ''
+
+    lines = []
+    i = 0
+
+    # If odd number of pādas, first one is standalone (speaker attribution)
+    if len(padas) % 2 == 1:
+        lines.append(padas[0])
+        i = 1
+
+    # Pair remaining pādas
+    while i < len(padas) - 1:
+        lines.append(padas[i] + ' ' + padas[i + 1])
+        i += 2
+
+    # Handle any trailing single pāda
+    if i < len(padas):
+        lines.append(padas[i])
+
+    return '\n'.join(lines)
+
+
+def get_css_class(tag: Tag) -> str:
+    """Get the primary CSS class of a tag."""
+    classes = tag.get('class', [])
+    return classes[0] if classes else ''
+
+
+def _finalize_verse(verse: dict):
+    """Build transliteration and Devanagari from pādas. Remove internal fields."""
+    padas = verse.get('_padas', [])
+    # Transliteration: one pāda per line (original structure)
+    verse['transliteration'] = '\n'.join(padas)
+    # Devanagari: pair pādas into 2-line verse structure
+    if padas:
+        verse['devanagari'] = iast_to_devanagari(pair_padas(padas))
+    del verse['_padas']
+
+
+def extract_verses(soup: BeautifulSoup, max_chapter=1, max_verse=5):
+    """Extract verses from parsed HTML using state machine."""
+    paragraphs = soup.find_all('p')
+
+    verses = []
+    current_verse = None
+    state = 'IDLE'  # IDLE, TRANSLITERATION, SYNONYMS, TRANSLATION, PURPORT
+    current_chapter = 0
+    chapter_title = ""
+
+    for p in paragraphs:
+        cls = get_css_class(p)
+        text = p.get_text().strip()
+
+        # Chapter title
+        if cls == 'p11':
+            decoded = decode_text(p)
+            match = re.match(r'^(\d+)\.\s+(.+)$', decoded, re.DOTALL)
+            if match:
+                current_chapter = int(match.group(1))
+                chapter_title = match.group(2).replace('\n', ' ').strip()
+                if current_chapter > max_chapter:
+                    break
+
+        # Verse reference (e.g., "Bg 1.1")
+        elif cls == 'p25':
+            # Finalize previous verse
+            if current_verse and state != 'IDLE':
+                verses.append(current_verse)
+                current_verse = None
+                state = 'IDLE'
+
+            match = re.search(r'Bg\s+(\d+)\.(\d+)', text)
+            if match:
+                ch = int(match.group(1))
+                vs = int(match.group(2))
+
+                if ch > max_chapter or (ch == max_chapter and vs > max_verse):
+                    break
+
+                current_verse = {
+                    'chapter': ch,
+                    'verse': vs,
+                    'reference': decode_scagoudy(text.strip()),
+                    'devanagari': '',
+                    'transliteration': '',
+                    '_padas': [],  # raw pādas, paired into lines later
+                    'synonyms': [],
+                    'translation': '',
+                    'purport': []
+                }
+                state = 'VERSE_START'
+
+        # TEXT header (e.g., "TEXT 1")
+        elif cls == 'p26' and current_verse:
+            pass  # Just confirms we're in a verse
+
+        # TEXT label
+        elif cls == 'p27' and current_verse:
+            pass  # Skip
+
+        # Devanagari in Indevr font — SKIP (will derive from transliteration)
+        elif cls == 'p28' and current_verse:
+            state = 'TRANSLITERATION'
+
+        # Transliteration lines (individual pādas)
+        elif cls == 'p29' and current_verse and state in ('TRANSLITERATION', 'VERSE_START'):
+            state = 'TRANSLITERATION'
+            line = decode_text(p)
+            current_verse['_padas'].append(line)
+
+        # Synonyms
+        elif cls == 'p30' and current_verse:
+            state = 'SYNONYMS'
+            current_verse['synonyms'] = parse_synonym_html(p)
+
+        # Section headers (TRANSLATION / PURPORT)
+        elif cls in ('p31', 'p33') and current_verse:
+            header_text = text.strip().upper()
+            if 'TRANSLATION' in header_text:
+                state = 'TRANSLATION'
+            elif 'PURPORT' in header_text:
+                state = 'PURPORT'
+
+        # Translation text
+        elif cls == 'p32' and current_verse and state == 'TRANSLATION':
+            current_verse['translation'] = decode_html_content(p)
+            state = 'POST_TRANSLATION'
+
+        # Purport paragraphs (exclude Collier markers)
+        elif state == 'PURPORT' and current_verse and cls in ('p6', 'p12', 'p13', 'p4', 'p3', 'p36', 'p19', 'p20'):
+            content = decode_html_content(p)
+            if content and 'Collier' not in text:
+                current_verse['purport'].append(content)
+
+        # Collier end marker (can be p24 or p34)
+        elif cls in ('p24', 'p34') and current_verse:
+            _finalize_verse(current_verse)
+            verses.append(current_verse)
+            current_verse = None
+            state = 'IDLE'
+
+    # Don't forget the last verse if no Collier marker
+    if current_verse:
+        _finalize_verse(current_verse)
+        verses.append(current_verse)
+
+    return verses, current_chapter, chapter_title
+
+
+def write_json(data, filepath):
+    """Write JSON with proper Unicode and formatting."""
+    os.makedirs(os.path.dirname(filepath), exist_ok=True)
+    with open(filepath, 'w', encoding='utf-8') as f:
+        json.dump(data, f, ensure_ascii=False, indent=2)
+    print(f"  Written: {filepath}")
+
+
+def main():
+    base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    html_path = os.path.join(base_dir, 'public', 'html', 'bg.html')
+    output_dir = os.path.join(base_dir, 'app', 'lib', 'books', 'bg')
+
+    print(f"Reading {html_path}...")
+    with open(html_path, 'r', encoding='utf-8') as f:
+        soup = BeautifulSoup(f, 'html.parser')
+
+    print("Extracting verses (Chapter 1, all verses)...")
+    verses, chapter_num, chapter_title = extract_verses(soup, max_chapter=1, max_verse=999)
+
+    print(f"\nExtracted {len(verses)} verses")
+
+    # Write intro.json
+    intro = {
+        "book": "bg",
+        "title": "Bhagavad-gītā As It Is",
+        "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
+        "totalChapters": 18,
+        "chapters": [
+            {
+                "number": 1,
+                "title": chapter_title or "Observing the Armies on the Battlefield of Kurukṣetra",
+                "totalVerses": 46
+            }
+        ]
+    }
+    write_json(intro, os.path.join(output_dir, 'intro.json'))
+
+    # Write verse JSON files
+    for verse in verses:
+        ch = str(verse['chapter']).zfill(2)
+        vs = str(verse['verse']).zfill(2)
+        filepath = os.path.join(output_dir, 'chapters', ch, 'verses', f'{vs}.json')
+        write_json(verse, filepath)
+
+    # Print summary for verification
+    print("\n--- Verification ---")
+    for verse in verses:
+        print(f"\n=== {verse['reference']} ===")
+        print(f"Devanagari: {verse['devanagari'][:80]}...")
+        print(f"IAST: {verse['transliteration'][:80]}...")
+        print(f"Synonyms: {len(verse['synonyms'])} entries")
+        print(f"Translation: {verse['translation'][:80]}...")
+        print(f"Purport: {len(verse['purport'])} paragraphs")
+
+
+if __name__ == '__main__':
+    main()

scripts/inject-fonts.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+"""Inject @font-face declarations into converted HTML files so browsers can render custom fonts."""
+
+import os
+import glob
+
+HTML_DIR = os.path.join(os.path.dirname(__file__), "..", "public", "html")
+
+# Map: TTF filename -> (family name, weight, style)
+FONT_MAP = {
+    "balaram_.ttf": ("Balaram", "normal", "normal"),
+    "bab_____.ttf": ("Balaram", "bold", "normal"),
+    "bai_____.ttf": ("Balaram", "normal", "italic"),
+    "babi____.ttf": ("Balaram", "bold", "italic"),
+    "SCAGRG__.TTF": ("ScaGoudy", "normal", "normal"),
+    "SCAGB___.TTF": ("ScaGoudy", "bold", "normal"),
+    "SCAGI___.TTF": ("ScaGoudy", "normal", "italic"),
+    "SCAGBI__.TTF": ("ScaGoudy", "bold", "italic"),
+    "SCATRG__.TTF": ("ScaTimes", "normal", "normal"),
+    "SCATB___.TTF": ("ScaTimes", "bold", "normal"),
+    "SCATI___.TTF": ("ScaTimes", "normal", "italic"),
+    "SCAPRG__.TTF": ("ScaPalatino", "normal", "normal"),
+    "SCAPB___.TTF": ("ScaPalatino", "bold", "normal"),
+    "SCAPI___.TTF": ("ScaPalatino", "normal", "italic"),
+    "SCAHRG__.TTF": ("ScaHelvetica", "normal", "normal"),
+    "SCAHB___.TTF": ("ScaHelvetica", "bold", "normal"),
+    "SCAORG__.TTF": ("ScaOptima", "normal", "normal"),
+    "SCAOB___.TTF": ("ScaOptima", "bold", "normal"),
+    "SCAOI___.TTF": ("ScaOptima", "normal", "italic"),
+    "SCAKRG__.TTF": ("ScaKorinna", "normal", "normal"),
+    "SCAKB___.TTF": ("ScaKorinna", "bold", "normal"),
+    "SCAKI___.TTF": ("ScaKorinna", "normal", "italic"),
+    "SCABRG__.TTF": ("ScaBenguit", "normal", "normal"),
+    "SCABB___.TTF": ("ScaBenguit", "bold", "normal"),
+    "SCACRG__.TTF": ("ScaCheltenham", "normal", "normal"),
+    "SCACB___.TTF": ("ScaCheltenham", "bold", "normal"),
+    "SCACI___.TTF": ("ScaCheltenham", "normal", "italic"),
+    "SCASRG__.TTF": ("ScaSabon", "normal", "normal"),
+    "SCASB___.TTF": ("ScaSabon", "bold", "normal"),
+    "SCASI___.TTF": ("ScaSabon", "normal", "italic"),
+    "SCAFQRG_.TTF": ("ScaFrizQuadrata", "normal", "normal"),
+    "SCAFQB__.TTF": ("ScaFrizQuadrata", "bold", "normal"),
+    "S_TMS___.TTF": ("Sanskrit-Times", "normal", "normal"),
+    "S_TMS_B_.TTF": ("Sanskrit-Times", "bold", "normal"),
+    "S_TMS__I.TTF": ("Sanskrit-Times", "normal", "italic"),
+    "S_TMS_BI.TTF": ("Sanskrit-Times", "bold", "italic"),
+    "S_HLV___.TTF": ("Sanskrit-Helvetica", "normal", "normal"),
+    "S_HLV_B_.TTF": ("Sanskrit-Helvetica", "bold", "normal"),
+    "S_HLV__O.TTF": ("Sanskrit-Helvetica", "normal", "italic"),
+    "S_HLV_BO.TTF": ("Sanskrit-Helvetica", "bold", "italic"),
+    "S_NCALN.TTF": ("Sanskrit-New Caledonia", "normal", "normal"),
+    "S_NCALB.TTF": ("Sanskrit-New Caledonia", "bold", "normal"),
+    "S_NCALI.TTF": ("Sanskrit-New Caledonia", "normal", "italic"),
+    "S_NCALBI.TTF": ("Sanskrit-New Caledonia", "bold", "italic"),
+    "RMDEVA.TTF": ("RM Devanagari", "normal", "normal"),
+    "BELLCN__.TTF": ("BellCentennial", "normal", "normal"),
+    "inbenr11.ttf": ("Inbenr", "normal", "normal"),
+    "inbenb11.ttf": ("Inbenb", "normal", "normal"),
+    "inbeni11.ttf": ("Inbeni", "normal", "normal"),
+    "inbeno11.ttf": ("Inbeno", "normal", "normal"),
+    "indevr20.ttf": ("Indevr", "normal", "normal"),
+}
+
+def build_font_face_css():
+    rules = []
+    for filename, (family, weight, style) in sorted(FONT_MAP.items()):
+        rules.append(
+            f"@font-face {{\n"
+            f"  font-family: '{family}';\n"
+            f"  src: url('/fonts/{filename}') format('truetype');\n"
+            f"  font-weight: {weight};\n"
+            f"  font-style: {style};\n"
+            f"  font-display: swap;\n"
+            f"}}"
+        )
+    return "\n".join(rules)
+
+def inject_into_html(html_path, font_css):
+    with open(html_path, "r", encoding="utf-8", errors="replace") as f:
+        content = f.read()
+
+    # Remove old injection if present
+    if "/* RTF-VIEWER-FONTS */" in content:
+        import re
+        content = re.sub(
+            r'<style type="text/css">\s*/\* RTF-VIEWER-FONTS \*/.*?</style>\s*',
+            '',
+            content,
+            flags=re.DOTALL
+        )
+
+    style_tag = f'<style type="text/css">\n/* RTF-VIEWER-FONTS */\n{font_css}\n</style>'
+
+    if "</head>" in content:
+        content = content.replace("</head>", f"{style_tag}\n</head>")
+    elif "<body" in content:
+        content = content.replace("<body", f"{style_tag}\n<body")
+    else:
+        content = style_tag + "\n" + content
+
+    with open(html_path, "w", encoding="utf-8") as f:
+        f.write(content)
+
+    print(f"  Injected fonts into {os.path.basename(html_path)}")
+
+if __name__ == "__main__":
+    font_css = build_font_face_css()
+    html_files = glob.glob(os.path.join(HTML_DIR, "*.html"))
+
+    if not html_files:
+        print("No HTML files found in", HTML_DIR)
+        exit(1)
+
+    print(f"Injecting @font-face rules into {len(html_files)} HTML files...")
+    for html_path in sorted(html_files):
+        inject_into_html(html_path, font_css)
+    print("Done!")