feat: extract CC
This commit is contained in:
gosangam
@@ -0,0 +1,787 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract Caitanya-caritāmṛta verses from HTML into structured JSON files.
|
||||
|
||||
Structure: Līlā (Ādi/Madhya/Antya) → Chapter → Verse
|
||||
Output goes to app/lib/books/cc/
|
||||
|
||||
The CC HTML uses a different class mapping than BG/SB but shares the same
|
||||
ScaGoudy font encoding and the same verse structure:
|
||||
Bengali (skip) → Transliteration pādas → Synonyms → Translation → Purport
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
from indic_transliteration import sanscript
|
||||
from indic_transliteration.sanscript import transliterate
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ScaGoudy → IAST Unicode mapping (shared with bg/sb scripts)
|
||||
# ---------------------------------------------------------------------------
|
||||
SCAGOUDY_TO_IAST = {
|
||||
"ä": "ā",
|
||||
"Ä": "Ā",
|
||||
"å": "ṛ",
|
||||
"Å": "Ṛ",
|
||||
"ñ": "ṣ",
|
||||
"Ñ": "Ṣ",
|
||||
"ö": "ṭ",
|
||||
"Ö": "Ṭ",
|
||||
"ë": "ṇ",
|
||||
"Ë": "Ṇ",
|
||||
"ò": "ḍ",
|
||||
"Ò": "Ḍ",
|
||||
"ç": "ś",
|
||||
"Ç": "Ś",
|
||||
"ì": "ṅ",
|
||||
"ï": "ñ",
|
||||
"ü": "ū",
|
||||
"Ü": "Ū",
|
||||
"é": "ī",
|
||||
"É": "Ī",
|
||||
"à": "ṁ",
|
||||
"ù": "ḥ",
|
||||
}
|
||||
|
||||
|
||||
def decode_scagoudy(text: str) -> str:
|
||||
"""Convert ScaGoudy-encoded text to proper IAST Unicode."""
|
||||
for src, dst in SCAGOUDY_TO_IAST.items():
|
||||
text = text.replace(src, dst)
|
||||
return text
|
||||
|
||||
|
||||
def decode_html_content(tag: Tag) -> str:
|
||||
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
|
||||
inner = "".join(str(child) for child in tag.children)
|
||||
inner = decode_scagoudy(inner)
|
||||
inner = re.sub(r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner)
|
||||
inner = re.sub(r"\s+", " ", inner).strip()
|
||||
return inner
|
||||
|
||||
|
||||
def decode_text(tag: Tag) -> str:
|
||||
"""Extract plain text from a tag and decode ScaGoudy."""
|
||||
text = tag.get_text()
|
||||
return decode_scagoudy(text).strip()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Devanagari helpers (same as bg/sb scripts)
|
||||
# ---------------------------------------------------------------------------
|
||||
def normalize_devanagari_anusvara(text: str) -> str:
|
||||
"""Replace explicit class nasals before same-class consonants with anusvāra."""
|
||||
replacements = [
|
||||
(r"ङ्([कखगघ])", r"ं\1"),
|
||||
(r"ञ्([चछजझ])", r"ं\1"),
|
||||
]
|
||||
for pattern, repl in replacements:
|
||||
text = re.sub(pattern, repl, text)
|
||||
return text
|
||||
|
||||
|
||||
def join_virama_words(text: str) -> str:
|
||||
"""Join Devanagari words where the previous word ends with virāma.
|
||||
|
||||
Handles both virāma+consonant and virāma+vowel cases.
|
||||
Uses [ \\t]+ instead of \\s+ to avoid joining across newlines.
|
||||
"""
|
||||
text = re.sub(r"्[ \t]+([क-ह])", r"्\1", text)
|
||||
|
||||
_VOWEL_TO_MATRA = {
|
||||
"अ": "",
|
||||
"आ": "ा",
|
||||
"इ": "ि",
|
||||
"ई": "ी",
|
||||
"उ": "ु",
|
||||
"ऊ": "ू",
|
||||
"ऋ": "ृ",
|
||||
"ॠ": "ॄ",
|
||||
"ऌ": "ॢ",
|
||||
"ॡ": "ॣ",
|
||||
"ए": "े",
|
||||
"ऐ": "ै",
|
||||
"ओ": "ो",
|
||||
"औ": "ौ",
|
||||
}
|
||||
|
||||
def _virama_vowel_replace(m):
|
||||
vowel = m.group(1)
|
||||
return _VOWEL_TO_MATRA.get(vowel, vowel)
|
||||
|
||||
text = re.sub(r"्[ \t]+([अ-औॠॡ])", _virama_vowel_replace, text)
|
||||
return text
|
||||
|
||||
|
||||
def iast_to_devanagari(iast_text: str) -> str:
|
||||
"""Convert IAST transliteration to Devanagari script."""
|
||||
lines = iast_text.split("\n")
|
||||
result_lines = []
|
||||
for line in lines:
|
||||
clean = line.replace("-", "")
|
||||
clean = clean.replace("ṁ", "ṃ")
|
||||
deva = transliterate(clean, sanscript.IAST, sanscript.DEVANAGARI)
|
||||
deva = normalize_devanagari_anusvara(deva)
|
||||
deva = join_virama_words(deva)
|
||||
result_lines.append(deva)
|
||||
return "\n".join(result_lines)
|
||||
|
||||
|
||||
def iast_to_bengali(iast_text: str) -> str:
|
||||
"""Convert IAST transliteration to Bengali script."""
|
||||
lines = iast_text.split("\n")
|
||||
result_lines = []
|
||||
for line in lines:
|
||||
clean = line.replace("-", "")
|
||||
clean = clean.replace("ṁ", "ṃ")
|
||||
beng = transliterate(clean, sanscript.IAST, sanscript.BENGALI)
|
||||
# Normalize anusvāra for Bengali (same classes as Devanagari)
|
||||
beng = re.sub(r"ঙ্([কখগঘ])", r"ং\1", beng)
|
||||
beng = re.sub(r"ঞ্([চছজঝ])", r"ং\1", beng)
|
||||
# Join virama words for Bengali (hasanta = ্)
|
||||
beng = re.sub(r"্[ \t]+([ক-হ])", r"্\1", beng)
|
||||
_BN_VOWEL_TO_MATRA = {
|
||||
"অ": "",
|
||||
"আ": "া",
|
||||
"ই": "ি",
|
||||
"ঈ": "ী",
|
||||
"উ": "ু",
|
||||
"ঊ": "ূ",
|
||||
"ঋ": "ৃ",
|
||||
"ৠ": "ৄ",
|
||||
"ঌ": "ৢ",
|
||||
"ৡ": "ৣ",
|
||||
"এ": "ে",
|
||||
"ঐ": "ৈ",
|
||||
"ও": "ো",
|
||||
"ঔ": "ৌ",
|
||||
}
|
||||
|
||||
def _bn_virama_vowel_replace(m):
|
||||
vowel = m.group(1)
|
||||
return _BN_VOWEL_TO_MATRA.get(vowel, vowel)
|
||||
|
||||
beng = re.sub(r"্[ \t]+([অ-ঔৠৡ])", _bn_virama_vowel_replace, beng)
|
||||
result_lines.append(beng)
|
||||
return "\n".join(result_lines)
|
||||
|
||||
|
||||
def pair_padas(padas: list) -> str:
|
||||
"""Pair pādas into verse lines (2 pādas per line)."""
|
||||
if not padas:
|
||||
return ""
|
||||
lines = []
|
||||
i = 0
|
||||
if len(padas) % 2 == 1:
|
||||
lines.append(padas[0])
|
||||
i = 1
|
||||
while i < len(padas) - 1:
|
||||
lines.append(padas[i] + " " + padas[i + 1])
|
||||
i += 2
|
||||
if i < len(padas):
|
||||
lines.append(padas[i])
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Synonym parser
|
||||
# ---------------------------------------------------------------------------
|
||||
def parse_synonyms(text: str) -> list:
|
||||
"""Parse synonym text into structured array."""
|
||||
result = []
|
||||
entries = text.split(";")
|
||||
for entry in entries:
|
||||
entry = entry.strip()
|
||||
if not entry:
|
||||
continue
|
||||
if "—" in entry:
|
||||
parts = entry.split("—", 1)
|
||||
elif "--" in entry:
|
||||
parts = entry.split("--", 1)
|
||||
else:
|
||||
continue
|
||||
if len(parts) == 2:
|
||||
word = parts[0].strip().strip("., ")
|
||||
meaning = parts[1].strip()
|
||||
if word and meaning:
|
||||
result.append({"word": word, "meanings": [meaning]})
|
||||
return result
|
||||
|
||||
|
||||
def parse_synonym_html(tag: Tag) -> list:
|
||||
text = decode_text(tag)
|
||||
return parse_synonyms(text)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CSS class sets for CC
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Bengali font classes — content to skip (derive from transliteration)
|
||||
BENGALI_CLASSES = frozenset(
|
||||
[
|
||||
"p23",
|
||||
"p29",
|
||||
"p35",
|
||||
"p61",
|
||||
"p62",
|
||||
"p63",
|
||||
"p64",
|
||||
"p65",
|
||||
"p66",
|
||||
"p67",
|
||||
"p68",
|
||||
"p69",
|
||||
"p70",
|
||||
"p72",
|
||||
"p73",
|
||||
"p75",
|
||||
"p83",
|
||||
"p84",
|
||||
"p87",
|
||||
"p88",
|
||||
]
|
||||
)
|
||||
|
||||
# Transliteration pāda classes
|
||||
TRANSLITERATION_CLASSES = frozenset(["p24"])
|
||||
|
||||
# Maybe-transliteration: center-aligned italic that can be transliteration
|
||||
# or embedded verse quotes in purport depending on context
|
||||
MAYBE_TRANSLITERATION_CLASSES = frozenset(["p36", "p38", "p15"])
|
||||
|
||||
# Synonym classes (color #000080)
|
||||
SYNONYM_CLASSES = frozenset(["p26"])
|
||||
|
||||
# Section header classes — SYNONYMS, TRANSLATION, or PURPORT
|
||||
SECTION_HEADER_CLASSES = frozenset(["p25", "p27"])
|
||||
|
||||
# Translation text classes (color #800000)
|
||||
TRANSLATION_CLASSES = frozenset(["p28", "p34"])
|
||||
|
||||
# Purport paragraph classes
|
||||
PURPORT_CLASSES = frozenset(
|
||||
[
|
||||
"p31",
|
||||
"p32",
|
||||
"p33",
|
||||
"p2",
|
||||
"p4",
|
||||
"p5",
|
||||
"p6",
|
||||
"p14",
|
||||
"p16",
|
||||
"p17",
|
||||
"p37",
|
||||
"p40",
|
||||
"p41",
|
||||
"p42",
|
||||
"p43",
|
||||
"p44",
|
||||
"p45",
|
||||
"p46",
|
||||
"p47",
|
||||
"p48",
|
||||
"p49",
|
||||
"p50",
|
||||
"p51",
|
||||
"p52",
|
||||
"p53",
|
||||
"p54",
|
||||
"p55",
|
||||
"p56",
|
||||
"p57",
|
||||
"p58",
|
||||
"p71",
|
||||
"p74",
|
||||
"p76",
|
||||
"p77",
|
||||
"p78",
|
||||
"p79",
|
||||
"p80",
|
||||
"p81",
|
||||
"p82",
|
||||
"p85",
|
||||
"p86",
|
||||
"p89",
|
||||
"p90",
|
||||
"p91",
|
||||
"p92",
|
||||
"p93",
|
||||
"p94",
|
||||
"p95",
|
||||
]
|
||||
)
|
||||
|
||||
# Embedded verse quotations in purport (center-aligned)
|
||||
PURPORT_VERSE_CLASSES = frozenset(["p36", "p15", "p38", "p18", "p59", "p60"])
|
||||
|
||||
# Cross-reference notes (green, center) — skip
|
||||
CITATION_CLASSES = frozenset(["p19", "p30"])
|
||||
|
||||
# Chapter end markers ("Thus end the Bhaktivedanta purports…")
|
||||
CHAPTER_END_CLASSES = frozenset(["p37", "p93"])
|
||||
|
||||
# Chapter bookmark class
|
||||
CHAPTER_BOOKMARK_CLASS = "p11"
|
||||
|
||||
# Verse reference marker
|
||||
VERSE_REF_CLASS = "p20"
|
||||
|
||||
# TEXT N header
|
||||
TEXT_HEADER_CLASS = "p21"
|
||||
|
||||
# TEXT label (maroon)
|
||||
TEXT_LABEL_CLASS = "p22"
|
||||
|
||||
# Līlā banner classes — skip
|
||||
LILA_BANNER_CLASSES = frozenset(["p8", "p39"])
|
||||
|
||||
# Volume marker — skip
|
||||
VOLUME_MARKER_CLASS = "p7"
|
||||
|
||||
# Chapter number/heading — skip
|
||||
CHAPTER_HEADING_CLASSES = frozenset(["p12", "p9", "p10", "p13"])
|
||||
|
||||
# Skip classes (decorative, front matter, etc.)
|
||||
SKIP_CLASSES = frozenset(
|
||||
[
|
||||
"p1", # ScaOptima title
|
||||
"p3", # center indent text (front matter)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def get_css_class(tag: Tag) -> str:
|
||||
"""Get the primary CSS class of a tag."""
|
||||
classes = tag.get("class", [])
|
||||
return classes[0] if classes else ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Līlā name mapping
|
||||
# ---------------------------------------------------------------------------
|
||||
LILA_NAMES = {
|
||||
"adi": "Ādi-līlā",
|
||||
"madhya": "Madhya-līlā",
|
||||
"antya": "Antya-līlā",
|
||||
}
|
||||
|
||||
LILA_ENCODED_TO_KEY = {
|
||||
"ädi": "adi",
|
||||
"adi": "adi",
|
||||
"ādi": "adi",
|
||||
"madhya": "madhya",
|
||||
"antya": "antya",
|
||||
}
|
||||
|
||||
|
||||
def normalize_lila_name(raw: str) -> str:
|
||||
"""Normalize a līlā name from ScaGoudy-encoded text to a key."""
|
||||
lower = raw.strip().lower()
|
||||
# After ScaGoudy decoding, Ädi becomes Ādi
|
||||
decoded = decode_scagoudy(lower)
|
||||
for pattern, key in LILA_ENCODED_TO_KEY.items():
|
||||
if decoded.startswith(pattern) or lower.startswith(pattern):
|
||||
return key
|
||||
return lower
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Reference parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Matches "Ädi 1.1" or "Madhya 8.201" or "Antya 1.3-4"
|
||||
_VERSE_REF_RE = re.compile(
|
||||
r"(Ädi|Adi|Ādi|Madhya|Antya)\s+(\d+)\.(\d+)(?:\s*-\s*(\d+))?",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _parse_verse_ref(text: str):
|
||||
"""Parse a verse reference string.
|
||||
|
||||
Returns (lila_key, chapter, verse, verse_end, clean_ref) or None.
|
||||
"""
|
||||
text = decode_scagoudy(text.strip())
|
||||
m = _VERSE_REF_RE.search(text)
|
||||
if not m:
|
||||
return None
|
||||
|
||||
lila_raw = m.group(1)
|
||||
chapter = int(m.group(2))
|
||||
verse = int(m.group(3))
|
||||
verse_end = int(m.group(4)) if m.group(4) else None
|
||||
|
||||
lila_key = normalize_lila_name(lila_raw)
|
||||
lila_display = {"adi": "Ādi", "madhya": "Madhya", "antya": "Antya"}.get(
|
||||
lila_key, lila_raw
|
||||
)
|
||||
|
||||
if verse_end is not None:
|
||||
clean_ref = f"Cc {lila_display} {chapter}.{verse}-{verse_end}"
|
||||
else:
|
||||
clean_ref = f"Cc {lila_display} {chapter}.{verse}"
|
||||
|
||||
return (lila_key, chapter, verse, verse_end, clean_ref)
|
||||
|
||||
|
||||
def _parse_chapter_bookmark(text: str):
|
||||
"""Parse a chapter bookmark like 'Ädi 1: The Spiritual Masters'.
|
||||
|
||||
Returns (lila_key, chapter, title) or None.
|
||||
"""
|
||||
text = decode_scagoudy(text.strip())
|
||||
# Match "Ādi 1: Title" or "Madhya 9b: Title"
|
||||
m = re.match(r"(Ādi|Adi|Madhya|Antya)\s+(\d+)b?\s*:\s*(.+)", text, re.IGNORECASE)
|
||||
if m:
|
||||
lila_key = normalize_lila_name(m.group(1))
|
||||
chapter = int(m.group(2))
|
||||
title = m.group(3).strip()
|
||||
return (lila_key, chapter, title)
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# State machine
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _finalize_verse(verse: dict):
|
||||
"""Build transliteration, Devanagari and Bengali from pādas."""
|
||||
padas = verse.get("_padas", [])
|
||||
verse["transliteration"] = "\n".join(padas)
|
||||
if padas:
|
||||
paired = pair_padas(padas)
|
||||
verse["devanagari"] = iast_to_devanagari(paired)
|
||||
verse["bengali"] = iast_to_bengali(paired)
|
||||
else:
|
||||
verse["devanagari"] = ""
|
||||
verse["bengali"] = ""
|
||||
for key in ("_padas", "_state"):
|
||||
if key in verse:
|
||||
del verse[key]
|
||||
|
||||
|
||||
def extract_verses(soup: BeautifulSoup):
|
||||
"""Extract verses from parsed CC HTML using a state machine.
|
||||
|
||||
Returns (verses, chapter_titles) where chapter_titles is:
|
||||
{ (lila_key, chapter): title_string }
|
||||
"""
|
||||
paragraphs = soup.find_all("p")
|
||||
|
||||
verses = []
|
||||
current_verse = None
|
||||
state = "IDLE"
|
||||
|
||||
chapter_titles = {} # (lila_key, chapter) -> title
|
||||
|
||||
for p in paragraphs:
|
||||
cls = get_css_class(p)
|
||||
text = p.get_text().strip()
|
||||
|
||||
# --- Chapter bookmark (p11) ---
|
||||
if cls == CHAPTER_BOOKMARK_CLASS:
|
||||
parsed = _parse_chapter_bookmark(text)
|
||||
if parsed:
|
||||
lila_key, chapter, title = parsed
|
||||
chapter_titles[(lila_key, chapter)] = title
|
||||
continue
|
||||
|
||||
# --- Volume/līlā markers — skip ---
|
||||
if cls == VOLUME_MARKER_CLASS or cls in LILA_BANNER_CLASSES:
|
||||
continue
|
||||
|
||||
# --- Chapter heading/number — skip ---
|
||||
if cls in CHAPTER_HEADING_CLASSES or cls in SKIP_CLASSES:
|
||||
continue
|
||||
|
||||
# --- Verse reference (p20) ---
|
||||
if cls == VERSE_REF_CLASS:
|
||||
# Finalize previous verse
|
||||
if current_verse:
|
||||
_finalize_verse(current_verse)
|
||||
verses.append(current_verse)
|
||||
current_verse = None
|
||||
state = "IDLE"
|
||||
|
||||
ref = _parse_verse_ref(text)
|
||||
if ref is None:
|
||||
continue
|
||||
|
||||
lila_key, chapter, verse_num, verse_end, clean_ref = ref
|
||||
|
||||
current_verse = {
|
||||
"book_title": "Śrī Caitanya-caritāmṛta",
|
||||
"lila": lila_key,
|
||||
"chapter": chapter,
|
||||
"verse": verse_num,
|
||||
"reference": clean_ref,
|
||||
"devanagari": "",
|
||||
"bengali": "",
|
||||
"transliteration": "",
|
||||
"_padas": [],
|
||||
"synonyms": [],
|
||||
"translation": "",
|
||||
"purport": [],
|
||||
"_state": "VERSE_START",
|
||||
}
|
||||
if verse_end is not None:
|
||||
current_verse["verse_end"] = verse_end
|
||||
state = "VERSE_START"
|
||||
continue
|
||||
|
||||
# --- TEXT N header (p21) ---
|
||||
if cls == TEXT_HEADER_CLASS and current_verse:
|
||||
continue
|
||||
|
||||
# --- TEXT label (p22) ---
|
||||
if cls == TEXT_LABEL_CLASS:
|
||||
continue
|
||||
|
||||
# --- Bengali (Inben fonts) — SKIP ---
|
||||
if cls in BENGALI_CLASSES and current_verse:
|
||||
state = "BENGALI"
|
||||
current_verse["_state"] = state
|
||||
continue
|
||||
|
||||
# --- Transliteration pādas (p24) ---
|
||||
if (
|
||||
cls in TRANSLITERATION_CLASSES
|
||||
and current_verse
|
||||
and state in ("VERSE_START", "BENGALI", "TRANSLITERATION")
|
||||
):
|
||||
state = "TRANSLITERATION"
|
||||
current_verse["_state"] = state
|
||||
line = decode_text(p)
|
||||
if line:
|
||||
current_verse["_padas"].append(line)
|
||||
continue
|
||||
|
||||
# --- Maybe-transliteration (context-dependent) ---
|
||||
if (
|
||||
cls in MAYBE_TRANSLITERATION_CLASSES
|
||||
and current_verse
|
||||
and state in ("BENGALI", "TRANSLITERATION")
|
||||
):
|
||||
state = "TRANSLITERATION"
|
||||
current_verse["_state"] = state
|
||||
line = decode_text(p)
|
||||
if line:
|
||||
current_verse["_padas"].append(line)
|
||||
continue
|
||||
|
||||
# --- Synonyms (p26) ---
|
||||
if cls in SYNONYM_CLASSES and current_verse:
|
||||
state = "SYNONYMS"
|
||||
current_verse["_state"] = state
|
||||
current_verse["synonyms"] = parse_synonym_html(p)
|
||||
continue
|
||||
|
||||
# --- Section headers: SYNONYMS / TRANSLATION / PURPORT (p25, p27) ---
|
||||
if cls in SECTION_HEADER_CLASSES and current_verse:
|
||||
header_text = text.strip().upper()
|
||||
if "SYNONYM" in header_text:
|
||||
state = "SYNONYMS_HEADER"
|
||||
current_verse["_state"] = state
|
||||
elif "TRANSLATION" in header_text:
|
||||
state = "TRANSLATION"
|
||||
current_verse["_state"] = state
|
||||
elif "PURPORT" in header_text:
|
||||
state = "PURPORT"
|
||||
current_verse["_state"] = state
|
||||
continue
|
||||
|
||||
# --- Translation text (p28, p34) ---
|
||||
if cls in TRANSLATION_CLASSES and current_verse and state == "TRANSLATION":
|
||||
current_verse["translation"] = decode_html_content(p)
|
||||
state = "POST_TRANSLATION"
|
||||
current_verse["_state"] = state
|
||||
continue
|
||||
|
||||
# --- Handle mislabeled TRANSLATION as PURPORT ---
|
||||
if (
|
||||
cls in TRANSLATION_CLASSES
|
||||
and current_verse
|
||||
and state == "PURPORT"
|
||||
and not current_verse.get("translation")
|
||||
):
|
||||
current_verse["translation"] = decode_html_content(p)
|
||||
state = "POST_TRANSLATION"
|
||||
current_verse["_state"] = state
|
||||
continue
|
||||
|
||||
# --- Purport paragraphs ---
|
||||
if state == "PURPORT" and current_verse and cls in PURPORT_CLASSES:
|
||||
content = decode_html_content(p)
|
||||
if content and "Thus end the Bhaktivedanta" not in text:
|
||||
current_verse["purport"].append(content)
|
||||
continue
|
||||
|
||||
# --- Embedded verse quotations in purport ---
|
||||
if state == "PURPORT" and current_verse and cls in PURPORT_VERSE_CLASSES:
|
||||
content = decode_html_content(p)
|
||||
if content:
|
||||
current_verse["purport"].append(content)
|
||||
continue
|
||||
|
||||
# --- Cross-reference citations (p19, p30) — skip ---
|
||||
if cls in CITATION_CLASSES:
|
||||
continue
|
||||
|
||||
# --- Chapter end marker ---
|
||||
if cls in CHAPTER_END_CLASSES:
|
||||
# Check if this is really a "Thus end" marker
|
||||
if "Thus end" in text or "thus end" in text:
|
||||
if current_verse:
|
||||
_finalize_verse(current_verse)
|
||||
verses.append(current_verse)
|
||||
current_verse = None
|
||||
state = "IDLE"
|
||||
continue
|
||||
# Otherwise it might be a purport paragraph with class p37
|
||||
if state == "PURPORT" and current_verse:
|
||||
content = decode_html_content(p)
|
||||
if content:
|
||||
current_verse["purport"].append(content)
|
||||
continue
|
||||
|
||||
# Don't forget the last verse
|
||||
if current_verse:
|
||||
_finalize_verse(current_verse)
|
||||
verses.append(current_verse)
|
||||
|
||||
return verses, chapter_titles
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# JSON output
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def write_json(data, filepath):
|
||||
"""Write JSON with proper Unicode and formatting."""
|
||||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||||
with open(filepath, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
print(f" Written: {filepath}")
|
||||
|
||||
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "cc.html")
|
||||
output_dir = os.path.join(base_dir, "app", "lib", "books", "cc")
|
||||
|
||||
print(f"Reading {html_path}...")
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
soup = BeautifulSoup(f, "html.parser")
|
||||
|
||||
print("Extracting all CC verses...")
|
||||
verses, chapter_titles = extract_verses(soup)
|
||||
|
||||
print(f"\nExtracted {len(verses)} verses across {len(chapter_titles)} chapters")
|
||||
|
||||
# Group by (lila, chapter)
|
||||
lilas_info = {} # lila_key -> { chapter -> verse_count }
|
||||
for verse in verses:
|
||||
lk = verse["lila"]
|
||||
ch = verse["chapter"]
|
||||
if lk not in lilas_info:
|
||||
lilas_info[lk] = {}
|
||||
if ch not in lilas_info[lk]:
|
||||
lilas_info[lk][ch] = 0
|
||||
lilas_info[lk][ch] += 1
|
||||
|
||||
# Build intro.json
|
||||
lilas_list = []
|
||||
lila_order = ["adi", "madhya", "antya"]
|
||||
for lila_key in lila_order:
|
||||
if lila_key not in lilas_info:
|
||||
continue
|
||||
chapters_in_lila = lilas_info[lila_key]
|
||||
chapters_list = []
|
||||
for ch_num in sorted(chapters_in_lila.keys()):
|
||||
title = chapter_titles.get((lila_key, ch_num), "")
|
||||
chapters_list.append(
|
||||
{
|
||||
"number": ch_num,
|
||||
"title": title,
|
||||
"totalVerses": chapters_in_lila[ch_num],
|
||||
}
|
||||
)
|
||||
total_verses = sum(chapters_in_lila.values())
|
||||
lilas_list.append(
|
||||
{
|
||||
"key": lila_key,
|
||||
"name": LILA_NAMES.get(lila_key, lila_key),
|
||||
"totalChapters": len(chapters_list),
|
||||
"totalVerses": total_verses,
|
||||
"chapters": chapters_list,
|
||||
}
|
||||
)
|
||||
|
||||
intro = {
|
||||
"book": "cc",
|
||||
"title": "Śrī Caitanya-caritāmṛta",
|
||||
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
||||
"totalLilas": len(lilas_list),
|
||||
"lilas": lilas_list,
|
||||
}
|
||||
write_json(intro, os.path.join(output_dir, "intro.json"))
|
||||
|
||||
# Write verse JSON files:
|
||||
# app/lib/books/cc/lilas/{lila}/chapters/{ch}/verses/{vs}.json
|
||||
for verse in verses:
|
||||
lila = verse["lila"]
|
||||
ch = str(verse["chapter"]).zfill(2)
|
||||
vs = str(verse["verse"]).zfill(3)
|
||||
filepath = os.path.join(
|
||||
output_dir, "lilas", lila, "chapters", ch, "verses", f"{vs}.json"
|
||||
)
|
||||
write_json(verse, filepath)
|
||||
|
||||
# Print summary
|
||||
print("\n--- Verification ---")
|
||||
for lila_key in lila_order:
|
||||
if lila_key not in lilas_info:
|
||||
continue
|
||||
chapters_in_lila = lilas_info[lila_key]
|
||||
total_verses_in_lila = sum(chapters_in_lila.values())
|
||||
print(
|
||||
f"\n{LILA_NAMES.get(lila_key, lila_key)}: "
|
||||
f"{len(chapters_in_lila)} chapters, {total_verses_in_lila} verses"
|
||||
)
|
||||
for ch_num in sorted(chapters_in_lila.keys()):
|
||||
ch_verses = [
|
||||
v for v in verses if v["lila"] == lila_key and v["chapter"] == ch_num
|
||||
]
|
||||
if ch_verses:
|
||||
first = ch_verses[0]
|
||||
last = ch_verses[-1]
|
||||
title = chapter_titles.get((lila_key, ch_num), "")
|
||||
print(
|
||||
f" Ch {ch_num:>2}: {len(ch_verses):>4} verses "
|
||||
f"({first['reference']} → {last['reference']}) {title}"
|
||||
)
|
||||
|
||||
# Quick quality check: show first verse
|
||||
if verses:
|
||||
print("\n--- Sample (first verse) ---")
|
||||
v = verses[0]
|
||||
print(f"Reference: {v['reference']}")
|
||||
print(f"Devanagari:\n{v['devanagari']}")
|
||||
print(f"Bengali:\n{v['bengali']}")
|
||||
print(f"Transliteration:\n{v['transliteration']}")
|
||||
print(f"Synonyms: {len(v['synonyms'])} entries")
|
||||
if v["synonyms"]:
|
||||
print(f" First: {v['synonyms'][0]}")
|
||||
print(f"Translation: {v['translation'][:120]}...")
|
||||
print(f"Purport: {len(v['purport'])} paragraphs")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,467 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fix Bengali font encoding in cc.html.
|
||||
|
||||
When textutil converts the CC RTF to HTML, it sees that the Inben* fonts
|
||||
are declared with \fcharset2 (Symbol encoding). It then maps the raw bytes
|
||||
(0x20–0xFF) through the Apple Symbol encoding table to Unicode codepoints
|
||||
like ℘ (U+2118), ⇓ (U+21D3), ∈ (U+2208), etc.
|
||||
|
||||
But the Inben* fonts actually have their glyphs in the Private Use Area at
|
||||
U+F020–U+F0FF (i.e., U+F000 + original_byte). So the browser can't find
|
||||
the glyphs and falls back to system fonts, showing math symbols and Apple
|
||||
logos instead of Bengali script.
|
||||
|
||||
This script:
|
||||
1. Identifies all CSS classes that use Inben* fonts
|
||||
2. Finds all <p> tags with those classes
|
||||
3. Remaps the incorrectly-encoded Unicode codepoints back to the PUA range
|
||||
that the font actually uses
|
||||
4. Writes the fixed HTML back to disk
|
||||
|
||||
Usage:
|
||||
python3 scripts/fix-bengali-encoding.py
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Apple Symbol encoding table: byte → Unicode codepoint
|
||||
# This is the mapping that textutil applied when converting the RTF.
|
||||
# Source: https://unicode.org/Public/MAPPINGS/VENDORS/APPLE/SYMBOL.TXT
|
||||
# ---------------------------------------------------------------------------
|
||||
APPLE_SYMBOL_BYTE_TO_UNICODE = {
|
||||
0x20: 0x0020, # SPACE
|
||||
0x21: 0x0021, # EXCLAMATION MARK
|
||||
0x22: 0x2200, # FOR ALL
|
||||
0x23: 0x0023, # NUMBER SIGN
|
||||
0x24: 0x2203, # THERE EXISTS
|
||||
0x25: 0x0025, # PERCENT SIGN
|
||||
0x26: 0x0026, # AMPERSAND
|
||||
0x27: 0x220D, # SMALL CONTAINS AS MEMBER
|
||||
0x28: 0x0028, # LEFT PARENTHESIS
|
||||
0x29: 0x0029, # RIGHT PARENTHESIS
|
||||
0x2A: 0x2217, # ASTERISK OPERATOR
|
||||
0x2B: 0x002B, # PLUS SIGN
|
||||
0x2C: 0x002C, # COMMA
|
||||
0x2D: 0x2212, # MINUS SIGN
|
||||
0x2E: 0x002E, # FULL STOP
|
||||
0x2F: 0x002F, # SOLIDUS
|
||||
0x30: 0x0030, # DIGIT ZERO
|
||||
0x31: 0x0031, # DIGIT ONE
|
||||
0x32: 0x0032, # DIGIT TWO
|
||||
0x33: 0x0033, # DIGIT THREE
|
||||
0x34: 0x0034, # DIGIT FOUR
|
||||
0x35: 0x0035, # DIGIT FIVE
|
||||
0x36: 0x0036, # DIGIT SIX
|
||||
0x37: 0x0037, # DIGIT SEVEN
|
||||
0x38: 0x0038, # DIGIT EIGHT
|
||||
0x39: 0x0039, # DIGIT NINE
|
||||
0x3A: 0x003A, # COLON
|
||||
0x3B: 0x003B, # SEMICOLON
|
||||
0x3C: 0x003C, # LESS-THAN SIGN
|
||||
0x3D: 0x003D, # EQUALS SIGN
|
||||
0x3E: 0x003E, # GREATER-THAN SIGN
|
||||
0x3F: 0x003F, # QUESTION MARK
|
||||
0x40: 0x2245, # APPROXIMATELY EQUAL TO
|
||||
0x41: 0x0391, # GREEK CAPITAL LETTER ALPHA
|
||||
0x42: 0x0392, # GREEK CAPITAL LETTER BETA
|
||||
0x43: 0x03A7, # GREEK CAPITAL LETTER CHI
|
||||
0x44: 0x0394, # GREEK CAPITAL LETTER DELTA
|
||||
0x45: 0x0395, # GREEK CAPITAL LETTER EPSILON
|
||||
0x46: 0x03A6, # GREEK CAPITAL LETTER PHI
|
||||
0x47: 0x0393, # GREEK CAPITAL LETTER GAMMA
|
||||
0x48: 0x0397, # GREEK CAPITAL LETTER ETA
|
||||
0x49: 0x0399, # GREEK CAPITAL LETTER IOTA
|
||||
0x4A: 0x03D1, # GREEK THETA SYMBOL
|
||||
0x4B: 0x039A, # GREEK CAPITAL LETTER KAPPA
|
||||
0x4C: 0x039B, # GREEK CAPITAL LETTER LAMDA
|
||||
0x4D: 0x039C, # GREEK CAPITAL LETTER MU
|
||||
0x4E: 0x039D, # GREEK CAPITAL LETTER NU
|
||||
0x4F: 0x039F, # GREEK CAPITAL LETTER OMICRON
|
||||
0x50: 0x03A0, # GREEK CAPITAL LETTER PI
|
||||
0x51: 0x0398, # GREEK CAPITAL LETTER THETA
|
||||
0x52: 0x03A1, # GREEK CAPITAL LETTER RHO
|
||||
0x53: 0x03A3, # GREEK CAPITAL LETTER SIGMA
|
||||
0x54: 0x03A4, # GREEK CAPITAL LETTER TAU
|
||||
0x55: 0x03A5, # GREEK CAPITAL LETTER UPSILON
|
||||
0x56: 0x03C2, # GREEK SMALL LETTER FINAL SIGMA
|
||||
0x57: 0x03A9, # GREEK CAPITAL LETTER OMEGA
|
||||
0x58: 0x039E, # GREEK CAPITAL LETTER XI
|
||||
0x59: 0x03A8, # GREEK CAPITAL LETTER PSI
|
||||
0x5A: 0x0396, # GREEK CAPITAL LETTER ZETA
|
||||
0x5B: 0x005B, # LEFT SQUARE BRACKET
|
||||
0x5C: 0x2234, # THEREFORE
|
||||
0x5D: 0x005D, # RIGHT SQUARE BRACKET
|
||||
0x5E: 0x22A5, # UP TACK
|
||||
0x5F: 0x005F, # LOW LINE
|
||||
0x60: 0xF8E5, # RADICAL EXTENDER (Apple PUA)
|
||||
0x61: 0x03B1, # GREEK SMALL LETTER ALPHA
|
||||
0x62: 0x03B2, # GREEK SMALL LETTER BETA
|
||||
0x63: 0x03C7, # GREEK SMALL LETTER CHI
|
||||
0x64: 0x03B4, # GREEK SMALL LETTER DELTA
|
||||
0x65: 0x03B5, # GREEK SMALL LETTER EPSILON
|
||||
0x66: 0x03C6, # GREEK SMALL LETTER PHI
|
||||
0x67: 0x03B3, # GREEK SMALL LETTER GAMMA
|
||||
0x68: 0x03B7, # GREEK SMALL LETTER ETA
|
||||
0x69: 0x03B9, # GREEK SMALL LETTER IOTA
|
||||
0x6A: 0x03D5, # GREEK PHI SYMBOL
|
||||
0x6B: 0x03BA, # GREEK SMALL LETTER KAPPA
|
||||
0x6C: 0x03BB, # GREEK SMALL LETTER LAMDA
|
||||
0x6D: 0x03BC, # GREEK SMALL LETTER MU
|
||||
0x6E: 0x03BD, # GREEK SMALL LETTER NU
|
||||
0x6F: 0x03BF, # GREEK SMALL LETTER OMICRON
|
||||
0x70: 0x03C0, # GREEK SMALL LETTER PI
|
||||
0x71: 0x03B8, # GREEK SMALL LETTER THETA
|
||||
0x72: 0x03C1, # GREEK SMALL LETTER RHO
|
||||
0x73: 0x03C3, # GREEK SMALL LETTER SIGMA
|
||||
0x74: 0x03C4, # GREEK SMALL LETTER TAU
|
||||
0x75: 0x03C5, # GREEK SMALL LETTER UPSILON
|
||||
0x76: 0x03D6, # GREEK PI SYMBOL
|
||||
0x77: 0x03C9, # GREEK SMALL LETTER OMEGA
|
||||
0x78: 0x03BE, # GREEK SMALL LETTER XI
|
||||
0x79: 0x03C8, # GREEK SMALL LETTER PSI
|
||||
0x7A: 0x03B6, # GREEK SMALL LETTER ZETA
|
||||
0x7B: 0x007B, # LEFT CURLY BRACKET
|
||||
0x7C: 0x007C, # VERTICAL LINE
|
||||
0x7D: 0x007D, # RIGHT CURLY BRACKET
|
||||
0x7E: 0x223C, # TILDE OPERATOR
|
||||
0xA0: 0x20AC, # EURO SIGN
|
||||
0xA1: 0x03D2, # GREEK UPSILON WITH HOOK SYMBOL
|
||||
0xA2: 0x2032, # PRIME
|
||||
0xA3: 0x2264, # LESS-THAN OR EQUAL TO
|
||||
0xA4: 0x2044, # FRACTION SLASH
|
||||
0xA5: 0x221E, # INFINITY
|
||||
0xA6: 0x0192, # LATIN SMALL LETTER F WITH HOOK
|
||||
0xA7: 0x2663, # BLACK CLUB SUIT
|
||||
0xA8: 0x2666, # BLACK DIAMOND SUIT
|
||||
0xA9: 0x2665, # BLACK HEART SUIT
|
||||
0xAA: 0x2660, # BLACK SPADE SUIT
|
||||
0xAB: 0x2194, # LEFT RIGHT ARROW
|
||||
0xAC: 0x2190, # LEFTWARDS ARROW
|
||||
0xAD: 0x2191, # UPWARDS ARROW
|
||||
0xAE: 0x2192, # RIGHTWARDS ARROW
|
||||
0xAF: 0x2193, # DOWNWARDS ARROW
|
||||
0xB0: 0x00B0, # DEGREE SIGN
|
||||
0xB1: 0x00B1, # PLUS-MINUS SIGN
|
||||
0xB2: 0x2033, # DOUBLE PRIME
|
||||
0xB3: 0x2265, # GREATER-THAN OR EQUAL TO
|
||||
0xB4: 0x00D7, # MULTIPLICATION SIGN
|
||||
0xB5: 0x221D, # PROPORTIONAL TO
|
||||
0xB6: 0x2202, # PARTIAL DIFFERENTIAL
|
||||
0xB7: 0x2022, # BULLET
|
||||
0xB8: 0x00F7, # DIVISION SIGN
|
||||
0xB9: 0x2260, # NOT EQUAL TO
|
||||
0xBA: 0x2261, # IDENTICAL TO
|
||||
0xBB: 0x2248, # ALMOST EQUAL TO
|
||||
0xBC: 0x2026, # HORIZONTAL ELLIPSIS
|
||||
0xBD: 0x23D0, # VERTICAL LINE EXTENSION
|
||||
0xBE: 0x23AF, # HORIZONTAL LINE EXTENSION
|
||||
0xBF: 0x21B5, # DOWNWARDS ARROW WITH CORNER LEFTWARDS
|
||||
0xC0: 0x2135, # ALEF SYMBOL
|
||||
0xC1: 0x2111, # BLACK-LETTER CAPITAL I
|
||||
0xC2: 0x211C, # BLACK-LETTER CAPITAL R
|
||||
0xC3: 0x2118, # SCRIPT CAPITAL P (Weierstrass)
|
||||
0xC4: 0x2297, # CIRCLED TIMES
|
||||
0xC5: 0x2295, # CIRCLED PLUS
|
||||
0xC6: 0x2205, # EMPTY SET
|
||||
0xC7: 0x2229, # INTERSECTION
|
||||
0xC8: 0x222A, # UNION
|
||||
0xC9: 0x2283, # SUPERSET OF
|
||||
0xCA: 0x2287, # SUPERSET OF OR EQUAL TO
|
||||
0xCB: 0x2284, # NOT A SUBSET OF
|
||||
0xCC: 0x2282, # SUBSET OF
|
||||
0xCD: 0x2286, # SUBSET OF OR EQUAL TO
|
||||
0xCE: 0x2208, # ELEMENT OF
|
||||
0xCF: 0x2209, # NOT AN ELEMENT OF
|
||||
0xD0: 0x2220, # ANGLE
|
||||
0xD1: 0x2207, # NABLA
|
||||
0xD2: 0x00AE, # REGISTERED SIGN (also at 0xE2)
|
||||
0xD3: 0x00A9, # COPYRIGHT SIGN (also at 0xE3)
|
||||
0xD4: 0x2122, # TRADE MARK SIGN (also at 0xE4)
|
||||
0xD5: 0x220F, # N-ARY PRODUCT
|
||||
0xD6: 0x221A, # SQUARE ROOT
|
||||
0xD7: 0x22C5, # DOT OPERATOR
|
||||
0xD8: 0x00AC, # NOT SIGN
|
||||
0xD9: 0x2227, # LOGICAL AND
|
||||
0xDA: 0x2228, # LOGICAL OR
|
||||
0xDB: 0x21D4, # LEFT RIGHT DOUBLE ARROW
|
||||
0xDC: 0x21D0, # LEFTWARDS DOUBLE ARROW
|
||||
0xDD: 0x21D1, # UPWARDS DOUBLE ARROW
|
||||
0xDE: 0x21D2, # RIGHTWARDS DOUBLE ARROW
|
||||
0xDF: 0x21D3, # DOWNWARDS DOUBLE ARROW
|
||||
0xE0: 0x25CA, # LOZENGE
|
||||
0xE1: 0x3008, # LEFT ANGLE BRACKET
|
||||
0xE2: 0x00AE, # REGISTERED SIGN (duplicate)
|
||||
0xE3: 0x00A9, # COPYRIGHT SIGN (duplicate)
|
||||
0xE4: 0x2122, # TRADE MARK SIGN (duplicate)
|
||||
0xE5: 0x2211, # N-ARY SUMMATION
|
||||
0xE6: 0x239B, # LEFT PARENTHESIS UPPER HOOK
|
||||
0xE7: 0x239C, # LEFT PARENTHESIS EXTENSION
|
||||
0xE8: 0x239D, # LEFT PARENTHESIS LOWER HOOK
|
||||
0xE9: 0x23A1, # LEFT SQUARE BRACKET UPPER CORNER
|
||||
0xEA: 0x23A2, # LEFT SQUARE BRACKET EXTENSION
|
||||
0xEB: 0x23A3, # LEFT SQUARE BRACKET LOWER CORNER
|
||||
0xEC: 0x23A7, # LEFT CURLY BRACKET UPPER HOOK
|
||||
0xED: 0x23A8, # LEFT CURLY BRACKET MIDDLE PIECE
|
||||
0xEE: 0x23A9, # LEFT CURLY BRACKET LOWER HOOK
|
||||
0xEF: 0x23AA, # CURLY BRACKET EXTENSION
|
||||
0xF0: 0xF8FF, # Apple logo (Apple PUA)
|
||||
0xF1: 0x3009, # RIGHT ANGLE BRACKET
|
||||
0xF2: 0x222B, # INTEGRAL
|
||||
0xF3: 0x2320, # TOP HALF INTEGRAL
|
||||
0xF4: 0x23AE, # INTEGRAL EXTENSION
|
||||
0xF5: 0x2321, # BOTTOM HALF INTEGRAL
|
||||
0xF6: 0x239E, # RIGHT PARENTHESIS UPPER HOOK
|
||||
0xF7: 0x239F, # RIGHT PARENTHESIS EXTENSION
|
||||
0xF8: 0x23A0, # RIGHT PARENTHESIS LOWER HOOK
|
||||
0xF9: 0x23A4, # RIGHT SQUARE BRACKET UPPER CORNER
|
||||
0xFA: 0x23A5, # RIGHT SQUARE BRACKET EXTENSION
|
||||
0xFB: 0x23A6, # RIGHT SQUARE BRACKET LOWER CORNER
|
||||
0xFC: 0x23AB, # RIGHT CURLY BRACKET UPPER HOOK
|
||||
0xFD: 0x23AC, # RIGHT CURLY BRACKET MIDDLE PIECE
|
||||
0xFE: 0x23AD, # RIGHT CURLY BRACKET LOWER HOOK
|
||||
0xFF: 0x23AF, # HORIZONTAL LINE EXTENSION (duplicate of 0xBE)
|
||||
}
|
||||
|
||||
# Some codepoints in the Apple Symbol table map to the same Unicode value
|
||||
# (e.g., 0xD2 and 0xE2 both map to U+00AE). When building the reverse
|
||||
# mapping we can't distinguish them, but the font has separate glyphs at
|
||||
# both PUA positions. For the duplicates (0xD2/0xE2, 0xD3/0xE3, 0xD4/0xE4,
|
||||
# 0xBE/0xFF), textutil seems to always produce the Unicode codepoint, so
|
||||
# we'll map it to the lower byte position (0xD2, 0xD3, 0xD4, 0xBE).
|
||||
# If the higher position glyph is needed, the font's fallback will handle it.
|
||||
|
||||
|
||||
def build_unicode_to_pua_map():
|
||||
"""Build the reverse mapping: Unicode codepoint → PUA codepoint (U+F0xx).
|
||||
|
||||
For each byte value in the Symbol encoding, the font stores its glyph at
|
||||
U+F000 + byte. textutil converted byte → Unicode via the Symbol table.
|
||||
We reverse that: Unicode → U+F000 + byte.
|
||||
"""
|
||||
unicode_to_pua = {}
|
||||
for byte_val, unicode_cp in APPLE_SYMBOL_BYTE_TO_UNICODE.items():
|
||||
pua_cp = 0xF000 + byte_val
|
||||
# For duplicates, prefer the lower byte (first occurrence wins)
|
||||
if unicode_cp not in unicode_to_pua:
|
||||
unicode_to_pua[unicode_cp] = pua_cp
|
||||
# Also handle U+F87F which appears in some files — this is Apple PUA for
|
||||
# the "delete" character (byte 0x7F), which isn't in the standard Symbol
|
||||
# table but textutil sometimes emits it
|
||||
unicode_to_pua[0xF87F] = 0xF07F
|
||||
# U+F8E5 (radical extender) maps to byte 0x60
|
||||
unicode_to_pua[0xF8E5] = 0xF060
|
||||
return unicode_to_pua
|
||||
|
||||
|
||||
def find_inben_classes(html: str) -> set:
|
||||
"""Find all CSS class names that reference Inben* fonts."""
|
||||
# Match class definitions in <style> like: p.p29 {...font: 18.0px Inbenr...}
|
||||
classes = set()
|
||||
for m in re.finditer(r"p\.(p\d+)\s*\{[^}]*font:\s*[^}]*Inben[a-z]*[^}]*\}", html):
|
||||
classes.add(m.group(1))
|
||||
return classes
|
||||
|
||||
|
||||
def find_inben_span_classes(html: str) -> set:
|
||||
"""Find all span CSS class names that reference Inben* fonts."""
|
||||
classes = set()
|
||||
for m in re.finditer(
|
||||
r"span\.(s\d+)\s*\{[^}]*font:\s*[^}]*Inben[a-z]*[^}]*\}", html
|
||||
):
|
||||
classes.add(m.group(1))
|
||||
return classes
|
||||
|
||||
|
||||
def remap_text(text: str, unicode_to_pua: dict) -> str:
|
||||
"""Remap Symbol-encoded Unicode codepoints to PUA codepoints."""
|
||||
result = []
|
||||
for ch in text:
|
||||
cp = ord(ch)
|
||||
if cp in unicode_to_pua:
|
||||
result.append(chr(unicode_to_pua[cp]))
|
||||
else:
|
||||
result.append(ch)
|
||||
return "".join(result)
|
||||
|
||||
|
||||
def fix_bengali_encoding(html: str) -> str:
|
||||
"""Fix Bengali font encoding in the HTML by remapping codepoints.
|
||||
|
||||
Strategy: We can't use BeautifulSoup because it would re-serialize the
|
||||
entire HTML and potentially break formatting. Instead, we use regex to
|
||||
find <p> and <span> tags with Bengali font classes and remap only their
|
||||
text content, preserving all HTML tags and attributes inside them.
|
||||
"""
|
||||
unicode_to_pua = build_unicode_to_pua_map()
|
||||
inben_p_classes = find_inben_classes(html)
|
||||
inben_s_classes = find_inben_span_classes(html)
|
||||
|
||||
if not inben_p_classes and not inben_s_classes:
|
||||
print("WARNING: No Inben* font classes found in CSS!")
|
||||
return html
|
||||
|
||||
print(f"Found Bengali font <p> classes: {sorted(inben_p_classes)}")
|
||||
print(f"Found Bengali font <span> classes: {sorted(inben_s_classes)}")
|
||||
|
||||
# Build a regex pattern that matches <p class="pXX"> ... </p> for Bengali classes
|
||||
# We need to handle nested tags (like <br>, <span>, <b>, <i>) inside the <p>.
|
||||
# The approach: match the opening tag, then remap characters in the content
|
||||
# while preserving HTML tags.
|
||||
|
||||
p_class_pattern = "|".join(re.escape(c) for c in sorted(inben_p_classes))
|
||||
s_class_pattern = (
|
||||
"|".join(re.escape(c) for c in sorted(inben_s_classes))
|
||||
if inben_s_classes
|
||||
else None
|
||||
)
|
||||
|
||||
total_remapped = 0
|
||||
|
||||
def remap_tag_content(m):
|
||||
"""Remap text content inside a tag, preserving inner HTML tags."""
|
||||
nonlocal total_remapped
|
||||
full_match = m.group(0)
|
||||
opening_tag = m.group(1)
|
||||
inner = m.group(2)
|
||||
closing_tag = m.group(3)
|
||||
|
||||
# Remap only text nodes (not HTML tags)
|
||||
def remap_text_nodes(inner_html):
|
||||
nonlocal total_remapped
|
||||
parts = re.split(r"(<[^>]+>)", inner_html)
|
||||
result = []
|
||||
for part in parts:
|
||||
if part.startswith("<"):
|
||||
# HTML tag — preserve as-is
|
||||
result.append(part)
|
||||
else:
|
||||
# Text node — remap codepoints
|
||||
remapped = remap_text(part, unicode_to_pua)
|
||||
if remapped != part:
|
||||
total_remapped += sum(
|
||||
1 for a, b in zip(part, remapped) if a != b
|
||||
)
|
||||
result.append(remapped)
|
||||
return "".join(result)
|
||||
|
||||
new_inner = remap_text_nodes(inner)
|
||||
return f"{opening_tag}{new_inner}{closing_tag}"
|
||||
|
||||
# Fix <p class="pXX"> tags
|
||||
if p_class_pattern:
|
||||
pattern = re.compile(
|
||||
rf"(<p\s+class=\"(?:{p_class_pattern})\"[^>]*>)(.*?)(</p>)",
|
||||
re.DOTALL,
|
||||
)
|
||||
html = pattern.sub(remap_tag_content, html)
|
||||
|
||||
# Fix <span class="sXX"> tags with Bengali fonts
|
||||
if s_class_pattern:
|
||||
pattern = re.compile(
|
||||
rf"(<span\s+class=\"(?:{s_class_pattern})\"[^>]*>)(.*?)(</span>)",
|
||||
re.DOTALL,
|
||||
)
|
||||
html = pattern.sub(remap_tag_content, html)
|
||||
|
||||
print(f"Remapped {total_remapped} characters")
|
||||
return html
|
||||
|
||||
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "cc.html")
|
||||
|
||||
if not os.path.exists(html_path):
|
||||
print(f"ERROR: {html_path} not found!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Reading {html_path}...")
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
html = f.read()
|
||||
|
||||
print(f"File size: {len(html):,} bytes")
|
||||
print()
|
||||
|
||||
# --- Verify the font files exist and have PUA mappings ---
|
||||
fonts_dir = os.path.join(base_dir, "public", "fonts")
|
||||
for font_name in ["inbenr11.ttf", "inbeni11.ttf", "inbenb11.ttf", "inbeno11.ttf"]:
|
||||
font_path = os.path.join(fonts_dir, font_name)
|
||||
if os.path.exists(font_path):
|
||||
print(f" ✓ Font found: {font_name}")
|
||||
else:
|
||||
print(f" ✗ Font MISSING: {font_name}")
|
||||
|
||||
print()
|
||||
|
||||
# --- Fix the encoding ---
|
||||
fixed_html = fix_bengali_encoding(html)
|
||||
|
||||
# --- Verify the fix ---
|
||||
unicode_to_pua = build_unicode_to_pua_map()
|
||||
|
||||
# Count remaining Symbol-encoded chars in Bengali paragraphs
|
||||
inben_classes = find_inben_classes(fixed_html)
|
||||
remaining = 0
|
||||
for m in re.finditer(
|
||||
rf'<p\s+class="(?:{"|".join(inben_classes)})"[^>]*>(.*?)</p>',
|
||||
fixed_html,
|
||||
re.DOTALL,
|
||||
):
|
||||
inner = re.sub(r"<[^>]+>", "", m.group(1)) # strip tags
|
||||
for ch in inner:
|
||||
cp = ord(ch)
|
||||
if cp in unicode_to_pua and not (0xF000 <= cp <= 0xF0FF):
|
||||
remaining += 1
|
||||
|
||||
pua_count = 0
|
||||
for m in re.finditer(
|
||||
rf'<p\s+class="(?:{"|".join(inben_classes)})"[^>]*>(.*?)</p>',
|
||||
fixed_html,
|
||||
re.DOTALL,
|
||||
):
|
||||
inner = re.sub(r"<[^>]+>", "", m.group(1))
|
||||
for ch in inner:
|
||||
cp = ord(ch)
|
||||
if 0xF000 <= cp <= 0xF0FF:
|
||||
pua_count += 1
|
||||
|
||||
print(f"\nVerification:")
|
||||
print(f" PUA characters (correct): {pua_count:,}")
|
||||
print(f" Remaining Symbol-encoded (incorrect): {remaining:,}")
|
||||
|
||||
if remaining > 0:
|
||||
print(f"\n WARNING: {remaining} characters could not be remapped!")
|
||||
print(" These may be characters outside the Symbol encoding range.")
|
||||
|
||||
# --- Write the fixed HTML ---
|
||||
print(f"\nWriting fixed HTML to {html_path}...")
|
||||
with open(html_path, "w", encoding="utf-8") as f:
|
||||
f.write(fixed_html)
|
||||
|
||||
print("Done!")
|
||||
|
||||
# --- Quick visual check ---
|
||||
print("\n--- Sample Bengali text (first 3 paragraphs) ---")
|
||||
count = 0
|
||||
for m in re.finditer(
|
||||
rf'<p\s+class="(?:{"|".join(inben_classes)})"[^>]*>(.*?)</p>',
|
||||
fixed_html,
|
||||
re.DOTALL,
|
||||
):
|
||||
inner = re.sub(r"<[^>]+>", "", m.group(1)).strip()
|
||||
if inner and len(inner) > 10:
|
||||
# Show hex of first few chars to verify they're in PUA range
|
||||
hex_preview = " ".join(f"U+{ord(c):04X}" for c in inner[:10])
|
||||
print(f" Text: {inner[:60]}")
|
||||
print(f" Hex: {hex_preview}")
|
||||
print()
|
||||
count += 1
|
||||
if count >= 3:
|
||||
break
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user