468 lines
17 KiB
Python
468 lines
17 KiB
Python
#!/usr/bin/env python3
|
||
"""Fix Bengali font encoding in cc.html.
|
||
|
||
When textutil converts the CC RTF to HTML, it sees that the Inben* fonts
|
||
are declared with \fcharset2 (Symbol encoding). It then maps the raw bytes
|
||
(0x20–0xFF) through the Apple Symbol encoding table to Unicode codepoints
|
||
like ℘ (U+2118), ⇓ (U+21D3), ∈ (U+2208), etc.
|
||
|
||
But the Inben* fonts actually have their glyphs in the Private Use Area at
|
||
U+F020–U+F0FF (i.e., U+F000 + original_byte). So the browser can't find
|
||
the glyphs and falls back to system fonts, showing math symbols and Apple
|
||
logos instead of Bengali script.
|
||
|
||
This script:
|
||
1. Identifies all CSS classes that use Inben* fonts
|
||
2. Finds all <p> tags with those classes
|
||
3. Remaps the incorrectly-encoded Unicode codepoints back to the PUA range
|
||
that the font actually uses
|
||
4. Writes the fixed HTML back to disk
|
||
|
||
Usage:
|
||
python3 scripts/fix-bengali-encoding.py
|
||
"""
|
||
|
||
import os
|
||
import re
|
||
import sys
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Apple Symbol encoding table: byte → Unicode codepoint
|
||
# This is the mapping that textutil applied when converting the RTF.
|
||
# Source: https://unicode.org/Public/MAPPINGS/VENDORS/APPLE/SYMBOL.TXT
|
||
# ---------------------------------------------------------------------------
|
||
APPLE_SYMBOL_BYTE_TO_UNICODE = {
|
||
0x20: 0x0020, # SPACE
|
||
0x21: 0x0021, # EXCLAMATION MARK
|
||
0x22: 0x2200, # FOR ALL
|
||
0x23: 0x0023, # NUMBER SIGN
|
||
0x24: 0x2203, # THERE EXISTS
|
||
0x25: 0x0025, # PERCENT SIGN
|
||
0x26: 0x0026, # AMPERSAND
|
||
0x27: 0x220D, # SMALL CONTAINS AS MEMBER
|
||
0x28: 0x0028, # LEFT PARENTHESIS
|
||
0x29: 0x0029, # RIGHT PARENTHESIS
|
||
0x2A: 0x2217, # ASTERISK OPERATOR
|
||
0x2B: 0x002B, # PLUS SIGN
|
||
0x2C: 0x002C, # COMMA
|
||
0x2D: 0x2212, # MINUS SIGN
|
||
0x2E: 0x002E, # FULL STOP
|
||
0x2F: 0x002F, # SOLIDUS
|
||
0x30: 0x0030, # DIGIT ZERO
|
||
0x31: 0x0031, # DIGIT ONE
|
||
0x32: 0x0032, # DIGIT TWO
|
||
0x33: 0x0033, # DIGIT THREE
|
||
0x34: 0x0034, # DIGIT FOUR
|
||
0x35: 0x0035, # DIGIT FIVE
|
||
0x36: 0x0036, # DIGIT SIX
|
||
0x37: 0x0037, # DIGIT SEVEN
|
||
0x38: 0x0038, # DIGIT EIGHT
|
||
0x39: 0x0039, # DIGIT NINE
|
||
0x3A: 0x003A, # COLON
|
||
0x3B: 0x003B, # SEMICOLON
|
||
0x3C: 0x003C, # LESS-THAN SIGN
|
||
0x3D: 0x003D, # EQUALS SIGN
|
||
0x3E: 0x003E, # GREATER-THAN SIGN
|
||
0x3F: 0x003F, # QUESTION MARK
|
||
0x40: 0x2245, # APPROXIMATELY EQUAL TO
|
||
0x41: 0x0391, # GREEK CAPITAL LETTER ALPHA
|
||
0x42: 0x0392, # GREEK CAPITAL LETTER BETA
|
||
0x43: 0x03A7, # GREEK CAPITAL LETTER CHI
|
||
0x44: 0x0394, # GREEK CAPITAL LETTER DELTA
|
||
0x45: 0x0395, # GREEK CAPITAL LETTER EPSILON
|
||
0x46: 0x03A6, # GREEK CAPITAL LETTER PHI
|
||
0x47: 0x0393, # GREEK CAPITAL LETTER GAMMA
|
||
0x48: 0x0397, # GREEK CAPITAL LETTER ETA
|
||
0x49: 0x0399, # GREEK CAPITAL LETTER IOTA
|
||
0x4A: 0x03D1, # GREEK THETA SYMBOL
|
||
0x4B: 0x039A, # GREEK CAPITAL LETTER KAPPA
|
||
0x4C: 0x039B, # GREEK CAPITAL LETTER LAMDA
|
||
0x4D: 0x039C, # GREEK CAPITAL LETTER MU
|
||
0x4E: 0x039D, # GREEK CAPITAL LETTER NU
|
||
0x4F: 0x039F, # GREEK CAPITAL LETTER OMICRON
|
||
0x50: 0x03A0, # GREEK CAPITAL LETTER PI
|
||
0x51: 0x0398, # GREEK CAPITAL LETTER THETA
|
||
0x52: 0x03A1, # GREEK CAPITAL LETTER RHO
|
||
0x53: 0x03A3, # GREEK CAPITAL LETTER SIGMA
|
||
0x54: 0x03A4, # GREEK CAPITAL LETTER TAU
|
||
0x55: 0x03A5, # GREEK CAPITAL LETTER UPSILON
|
||
0x56: 0x03C2, # GREEK SMALL LETTER FINAL SIGMA
|
||
0x57: 0x03A9, # GREEK CAPITAL LETTER OMEGA
|
||
0x58: 0x039E, # GREEK CAPITAL LETTER XI
|
||
0x59: 0x03A8, # GREEK CAPITAL LETTER PSI
|
||
0x5A: 0x0396, # GREEK CAPITAL LETTER ZETA
|
||
0x5B: 0x005B, # LEFT SQUARE BRACKET
|
||
0x5C: 0x2234, # THEREFORE
|
||
0x5D: 0x005D, # RIGHT SQUARE BRACKET
|
||
0x5E: 0x22A5, # UP TACK
|
||
0x5F: 0x005F, # LOW LINE
|
||
0x60: 0xF8E5, # RADICAL EXTENDER (Apple PUA)
|
||
0x61: 0x03B1, # GREEK SMALL LETTER ALPHA
|
||
0x62: 0x03B2, # GREEK SMALL LETTER BETA
|
||
0x63: 0x03C7, # GREEK SMALL LETTER CHI
|
||
0x64: 0x03B4, # GREEK SMALL LETTER DELTA
|
||
0x65: 0x03B5, # GREEK SMALL LETTER EPSILON
|
||
0x66: 0x03C6, # GREEK SMALL LETTER PHI
|
||
0x67: 0x03B3, # GREEK SMALL LETTER GAMMA
|
||
0x68: 0x03B7, # GREEK SMALL LETTER ETA
|
||
0x69: 0x03B9, # GREEK SMALL LETTER IOTA
|
||
0x6A: 0x03D5, # GREEK PHI SYMBOL
|
||
0x6B: 0x03BA, # GREEK SMALL LETTER KAPPA
|
||
0x6C: 0x03BB, # GREEK SMALL LETTER LAMDA
|
||
0x6D: 0x03BC, # GREEK SMALL LETTER MU
|
||
0x6E: 0x03BD, # GREEK SMALL LETTER NU
|
||
0x6F: 0x03BF, # GREEK SMALL LETTER OMICRON
|
||
0x70: 0x03C0, # GREEK SMALL LETTER PI
|
||
0x71: 0x03B8, # GREEK SMALL LETTER THETA
|
||
0x72: 0x03C1, # GREEK SMALL LETTER RHO
|
||
0x73: 0x03C3, # GREEK SMALL LETTER SIGMA
|
||
0x74: 0x03C4, # GREEK SMALL LETTER TAU
|
||
0x75: 0x03C5, # GREEK SMALL LETTER UPSILON
|
||
0x76: 0x03D6, # GREEK PI SYMBOL
|
||
0x77: 0x03C9, # GREEK SMALL LETTER OMEGA
|
||
0x78: 0x03BE, # GREEK SMALL LETTER XI
|
||
0x79: 0x03C8, # GREEK SMALL LETTER PSI
|
||
0x7A: 0x03B6, # GREEK SMALL LETTER ZETA
|
||
0x7B: 0x007B, # LEFT CURLY BRACKET
|
||
0x7C: 0x007C, # VERTICAL LINE
|
||
0x7D: 0x007D, # RIGHT CURLY BRACKET
|
||
0x7E: 0x223C, # TILDE OPERATOR
|
||
0xA0: 0x20AC, # EURO SIGN
|
||
0xA1: 0x03D2, # GREEK UPSILON WITH HOOK SYMBOL
|
||
0xA2: 0x2032, # PRIME
|
||
0xA3: 0x2264, # LESS-THAN OR EQUAL TO
|
||
0xA4: 0x2044, # FRACTION SLASH
|
||
0xA5: 0x221E, # INFINITY
|
||
0xA6: 0x0192, # LATIN SMALL LETTER F WITH HOOK
|
||
0xA7: 0x2663, # BLACK CLUB SUIT
|
||
0xA8: 0x2666, # BLACK DIAMOND SUIT
|
||
0xA9: 0x2665, # BLACK HEART SUIT
|
||
0xAA: 0x2660, # BLACK SPADE SUIT
|
||
0xAB: 0x2194, # LEFT RIGHT ARROW
|
||
0xAC: 0x2190, # LEFTWARDS ARROW
|
||
0xAD: 0x2191, # UPWARDS ARROW
|
||
0xAE: 0x2192, # RIGHTWARDS ARROW
|
||
0xAF: 0x2193, # DOWNWARDS ARROW
|
||
0xB0: 0x00B0, # DEGREE SIGN
|
||
0xB1: 0x00B1, # PLUS-MINUS SIGN
|
||
0xB2: 0x2033, # DOUBLE PRIME
|
||
0xB3: 0x2265, # GREATER-THAN OR EQUAL TO
|
||
0xB4: 0x00D7, # MULTIPLICATION SIGN
|
||
0xB5: 0x221D, # PROPORTIONAL TO
|
||
0xB6: 0x2202, # PARTIAL DIFFERENTIAL
|
||
0xB7: 0x2022, # BULLET
|
||
0xB8: 0x00F7, # DIVISION SIGN
|
||
0xB9: 0x2260, # NOT EQUAL TO
|
||
0xBA: 0x2261, # IDENTICAL TO
|
||
0xBB: 0x2248, # ALMOST EQUAL TO
|
||
0xBC: 0x2026, # HORIZONTAL ELLIPSIS
|
||
0xBD: 0x23D0, # VERTICAL LINE EXTENSION
|
||
0xBE: 0x23AF, # HORIZONTAL LINE EXTENSION
|
||
0xBF: 0x21B5, # DOWNWARDS ARROW WITH CORNER LEFTWARDS
|
||
0xC0: 0x2135, # ALEF SYMBOL
|
||
0xC1: 0x2111, # BLACK-LETTER CAPITAL I
|
||
0xC2: 0x211C, # BLACK-LETTER CAPITAL R
|
||
0xC3: 0x2118, # SCRIPT CAPITAL P (Weierstrass)
|
||
0xC4: 0x2297, # CIRCLED TIMES
|
||
0xC5: 0x2295, # CIRCLED PLUS
|
||
0xC6: 0x2205, # EMPTY SET
|
||
0xC7: 0x2229, # INTERSECTION
|
||
0xC8: 0x222A, # UNION
|
||
0xC9: 0x2283, # SUPERSET OF
|
||
0xCA: 0x2287, # SUPERSET OF OR EQUAL TO
|
||
0xCB: 0x2284, # NOT A SUBSET OF
|
||
0xCC: 0x2282, # SUBSET OF
|
||
0xCD: 0x2286, # SUBSET OF OR EQUAL TO
|
||
0xCE: 0x2208, # ELEMENT OF
|
||
0xCF: 0x2209, # NOT AN ELEMENT OF
|
||
0xD0: 0x2220, # ANGLE
|
||
0xD1: 0x2207, # NABLA
|
||
0xD2: 0x00AE, # REGISTERED SIGN (also at 0xE2)
|
||
0xD3: 0x00A9, # COPYRIGHT SIGN (also at 0xE3)
|
||
0xD4: 0x2122, # TRADE MARK SIGN (also at 0xE4)
|
||
0xD5: 0x220F, # N-ARY PRODUCT
|
||
0xD6: 0x221A, # SQUARE ROOT
|
||
0xD7: 0x22C5, # DOT OPERATOR
|
||
0xD8: 0x00AC, # NOT SIGN
|
||
0xD9: 0x2227, # LOGICAL AND
|
||
0xDA: 0x2228, # LOGICAL OR
|
||
0xDB: 0x21D4, # LEFT RIGHT DOUBLE ARROW
|
||
0xDC: 0x21D0, # LEFTWARDS DOUBLE ARROW
|
||
0xDD: 0x21D1, # UPWARDS DOUBLE ARROW
|
||
0xDE: 0x21D2, # RIGHTWARDS DOUBLE ARROW
|
||
0xDF: 0x21D3, # DOWNWARDS DOUBLE ARROW
|
||
0xE0: 0x25CA, # LOZENGE
|
||
0xE1: 0x3008, # LEFT ANGLE BRACKET
|
||
0xE2: 0x00AE, # REGISTERED SIGN (duplicate)
|
||
0xE3: 0x00A9, # COPYRIGHT SIGN (duplicate)
|
||
0xE4: 0x2122, # TRADE MARK SIGN (duplicate)
|
||
0xE5: 0x2211, # N-ARY SUMMATION
|
||
0xE6: 0x239B, # LEFT PARENTHESIS UPPER HOOK
|
||
0xE7: 0x239C, # LEFT PARENTHESIS EXTENSION
|
||
0xE8: 0x239D, # LEFT PARENTHESIS LOWER HOOK
|
||
0xE9: 0x23A1, # LEFT SQUARE BRACKET UPPER CORNER
|
||
0xEA: 0x23A2, # LEFT SQUARE BRACKET EXTENSION
|
||
0xEB: 0x23A3, # LEFT SQUARE BRACKET LOWER CORNER
|
||
0xEC: 0x23A7, # LEFT CURLY BRACKET UPPER HOOK
|
||
0xED: 0x23A8, # LEFT CURLY BRACKET MIDDLE PIECE
|
||
0xEE: 0x23A9, # LEFT CURLY BRACKET LOWER HOOK
|
||
0xEF: 0x23AA, # CURLY BRACKET EXTENSION
|
||
0xF0: 0xF8FF, # Apple logo (Apple PUA)
|
||
0xF1: 0x3009, # RIGHT ANGLE BRACKET
|
||
0xF2: 0x222B, # INTEGRAL
|
||
0xF3: 0x2320, # TOP HALF INTEGRAL
|
||
0xF4: 0x23AE, # INTEGRAL EXTENSION
|
||
0xF5: 0x2321, # BOTTOM HALF INTEGRAL
|
||
0xF6: 0x239E, # RIGHT PARENTHESIS UPPER HOOK
|
||
0xF7: 0x239F, # RIGHT PARENTHESIS EXTENSION
|
||
0xF8: 0x23A0, # RIGHT PARENTHESIS LOWER HOOK
|
||
0xF9: 0x23A4, # RIGHT SQUARE BRACKET UPPER CORNER
|
||
0xFA: 0x23A5, # RIGHT SQUARE BRACKET EXTENSION
|
||
0xFB: 0x23A6, # RIGHT SQUARE BRACKET LOWER CORNER
|
||
0xFC: 0x23AB, # RIGHT CURLY BRACKET UPPER HOOK
|
||
0xFD: 0x23AC, # RIGHT CURLY BRACKET MIDDLE PIECE
|
||
0xFE: 0x23AD, # RIGHT CURLY BRACKET LOWER HOOK
|
||
0xFF: 0x23AF, # HORIZONTAL LINE EXTENSION (duplicate of 0xBE)
|
||
}
|
||
|
||
# Some codepoints in the Apple Symbol table map to the same Unicode value
|
||
# (e.g., 0xD2 and 0xE2 both map to U+00AE). When building the reverse
|
||
# mapping we can't distinguish them, but the font has separate glyphs at
|
||
# both PUA positions. For the duplicates (0xD2/0xE2, 0xD3/0xE3, 0xD4/0xE4,
|
||
# 0xBE/0xFF), textutil seems to always produce the Unicode codepoint, so
|
||
# we'll map it to the lower byte position (0xD2, 0xD3, 0xD4, 0xBE).
|
||
# If the higher position glyph is needed, the font's fallback will handle it.
|
||
|
||
|
||
def build_unicode_to_pua_map():
|
||
"""Build the reverse mapping: Unicode codepoint → PUA codepoint (U+F0xx).
|
||
|
||
For each byte value in the Symbol encoding, the font stores its glyph at
|
||
U+F000 + byte. textutil converted byte → Unicode via the Symbol table.
|
||
We reverse that: Unicode → U+F000 + byte.
|
||
"""
|
||
unicode_to_pua = {}
|
||
for byte_val, unicode_cp in APPLE_SYMBOL_BYTE_TO_UNICODE.items():
|
||
pua_cp = 0xF000 + byte_val
|
||
# For duplicates, prefer the lower byte (first occurrence wins)
|
||
if unicode_cp not in unicode_to_pua:
|
||
unicode_to_pua[unicode_cp] = pua_cp
|
||
# Also handle U+F87F which appears in some files — this is Apple PUA for
|
||
# the "delete" character (byte 0x7F), which isn't in the standard Symbol
|
||
# table but textutil sometimes emits it
|
||
unicode_to_pua[0xF87F] = 0xF07F
|
||
# U+F8E5 (radical extender) maps to byte 0x60
|
||
unicode_to_pua[0xF8E5] = 0xF060
|
||
return unicode_to_pua
|
||
|
||
|
||
def find_inben_classes(html: str) -> set:
|
||
"""Find all CSS class names that reference Inben* fonts."""
|
||
# Match class definitions in <style> like: p.p29 {...font: 18.0px Inbenr...}
|
||
classes = set()
|
||
for m in re.finditer(r"p\.(p\d+)\s*\{[^}]*font:\s*[^}]*Inben[a-z]*[^}]*\}", html):
|
||
classes.add(m.group(1))
|
||
return classes
|
||
|
||
|
||
def find_inben_span_classes(html: str) -> set:
|
||
"""Find all span CSS class names that reference Inben* fonts."""
|
||
classes = set()
|
||
for m in re.finditer(
|
||
r"span\.(s\d+)\s*\{[^}]*font:\s*[^}]*Inben[a-z]*[^}]*\}", html
|
||
):
|
||
classes.add(m.group(1))
|
||
return classes
|
||
|
||
|
||
def remap_text(text: str, unicode_to_pua: dict) -> str:
|
||
"""Remap Symbol-encoded Unicode codepoints to PUA codepoints."""
|
||
result = []
|
||
for ch in text:
|
||
cp = ord(ch)
|
||
if cp in unicode_to_pua:
|
||
result.append(chr(unicode_to_pua[cp]))
|
||
else:
|
||
result.append(ch)
|
||
return "".join(result)
|
||
|
||
|
||
def fix_bengali_encoding(html: str) -> str:
|
||
"""Fix Bengali font encoding in the HTML by remapping codepoints.
|
||
|
||
Strategy: We can't use BeautifulSoup because it would re-serialize the
|
||
entire HTML and potentially break formatting. Instead, we use regex to
|
||
find <p> and <span> tags with Bengali font classes and remap only their
|
||
text content, preserving all HTML tags and attributes inside them.
|
||
"""
|
||
unicode_to_pua = build_unicode_to_pua_map()
|
||
inben_p_classes = find_inben_classes(html)
|
||
inben_s_classes = find_inben_span_classes(html)
|
||
|
||
if not inben_p_classes and not inben_s_classes:
|
||
print("WARNING: No Inben* font classes found in CSS!")
|
||
return html
|
||
|
||
print(f"Found Bengali font <p> classes: {sorted(inben_p_classes)}")
|
||
print(f"Found Bengali font <span> classes: {sorted(inben_s_classes)}")
|
||
|
||
# Build a regex pattern that matches <p class="pXX"> ... </p> for Bengali classes
|
||
# We need to handle nested tags (like <br>, <span>, <b>, <i>) inside the <p>.
|
||
# The approach: match the opening tag, then remap characters in the content
|
||
# while preserving HTML tags.
|
||
|
||
p_class_pattern = "|".join(re.escape(c) for c in sorted(inben_p_classes))
|
||
s_class_pattern = (
|
||
"|".join(re.escape(c) for c in sorted(inben_s_classes))
|
||
if inben_s_classes
|
||
else None
|
||
)
|
||
|
||
total_remapped = 0
|
||
|
||
def remap_tag_content(m):
|
||
"""Remap text content inside a tag, preserving inner HTML tags."""
|
||
nonlocal total_remapped
|
||
full_match = m.group(0)
|
||
opening_tag = m.group(1)
|
||
inner = m.group(2)
|
||
closing_tag = m.group(3)
|
||
|
||
# Remap only text nodes (not HTML tags)
|
||
def remap_text_nodes(inner_html):
|
||
nonlocal total_remapped
|
||
parts = re.split(r"(<[^>]+>)", inner_html)
|
||
result = []
|
||
for part in parts:
|
||
if part.startswith("<"):
|
||
# HTML tag — preserve as-is
|
||
result.append(part)
|
||
else:
|
||
# Text node — remap codepoints
|
||
remapped = remap_text(part, unicode_to_pua)
|
||
if remapped != part:
|
||
total_remapped += sum(
|
||
1 for a, b in zip(part, remapped) if a != b
|
||
)
|
||
result.append(remapped)
|
||
return "".join(result)
|
||
|
||
new_inner = remap_text_nodes(inner)
|
||
return f"{opening_tag}{new_inner}{closing_tag}"
|
||
|
||
# Fix <p class="pXX"> tags
|
||
if p_class_pattern:
|
||
pattern = re.compile(
|
||
rf"(<p\s+class=\"(?:{p_class_pattern})\"[^>]*>)(.*?)(</p>)",
|
||
re.DOTALL,
|
||
)
|
||
html = pattern.sub(remap_tag_content, html)
|
||
|
||
# Fix <span class="sXX"> tags with Bengali fonts
|
||
if s_class_pattern:
|
||
pattern = re.compile(
|
||
rf"(<span\s+class=\"(?:{s_class_pattern})\"[^>]*>)(.*?)(</span>)",
|
||
re.DOTALL,
|
||
)
|
||
html = pattern.sub(remap_tag_content, html)
|
||
|
||
print(f"Remapped {total_remapped} characters")
|
||
return html
|
||
|
||
|
||
def main():
|
||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
html_path = os.path.join(base_dir, "public", "html", "cc.html")
|
||
|
||
if not os.path.exists(html_path):
|
||
print(f"ERROR: {html_path} not found!")
|
||
sys.exit(1)
|
||
|
||
print(f"Reading {html_path}...")
|
||
with open(html_path, "r", encoding="utf-8") as f:
|
||
html = f.read()
|
||
|
||
print(f"File size: {len(html):,} bytes")
|
||
print()
|
||
|
||
# --- Verify the font files exist and have PUA mappings ---
|
||
fonts_dir = os.path.join(base_dir, "public", "fonts")
|
||
for font_name in ["inbenr11.ttf", "inbeni11.ttf", "inbenb11.ttf", "inbeno11.ttf"]:
|
||
font_path = os.path.join(fonts_dir, font_name)
|
||
if os.path.exists(font_path):
|
||
print(f" ✓ Font found: {font_name}")
|
||
else:
|
||
print(f" ✗ Font MISSING: {font_name}")
|
||
|
||
print()
|
||
|
||
# --- Fix the encoding ---
|
||
fixed_html = fix_bengali_encoding(html)
|
||
|
||
# --- Verify the fix ---
|
||
unicode_to_pua = build_unicode_to_pua_map()
|
||
|
||
# Count remaining Symbol-encoded chars in Bengali paragraphs
|
||
inben_classes = find_inben_classes(fixed_html)
|
||
remaining = 0
|
||
for m in re.finditer(
|
||
rf'<p\s+class="(?:{"|".join(inben_classes)})"[^>]*>(.*?)</p>',
|
||
fixed_html,
|
||
re.DOTALL,
|
||
):
|
||
inner = re.sub(r"<[^>]+>", "", m.group(1)) # strip tags
|
||
for ch in inner:
|
||
cp = ord(ch)
|
||
if cp in unicode_to_pua and not (0xF000 <= cp <= 0xF0FF):
|
||
remaining += 1
|
||
|
||
pua_count = 0
|
||
for m in re.finditer(
|
||
rf'<p\s+class="(?:{"|".join(inben_classes)})"[^>]*>(.*?)</p>',
|
||
fixed_html,
|
||
re.DOTALL,
|
||
):
|
||
inner = re.sub(r"<[^>]+>", "", m.group(1))
|
||
for ch in inner:
|
||
cp = ord(ch)
|
||
if 0xF000 <= cp <= 0xF0FF:
|
||
pua_count += 1
|
||
|
||
print(f"\nVerification:")
|
||
print(f" PUA characters (correct): {pua_count:,}")
|
||
print(f" Remaining Symbol-encoded (incorrect): {remaining:,}")
|
||
|
||
if remaining > 0:
|
||
print(f"\n WARNING: {remaining} characters could not be remapped!")
|
||
print(" These may be characters outside the Symbol encoding range.")
|
||
|
||
# --- Write the fixed HTML ---
|
||
print(f"\nWriting fixed HTML to {html_path}...")
|
||
with open(html_path, "w", encoding="utf-8") as f:
|
||
f.write(fixed_html)
|
||
|
||
print("Done!")
|
||
|
||
# --- Quick visual check ---
|
||
print("\n--- Sample Bengali text (first 3 paragraphs) ---")
|
||
count = 0
|
||
for m in re.finditer(
|
||
rf'<p\s+class="(?:{"|".join(inben_classes)})"[^>]*>(.*?)</p>',
|
||
fixed_html,
|
||
re.DOTALL,
|
||
):
|
||
inner = re.sub(r"<[^>]+>", "", m.group(1)).strip()
|
||
if inner and len(inner) > 10:
|
||
# Show hex of first few chars to verify they're in PUA range
|
||
hex_preview = " ".join(f"U+{ord(c):04X}" for c in inner[:10])
|
||
print(f" Text: {inner[:60]}")
|
||
print(f" Hex: {hex_preview}")
|
||
print()
|
||
count += 1
|
||
if count >= 3:
|
||
break
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|