336 lines
10 KiB
Python
336 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""Extract glossary entries from glossary.html into structured JSON.
|
|
|
|
The glossary HTML contains:
|
|
1. A main consolidated glossary (p3/p4 class entries, ~2500 entries)
|
|
Format: <span class="s1">Term—</span>Definition text
|
|
Or: Term—<span class="s3">Definition text</span>
|
|
2. Book-specific glossaries (p10/p25/p26/p22 etc.) that are subsets
|
|
|
|
We extract ALL unique entries, merging duplicates by preferring the longer
|
|
definition. Output goes to app/lib/glossary/index.json.
|
|
|
|
Usage:
|
|
python3 scripts/glossary_extract.py
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
from bs4 import BeautifulSoup, Tag
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ScaGoudy → IAST Unicode mapping (shared with other extract scripts)
|
|
# ---------------------------------------------------------------------------
|
|
SCAGOUDY_TO_IAST = {
|
|
"ä": "ā",
|
|
"Ä": "Ā",
|
|
"å": "ṛ",
|
|
"Å": "Ṛ",
|
|
"ñ": "ṣ",
|
|
"Ñ": "Ṣ",
|
|
"ö": "ṭ",
|
|
"Ö": "Ṭ",
|
|
"ë": "ṇ",
|
|
"Ë": "Ṇ",
|
|
"ò": "ḍ",
|
|
"Ò": "Ḍ",
|
|
"ç": "ś",
|
|
"Ç": "Ś",
|
|
"ì": "ṅ",
|
|
"ï": "ñ",
|
|
"ü": "ū",
|
|
"Ü": "Ū",
|
|
"é": "ī",
|
|
"É": "Ī",
|
|
"à": "ṁ",
|
|
"ù": "ḥ",
|
|
"è": "ṝ", # long vocalic r (U+1E5D)
|
|
"û": "ṁl", # anusvara + l (ScaGoudy ligature glyph)
|
|
"ÿ": "ḷ", # vocalic l (U+1E37)
|
|
}
|
|
|
|
|
|
def decode_scagoudy(text: str) -> str:
|
|
"""Convert ScaGoudy-encoded text to proper IAST Unicode."""
|
|
for src, dst in SCAGOUDY_TO_IAST.items():
|
|
text = text.replace(src, dst)
|
|
return text
|
|
|
|
|
|
def decode_html_content(tag: Tag) -> str:
|
|
"""Extract inner HTML from a tag, decode ScaGoudy, preserve <i>/<b> formatting."""
|
|
inner = "".join(str(child) for child in tag.children)
|
|
inner = decode_scagoudy(inner)
|
|
# Remove span tags with CSS classes (RTF conversion artifacts)
|
|
inner = re.sub(r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner)
|
|
inner = re.sub(r"\s+", " ", inner).strip()
|
|
return inner
|
|
|
|
|
|
def decode_text(tag: Tag) -> str:
|
|
"""Extract plain text from a tag and decode ScaGoudy."""
|
|
text = tag.get_text()
|
|
return decode_scagoudy(text).strip()
|
|
|
|
|
|
def get_css_class(tag: Tag) -> str:
|
|
"""Get the primary CSS class of a tag."""
|
|
classes = tag.get("class", [])
|
|
return classes[0] if classes else ""
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Entry classes
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Main glossary entry classes (consolidated glossary at the top)
|
|
MAIN_ENTRY_CLASSES = frozenset(["p3", "p4"])
|
|
|
|
# Book-specific glossary entry classes (smaller per-book glossaries below)
|
|
BOOK_ENTRY_CLASSES = frozenset(["p10", "p22", "p24", "p25", "p26", "p27", "p19", "p34"])
|
|
|
|
# Letter heading classes (A, B, C...) — skip
|
|
LETTER_HEADING_CLASSES = frozenset(
|
|
["p9", "p12", "p14", "p15", "p23", "p29", "p30", "p31"]
|
|
)
|
|
|
|
# Section heading classes (book titles, "ABOUT THE AUTHOR", etc.) — skip
|
|
SECTION_HEADING_CLASSES = frozenset(["p7", "p17", "p18", "p2"])
|
|
|
|
# Skip classes (whitespace, decorative, front matter)
|
|
SKIP_CLASSES = frozenset(["p1", "p6", "p8", "p11", "p13", "p16", "p28", "p32", "p33"])
|
|
|
|
|
|
def parse_entry_p3(tag: Tag):
|
|
"""Parse a main glossary entry (p3/p4 class).
|
|
|
|
Format: <span class="s1">Term—</span>Definition
|
|
Or: <span class="s1">Term</span>—Definition (sometimes the dash is outside)
|
|
Or: Term—<span class="s3">definition</span> (p4 class)
|
|
|
|
Returns (term, definition_html) or None.
|
|
"""
|
|
html = decode_html_content(tag)
|
|
if not html:
|
|
return None
|
|
|
|
# The term and definition are separated by an em-dash (—)
|
|
# Sometimes there's an italic dash: <i>—</i>
|
|
# Normalize italic dashes
|
|
html = re.sub(r"<i>\s*—\s*</i>", "—", html)
|
|
|
|
# Split on the FIRST em-dash
|
|
if "—" not in html:
|
|
return None
|
|
|
|
idx = html.index("—")
|
|
term_part = html[:idx].strip()
|
|
definition_part = html[idx + 1 :].strip()
|
|
|
|
# Clean up the term: strip HTML tags, trailing/leading punctuation
|
|
term = re.sub(r"<[^>]+>", "", term_part).strip()
|
|
term = term.strip("., ;:")
|
|
|
|
if not term or not definition_part:
|
|
return None
|
|
|
|
return (term, definition_part)
|
|
|
|
|
|
def parse_entry_book(tag: Tag):
|
|
"""Parse a book-specific glossary entry (p10/p25/p26 etc.).
|
|
|
|
Format: <i>Term</i>—definition
|
|
Or: <b>Term</b>—definition
|
|
Or: Term—definition (plain text)
|
|
|
|
Returns (term, definition_html) or None.
|
|
"""
|
|
html = decode_html_content(tag)
|
|
if not html:
|
|
return None
|
|
|
|
# Normalize italic dashes
|
|
html = re.sub(r"<i>\s*—\s*</i>", "—", html)
|
|
|
|
if "—" not in html:
|
|
return None
|
|
|
|
idx = html.index("—")
|
|
term_part = html[:idx].strip()
|
|
definition_part = html[idx + 1 :].strip()
|
|
|
|
term = re.sub(r"<[^>]+>", "", term_part).strip()
|
|
term = term.strip("., ;:")
|
|
|
|
if not term or not definition_part:
|
|
return None
|
|
|
|
return (term, definition_part)
|
|
|
|
|
|
def strip_html(html: str) -> str:
|
|
"""Strip all HTML tags from a string, returning plain text."""
|
|
return re.sub(r"<[^>]+>", "", html).strip()
|
|
|
|
|
|
def extract_glossary(soup: BeautifulSoup):
|
|
"""Extract all glossary entries from the parsed HTML.
|
|
|
|
Returns a dict: { normalized_term: { "term": str, "definition": str, "definition_html": str } }
|
|
Duplicates are merged by keeping the longer definition.
|
|
"""
|
|
entries = {}
|
|
|
|
for p in soup.find_all("p"):
|
|
cls = get_css_class(p)
|
|
|
|
# Skip non-entry paragraphs
|
|
if cls in LETTER_HEADING_CLASSES:
|
|
continue
|
|
if cls in SECTION_HEADING_CLASSES:
|
|
continue
|
|
if cls in SKIP_CLASSES:
|
|
continue
|
|
|
|
parsed = None
|
|
|
|
if cls in MAIN_ENTRY_CLASSES:
|
|
parsed = parse_entry_p3(p)
|
|
elif cls in BOOK_ENTRY_CLASSES:
|
|
parsed = parse_entry_book(p)
|
|
else:
|
|
# Try parsing unknown classes as potential entries
|
|
text = decode_text(p)
|
|
if "—" in text and len(text) > 5:
|
|
parsed = parse_entry_book(p)
|
|
|
|
if parsed is None:
|
|
continue
|
|
|
|
term, definition_html = parsed
|
|
definition_plain = strip_html(definition_html)
|
|
|
|
# Normalize the term for deduplication (lowercase, strip diacritics for matching)
|
|
norm_key = term.lower().strip()
|
|
|
|
if norm_key in entries:
|
|
# Keep the longer definition
|
|
existing = entries[norm_key]
|
|
if len(definition_plain) > len(strip_html(existing["definition_html"])):
|
|
entries[norm_key] = {
|
|
"term": term,
|
|
"definition": definition_plain,
|
|
"definition_html": definition_html,
|
|
}
|
|
else:
|
|
entries[norm_key] = {
|
|
"term": term,
|
|
"definition": definition_plain,
|
|
"definition_html": definition_html,
|
|
}
|
|
|
|
return entries
|
|
|
|
|
|
def build_letter_index(entries: list) -> dict:
|
|
"""Group entries by their first letter for alphabetical navigation."""
|
|
index = {}
|
|
for entry in entries:
|
|
term = entry["term"]
|
|
# Get the first real letter (skip diacritics for grouping)
|
|
first = term[0].upper() if term else "?"
|
|
# Normalize accented first letters to base letter for grouping
|
|
# Ā→A, Ś→S, etc.
|
|
base_map = {
|
|
"Ā": "A",
|
|
"Ä": "A",
|
|
"Ī": "I",
|
|
"É": "I",
|
|
"Ū": "U",
|
|
"Ü": "U",
|
|
"Ṛ": "R",
|
|
"Ṝ": "R",
|
|
"Ś": "S",
|
|
"Ç": "S",
|
|
"Ṣ": "S",
|
|
"Ñ": "N",
|
|
"Ṇ": "N",
|
|
"Ṅ": "N",
|
|
"Ṭ": "T",
|
|
"Ḍ": "D",
|
|
"Ḥ": "H",
|
|
}
|
|
letter = base_map.get(first, first)
|
|
if letter not in index:
|
|
index[letter] = []
|
|
index[letter].append(entry)
|
|
return index
|
|
|
|
|
|
def main():
|
|
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
html_path = os.path.join(base_dir, "public", "html", "glossary.html")
|
|
output_path = os.path.join(base_dir, "lib", "jsons", "glossary", "index.json")
|
|
|
|
if not os.path.exists(html_path):
|
|
print(f"ERROR: {html_path} not found!")
|
|
sys.exit(1)
|
|
|
|
print(f"Reading {html_path}...")
|
|
with open(html_path, "r", encoding="utf-8") as f:
|
|
soup = BeautifulSoup(f, "html.parser")
|
|
|
|
print("Extracting glossary entries...")
|
|
entries_dict = extract_glossary(soup)
|
|
|
|
# Convert to sorted list
|
|
entries_list = sorted(entries_dict.values(), key=lambda e: e["term"].lower())
|
|
|
|
print(f"Extracted {len(entries_list)} unique glossary entries")
|
|
|
|
# Build letter index
|
|
letter_index = build_letter_index(entries_list)
|
|
|
|
# Build output JSON
|
|
output = {
|
|
"title": "Glossary of Vedic Terms",
|
|
"description": "A comprehensive glossary of Sanskrit and Vedic terms from the books of His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda.",
|
|
"totalEntries": len(entries_list),
|
|
"letters": sorted(letter_index.keys()),
|
|
"entries": entries_list,
|
|
}
|
|
|
|
# Write JSON
|
|
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
|
with open(output_path, "w", encoding="utf-8") as f:
|
|
json.dump(output, f, ensure_ascii=False, indent=2)
|
|
print(f"Written: {output_path}")
|
|
|
|
# Print summary
|
|
print(f"\n--- Summary ---")
|
|
print(f"Total unique entries: {len(entries_list)}")
|
|
print(f"Letters: {', '.join(sorted(letter_index.keys()))}")
|
|
print(f"\nEntries per letter:")
|
|
for letter in sorted(letter_index.keys()):
|
|
count = len(letter_index[letter])
|
|
print(f" {letter}: {count}")
|
|
|
|
# Show first few entries as a sample
|
|
print(f"\n--- Sample entries ---")
|
|
for entry in entries_list[:5]:
|
|
defn = entry["definition"][:80]
|
|
print(f" {entry['term']}: {defn}...")
|
|
|
|
print(f"\n--- Last few entries ---")
|
|
for entry in entries_list[-3:]:
|
|
defn = entry["definition"][:80]
|
|
print(f" {entry['term']}: {defn}...")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|