#!/usr/bin/env python3
"""Extract glossary entries from glossary.html into structured JSON.
The glossary HTML contains:
1. A main consolidated glossary (p3/p4 class entries, ~2500 entries)
Format: Term—Definition text
Or: Term—Definition text
2. Book-specific glossaries (p10/p25/p26/p22 etc.) that are subsets
We extract ALL unique entries, merging duplicates by preferring the longer
definition. Output goes to app/lib/glossary/index.json.
Usage:
python3 scripts/glossary_extract.py
"""
import json
import os
import re
import sys
from bs4 import BeautifulSoup, Tag
# ---------------------------------------------------------------------------
# ScaGoudy → IAST Unicode mapping (shared with other extract scripts)
# ---------------------------------------------------------------------------
SCAGOUDY_TO_IAST = {
"ä": "ā",
"Ä": "Ā",
"å": "ṛ",
"Å": "Ṛ",
"ñ": "ṣ",
"Ñ": "Ṣ",
"ö": "ṭ",
"Ö": "Ṭ",
"ë": "ṇ",
"Ë": "Ṇ",
"ò": "ḍ",
"Ò": "Ḍ",
"ç": "ś",
"Ç": "Ś",
"ì": "ṅ",
"ï": "ñ",
"ü": "ū",
"Ü": "Ū",
"é": "ī",
"É": "Ī",
"à": "ṁ",
"ù": "ḥ",
"è": "ṝ", # long vocalic r (U+1E5D)
"û": "ṁl", # anusvara + l (ScaGoudy ligature glyph)
"ÿ": "ḷ", # vocalic l (U+1E37)
}
def decode_scagoudy(text: str) -> str:
"""Convert ScaGoudy-encoded text to proper IAST Unicode."""
for src, dst in SCAGOUDY_TO_IAST.items():
text = text.replace(src, dst)
return text
def decode_html_content(tag: Tag) -> str:
"""Extract inner HTML from a tag, decode ScaGoudy, preserve / formatting."""
inner = "".join(str(child) for child in tag.children)
inner = decode_scagoudy(inner)
# Remove span tags with CSS classes (RTF conversion artifacts)
inner = re.sub(r']*class="[^"]*"[^>]*>(.*?)', r"\1", inner)
inner = re.sub(r"\s+", " ", inner).strip()
return inner
def decode_text(tag: Tag) -> str:
"""Extract plain text from a tag and decode ScaGoudy."""
text = tag.get_text()
return decode_scagoudy(text).strip()
def get_css_class(tag: Tag) -> str:
"""Get the primary CSS class of a tag."""
classes = tag.get("class", [])
return classes[0] if classes else ""
# ---------------------------------------------------------------------------
# Entry classes
# ---------------------------------------------------------------------------
# Main glossary entry classes (consolidated glossary at the top)
MAIN_ENTRY_CLASSES = frozenset(["p3", "p4"])
# Book-specific glossary entry classes (smaller per-book glossaries below)
BOOK_ENTRY_CLASSES = frozenset(["p10", "p22", "p24", "p25", "p26", "p27", "p19", "p34"])
# Letter heading classes (A, B, C...) — skip
LETTER_HEADING_CLASSES = frozenset(
["p9", "p12", "p14", "p15", "p23", "p29", "p30", "p31"]
)
# Section heading classes (book titles, "ABOUT THE AUTHOR", etc.) — skip
SECTION_HEADING_CLASSES = frozenset(["p7", "p17", "p18", "p2"])
# Skip classes (whitespace, decorative, front matter)
SKIP_CLASSES = frozenset(["p1", "p6", "p8", "p11", "p13", "p16", "p28", "p32", "p33"])
def parse_entry_p3(tag: Tag):
"""Parse a main glossary entry (p3/p4 class).
Format: Term—Definition
Or: Term—Definition (sometimes the dash is outside)
Or: Term—definition (p4 class)
Returns (term, definition_html) or None.
"""
html = decode_html_content(tag)
if not html:
return None
# The term and definition are separated by an em-dash (—)
# Sometimes there's an italic dash: —
# Normalize italic dashes
html = re.sub(r"\s*—\s*", "—", html)
# Split on the FIRST em-dash
if "—" not in html:
return None
idx = html.index("—")
term_part = html[:idx].strip()
definition_part = html[idx + 1 :].strip()
# Clean up the term: strip HTML tags, trailing/leading punctuation
term = re.sub(r"<[^>]+>", "", term_part).strip()
term = term.strip("., ;:")
if not term or not definition_part:
return None
return (term, definition_part)
def parse_entry_book(tag: Tag):
"""Parse a book-specific glossary entry (p10/p25/p26 etc.).
Format: Term—definition
Or: Term—definition
Or: Term—definition (plain text)
Returns (term, definition_html) or None.
"""
html = decode_html_content(tag)
if not html:
return None
# Normalize italic dashes
html = re.sub(r"\s*—\s*", "—", html)
if "—" not in html:
return None
idx = html.index("—")
term_part = html[:idx].strip()
definition_part = html[idx + 1 :].strip()
term = re.sub(r"<[^>]+>", "", term_part).strip()
term = term.strip("., ;:")
if not term or not definition_part:
return None
return (term, definition_part)
def strip_html(html: str) -> str:
"""Strip all HTML tags from a string, returning plain text."""
return re.sub(r"<[^>]+>", "", html).strip()
def extract_glossary(soup: BeautifulSoup):
"""Extract all glossary entries from the parsed HTML.
Returns a dict: { normalized_term: { "term": str, "definition": str, "definition_html": str } }
Duplicates are merged by keeping the longer definition.
"""
entries = {}
for p in soup.find_all("p"):
cls = get_css_class(p)
# Skip non-entry paragraphs
if cls in LETTER_HEADING_CLASSES:
continue
if cls in SECTION_HEADING_CLASSES:
continue
if cls in SKIP_CLASSES:
continue
parsed = None
if cls in MAIN_ENTRY_CLASSES:
parsed = parse_entry_p3(p)
elif cls in BOOK_ENTRY_CLASSES:
parsed = parse_entry_book(p)
else:
# Try parsing unknown classes as potential entries
text = decode_text(p)
if "—" in text and len(text) > 5:
parsed = parse_entry_book(p)
if parsed is None:
continue
term, definition_html = parsed
definition_plain = strip_html(definition_html)
# Normalize the term for deduplication (lowercase, strip diacritics for matching)
norm_key = term.lower().strip()
if norm_key in entries:
# Keep the longer definition
existing = entries[norm_key]
if len(definition_plain) > len(strip_html(existing["definition_html"])):
entries[norm_key] = {
"term": term,
"definition": definition_plain,
"definition_html": definition_html,
}
else:
entries[norm_key] = {
"term": term,
"definition": definition_plain,
"definition_html": definition_html,
}
return entries
def build_letter_index(entries: list) -> dict:
"""Group entries by their first letter for alphabetical navigation."""
index = {}
for entry in entries:
term = entry["term"]
# Get the first real letter (skip diacritics for grouping)
first = term[0].upper() if term else "?"
# Normalize accented first letters to base letter for grouping
# Ā→A, Ś→S, etc.
base_map = {
"Ā": "A",
"Ä": "A",
"Ī": "I",
"É": "I",
"Ū": "U",
"Ü": "U",
"Ṛ": "R",
"Ṝ": "R",
"Ś": "S",
"Ç": "S",
"Ṣ": "S",
"Ñ": "N",
"Ṇ": "N",
"Ṅ": "N",
"Ṭ": "T",
"Ḍ": "D",
"Ḥ": "H",
}
letter = base_map.get(first, first)
if letter not in index:
index[letter] = []
index[letter].append(entry)
return index
def main():
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
html_path = os.path.join(base_dir, "public", "html", "glossary.html")
output_path = os.path.join(base_dir, "lib", "jsons", "glossary", "index.json")
if not os.path.exists(html_path):
print(f"ERROR: {html_path} not found!")
sys.exit(1)
print(f"Reading {html_path}...")
with open(html_path, "r", encoding="utf-8") as f:
soup = BeautifulSoup(f, "html.parser")
print("Extracting glossary entries...")
entries_dict = extract_glossary(soup)
# Convert to sorted list
entries_list = sorted(entries_dict.values(), key=lambda e: e["term"].lower())
print(f"Extracted {len(entries_list)} unique glossary entries")
# Build letter index
letter_index = build_letter_index(entries_list)
# Build output JSON
output = {
"title": "Glossary of Vedic Terms",
"description": "A comprehensive glossary of Sanskrit and Vedic terms from the books of His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda.",
"totalEntries": len(entries_list),
"letters": sorted(letter_index.keys()),
"entries": entries_list,
}
# Write JSON
os.makedirs(os.path.dirname(output_path), exist_ok=True)
with open(output_path, "w", encoding="utf-8") as f:
json.dump(output, f, ensure_ascii=False, indent=2)
print(f"Written: {output_path}")
# Print summary
print(f"\n--- Summary ---")
print(f"Total unique entries: {len(entries_list)}")
print(f"Letters: {', '.join(sorted(letter_index.keys()))}")
print(f"\nEntries per letter:")
for letter in sorted(letter_index.keys()):
count = len(letter_index[letter])
print(f" {letter}: {count}")
# Show first few entries as a sample
print(f"\n--- Sample entries ---")
for entry in entries_list[:5]:
defn = entry["definition"][:80]
print(f" {entry['term']}: {defn}...")
print(f"\n--- Last few entries ---")
for entry in entries_list[-3:]:
defn = entry["definition"][:80]
print(f" {entry['term']}: {defn}...")
if __name__ == "__main__":
main()