mongo fix
This commit is contained in:
gosangam
@@ -0,0 +1,335 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract glossary entries from glossary.html into structured JSON.
|
||||
|
||||
The glossary HTML contains:
|
||||
1. A main consolidated glossary (p3/p4 class entries, ~2500 entries)
|
||||
Format: <span class="s1">Term—</span>Definition text
|
||||
Or: Term—<span class="s3">Definition text</span>
|
||||
2. Book-specific glossaries (p10/p25/p26/p22 etc.) that are subsets
|
||||
|
||||
We extract ALL unique entries, merging duplicates by preferring the longer
|
||||
definition. Output goes to app/lib/glossary/index.json.
|
||||
|
||||
Usage:
|
||||
python3 scripts/glossary_extract.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ScaGoudy → IAST Unicode mapping (shared with other extract scripts)
|
||||
# ---------------------------------------------------------------------------
|
||||
SCAGOUDY_TO_IAST = {
|
||||
"ä": "ā",
|
||||
"Ä": "Ā",
|
||||
"å": "ṛ",
|
||||
"Å": "Ṛ",
|
||||
"ñ": "ṣ",
|
||||
"Ñ": "Ṣ",
|
||||
"ö": "ṭ",
|
||||
"Ö": "Ṭ",
|
||||
"ë": "ṇ",
|
||||
"Ë": "Ṇ",
|
||||
"ò": "ḍ",
|
||||
"Ò": "Ḍ",
|
||||
"ç": "ś",
|
||||
"Ç": "Ś",
|
||||
"ì": "ṅ",
|
||||
"ï": "ñ",
|
||||
"ü": "ū",
|
||||
"Ü": "Ū",
|
||||
"é": "ī",
|
||||
"É": "Ī",
|
||||
"à": "ṁ",
|
||||
"ù": "ḥ",
|
||||
"è": "ṝ", # long vocalic r (U+1E5D)
|
||||
"û": "ṁl", # anusvara + l (ScaGoudy ligature glyph)
|
||||
"ÿ": "ḷ", # vocalic l (U+1E37)
|
||||
}
|
||||
|
||||
|
||||
def decode_scagoudy(text: str) -> str:
|
||||
"""Convert ScaGoudy-encoded text to proper IAST Unicode."""
|
||||
for src, dst in SCAGOUDY_TO_IAST.items():
|
||||
text = text.replace(src, dst)
|
||||
return text
|
||||
|
||||
|
||||
def decode_html_content(tag: Tag) -> str:
|
||||
"""Extract inner HTML from a tag, decode ScaGoudy, preserve <i>/<b> formatting."""
|
||||
inner = "".join(str(child) for child in tag.children)
|
||||
inner = decode_scagoudy(inner)
|
||||
# Remove span tags with CSS classes (RTF conversion artifacts)
|
||||
inner = re.sub(r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner)
|
||||
inner = re.sub(r"\s+", " ", inner).strip()
|
||||
return inner
|
||||
|
||||
|
||||
def decode_text(tag: Tag) -> str:
|
||||
"""Extract plain text from a tag and decode ScaGoudy."""
|
||||
text = tag.get_text()
|
||||
return decode_scagoudy(text).strip()
|
||||
|
||||
|
||||
def get_css_class(tag: Tag) -> str:
|
||||
"""Get the primary CSS class of a tag."""
|
||||
classes = tag.get("class", [])
|
||||
return classes[0] if classes else ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Entry classes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Main glossary entry classes (consolidated glossary at the top)
|
||||
MAIN_ENTRY_CLASSES = frozenset(["p3", "p4"])
|
||||
|
||||
# Book-specific glossary entry classes (smaller per-book glossaries below)
|
||||
BOOK_ENTRY_CLASSES = frozenset(["p10", "p22", "p24", "p25", "p26", "p27", "p19", "p34"])
|
||||
|
||||
# Letter heading classes (A, B, C...) — skip
|
||||
LETTER_HEADING_CLASSES = frozenset(
|
||||
["p9", "p12", "p14", "p15", "p23", "p29", "p30", "p31"]
|
||||
)
|
||||
|
||||
# Section heading classes (book titles, "ABOUT THE AUTHOR", etc.) — skip
|
||||
SECTION_HEADING_CLASSES = frozenset(["p7", "p17", "p18", "p2"])
|
||||
|
||||
# Skip classes (whitespace, decorative, front matter)
|
||||
SKIP_CLASSES = frozenset(["p1", "p6", "p8", "p11", "p13", "p16", "p28", "p32", "p33"])
|
||||
|
||||
|
||||
def parse_entry_p3(tag: Tag):
|
||||
"""Parse a main glossary entry (p3/p4 class).
|
||||
|
||||
Format: <span class="s1">Term—</span>Definition
|
||||
Or: <span class="s1">Term</span>—Definition (sometimes the dash is outside)
|
||||
Or: Term—<span class="s3">definition</span> (p4 class)
|
||||
|
||||
Returns (term, definition_html) or None.
|
||||
"""
|
||||
html = decode_html_content(tag)
|
||||
if not html:
|
||||
return None
|
||||
|
||||
# The term and definition are separated by an em-dash (—)
|
||||
# Sometimes there's an italic dash: <i>—</i>
|
||||
# Normalize italic dashes
|
||||
html = re.sub(r"<i>\s*—\s*</i>", "—", html)
|
||||
|
||||
# Split on the FIRST em-dash
|
||||
if "—" not in html:
|
||||
return None
|
||||
|
||||
idx = html.index("—")
|
||||
term_part = html[:idx].strip()
|
||||
definition_part = html[idx + 1 :].strip()
|
||||
|
||||
# Clean up the term: strip HTML tags, trailing/leading punctuation
|
||||
term = re.sub(r"<[^>]+>", "", term_part).strip()
|
||||
term = term.strip("., ;:")
|
||||
|
||||
if not term or not definition_part:
|
||||
return None
|
||||
|
||||
return (term, definition_part)
|
||||
|
||||
|
||||
def parse_entry_book(tag: Tag):
|
||||
"""Parse a book-specific glossary entry (p10/p25/p26 etc.).
|
||||
|
||||
Format: <i>Term</i>—definition
|
||||
Or: <b>Term</b>—definition
|
||||
Or: Term—definition (plain text)
|
||||
|
||||
Returns (term, definition_html) or None.
|
||||
"""
|
||||
html = decode_html_content(tag)
|
||||
if not html:
|
||||
return None
|
||||
|
||||
# Normalize italic dashes
|
||||
html = re.sub(r"<i>\s*—\s*</i>", "—", html)
|
||||
|
||||
if "—" not in html:
|
||||
return None
|
||||
|
||||
idx = html.index("—")
|
||||
term_part = html[:idx].strip()
|
||||
definition_part = html[idx + 1 :].strip()
|
||||
|
||||
term = re.sub(r"<[^>]+>", "", term_part).strip()
|
||||
term = term.strip("., ;:")
|
||||
|
||||
if not term or not definition_part:
|
||||
return None
|
||||
|
||||
return (term, definition_part)
|
||||
|
||||
|
||||
def strip_html(html: str) -> str:
|
||||
"""Strip all HTML tags from a string, returning plain text."""
|
||||
return re.sub(r"<[^>]+>", "", html).strip()
|
||||
|
||||
|
||||
def extract_glossary(soup: BeautifulSoup):
|
||||
"""Extract all glossary entries from the parsed HTML.
|
||||
|
||||
Returns a dict: { normalized_term: { "term": str, "definition": str, "definition_html": str } }
|
||||
Duplicates are merged by keeping the longer definition.
|
||||
"""
|
||||
entries = {}
|
||||
|
||||
for p in soup.find_all("p"):
|
||||
cls = get_css_class(p)
|
||||
|
||||
# Skip non-entry paragraphs
|
||||
if cls in LETTER_HEADING_CLASSES:
|
||||
continue
|
||||
if cls in SECTION_HEADING_CLASSES:
|
||||
continue
|
||||
if cls in SKIP_CLASSES:
|
||||
continue
|
||||
|
||||
parsed = None
|
||||
|
||||
if cls in MAIN_ENTRY_CLASSES:
|
||||
parsed = parse_entry_p3(p)
|
||||
elif cls in BOOK_ENTRY_CLASSES:
|
||||
parsed = parse_entry_book(p)
|
||||
else:
|
||||
# Try parsing unknown classes as potential entries
|
||||
text = decode_text(p)
|
||||
if "—" in text and len(text) > 5:
|
||||
parsed = parse_entry_book(p)
|
||||
|
||||
if parsed is None:
|
||||
continue
|
||||
|
||||
term, definition_html = parsed
|
||||
definition_plain = strip_html(definition_html)
|
||||
|
||||
# Normalize the term for deduplication (lowercase, strip diacritics for matching)
|
||||
norm_key = term.lower().strip()
|
||||
|
||||
if norm_key in entries:
|
||||
# Keep the longer definition
|
||||
existing = entries[norm_key]
|
||||
if len(definition_plain) > len(strip_html(existing["definition_html"])):
|
||||
entries[norm_key] = {
|
||||
"term": term,
|
||||
"definition": definition_plain,
|
||||
"definition_html": definition_html,
|
||||
}
|
||||
else:
|
||||
entries[norm_key] = {
|
||||
"term": term,
|
||||
"definition": definition_plain,
|
||||
"definition_html": definition_html,
|
||||
}
|
||||
|
||||
return entries
|
||||
|
||||
|
||||
def build_letter_index(entries: list) -> dict:
|
||||
"""Group entries by their first letter for alphabetical navigation."""
|
||||
index = {}
|
||||
for entry in entries:
|
||||
term = entry["term"]
|
||||
# Get the first real letter (skip diacritics for grouping)
|
||||
first = term[0].upper() if term else "?"
|
||||
# Normalize accented first letters to base letter for grouping
|
||||
# Ā→A, Ś→S, etc.
|
||||
base_map = {
|
||||
"Ā": "A",
|
||||
"Ä": "A",
|
||||
"Ī": "I",
|
||||
"É": "I",
|
||||
"Ū": "U",
|
||||
"Ü": "U",
|
||||
"Ṛ": "R",
|
||||
"Ṝ": "R",
|
||||
"Ś": "S",
|
||||
"Ç": "S",
|
||||
"Ṣ": "S",
|
||||
"Ñ": "N",
|
||||
"Ṇ": "N",
|
||||
"Ṅ": "N",
|
||||
"Ṭ": "T",
|
||||
"Ḍ": "D",
|
||||
"Ḥ": "H",
|
||||
}
|
||||
letter = base_map.get(first, first)
|
||||
if letter not in index:
|
||||
index[letter] = []
|
||||
index[letter].append(entry)
|
||||
return index
|
||||
|
||||
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "glossary.html")
|
||||
output_path = os.path.join(base_dir, "app", "lib", "glossary", "index.json")
|
||||
|
||||
if not os.path.exists(html_path):
|
||||
print(f"ERROR: {html_path} not found!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Reading {html_path}...")
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
soup = BeautifulSoup(f, "html.parser")
|
||||
|
||||
print("Extracting glossary entries...")
|
||||
entries_dict = extract_glossary(soup)
|
||||
|
||||
# Convert to sorted list
|
||||
entries_list = sorted(entries_dict.values(), key=lambda e: e["term"].lower())
|
||||
|
||||
print(f"Extracted {len(entries_list)} unique glossary entries")
|
||||
|
||||
# Build letter index
|
||||
letter_index = build_letter_index(entries_list)
|
||||
|
||||
# Build output JSON
|
||||
output = {
|
||||
"title": "Glossary of Vedic Terms",
|
||||
"description": "A comprehensive glossary of Sanskrit and Vedic terms from the books of His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda.",
|
||||
"totalEntries": len(entries_list),
|
||||
"letters": sorted(letter_index.keys()),
|
||||
"entries": entries_list,
|
||||
}
|
||||
|
||||
# Write JSON
|
||||
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
||||
with open(output_path, "w", encoding="utf-8") as f:
|
||||
json.dump(output, f, ensure_ascii=False, indent=2)
|
||||
print(f"Written: {output_path}")
|
||||
|
||||
# Print summary
|
||||
print(f"\n--- Summary ---")
|
||||
print(f"Total unique entries: {len(entries_list)}")
|
||||
print(f"Letters: {', '.join(sorted(letter_index.keys()))}")
|
||||
print(f"\nEntries per letter:")
|
||||
for letter in sorted(letter_index.keys()):
|
||||
count = len(letter_index[letter])
|
||||
print(f" {letter}: {count}")
|
||||
|
||||
# Show first few entries as a sample
|
||||
print(f"\n--- Sample entries ---")
|
||||
for entry in entries_list[:5]:
|
||||
defn = entry["definition"][:80]
|
||||
print(f" {entry['term']}: {defn}...")
|
||||
|
||||
print(f"\n--- Last few entries ---")
|
||||
for entry in entries_list[-3:]:
|
||||
defn = entry["definition"][:80]
|
||||
print(f" {entry['term']}: {defn}...")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user