860 lines
28 KiB
Python
860 lines
28 KiB
Python
#!/usr/bin/env python3
|
|
"""Extract The Nectar of Instruction (1976 Edition) verses from other.html into structured JSON files."""
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
from bs4 import BeautifulSoup, Tag
|
|
from indic_transliteration import sanscript
|
|
from indic_transliteration.sanscript import transliterate
|
|
|
|
# --- ScaGoudy → IAST Unicode mapping ---
|
|
SCAGOUDY_TO_IAST = {
|
|
"ä": "ā",
|
|
"Ä": "Ā",
|
|
"å": "ṛ",
|
|
"Å": "Ṛ",
|
|
"ñ": "ṣ",
|
|
"Ñ": "Ṣ",
|
|
"ö": "ṭ",
|
|
"Ö": "Ṭ",
|
|
"ë": "ṇ",
|
|
"Ë": "Ṇ",
|
|
"ò": "ḍ",
|
|
"Ò": "Ḍ",
|
|
"ç": "ś",
|
|
"Ç": "Ś",
|
|
"ì": "ṅ",
|
|
"ï": "ñ",
|
|
"ü": "ū",
|
|
"Ü": "Ū",
|
|
"é": "ī",
|
|
"É": "Ī",
|
|
"à": "ṁ",
|
|
"ù": "ḥ",
|
|
"è": "ṝ", # long vocalic r (U+1E5D)
|
|
"û": "ṁl", # anusvara + l (ScaGoudy ligature glyph)
|
|
"ÿ": "ḷ", # vocalic l (U+1E37)
|
|
}
|
|
|
|
# --- Text number to title mapping ---
|
|
TEXT_TITLES = {
|
|
1: "Text One",
|
|
2: "Text Two",
|
|
3: "Text Three",
|
|
4: "Text Four",
|
|
5: "Text Five",
|
|
6: "Text Six",
|
|
7: "Text Seven",
|
|
8: "Text Eight",
|
|
9: "Text Nine",
|
|
10: "Text Ten",
|
|
11: "Text Eleven",
|
|
}
|
|
|
|
# --- CSS class constants ---
|
|
BOOK_HEADER_CLASS = "p12"
|
|
SECTION_BOOKMARK_CLASS = "p64" # NoI section bookmarks (Front pages, Preface)
|
|
VERSE_BOOKMARK_CLASS = "p49" # NoI verse bookmarks (NoI: verse N) AND appendixes
|
|
VERSE_HEADING_CLASS = "p50"
|
|
DEVANAGARI_CLASS = "p51" # RM Devanagari — SKIP
|
|
TEXT_LABEL_CLASS = "p52"
|
|
PADA_TRANSLIT_CLASS = "p5"
|
|
SYNONYM_CLASS = "p54"
|
|
TRANSLATION_LABEL_CLASS = "p55"
|
|
TRANSLATION_TEXT_CLASS = "p56"
|
|
PURPORT_LABEL_CLASS = "p57"
|
|
PURPORT_BODY_CLASS = "p48"
|
|
QUOTED_VERSE_CLASSES = (
|
|
"p33",
|
|
"p53",
|
|
"p66",
|
|
) # Quoted Sanskrit/Bengali verses in purport
|
|
BODY_TEXT_CLASS = "p44" # Verse translations within purport
|
|
MISC_TEXT_CLASS = "p67" # Miscellaneous text (1 occurrence in NoI)
|
|
SECTION_HEADING_CLASS = "p31"
|
|
BODY_CLASSES = ("p6", "p45")
|
|
SKIP_CLASSES = ("p9", "p15", "p19", "p25", "p32", "p37", "p65")
|
|
|
|
|
|
def decode_scagoudy(text: str) -> str:
|
|
"""Convert ScaGoudy-encoded text to proper IAST Unicode."""
|
|
for src, dst in SCAGOUDY_TO_IAST.items():
|
|
text = text.replace(src, dst)
|
|
return text
|
|
|
|
|
|
def decode_html_content(tag: Tag) -> str:
|
|
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
|
|
inner = "".join(str(child) for child in tag.children)
|
|
inner = decode_scagoudy(inner)
|
|
# Remove span tags with CSS classes (artifacts from RTF conversion)
|
|
inner = re.sub(
|
|
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
|
|
)
|
|
# Clean up Apple-tab-span leftovers (tab characters)
|
|
inner = inner.replace("\t", " ")
|
|
# Normalize whitespace
|
|
inner = re.sub(r"\s+", " ", inner).strip()
|
|
return inner
|
|
|
|
|
|
def decode_text(tag: Tag) -> str:
|
|
"""Extract plain text from a tag and decode ScaGoudy."""
|
|
text = tag.get_text()
|
|
return decode_scagoudy(text).strip()
|
|
|
|
|
|
def normalize_devanagari_anusvara(text: str) -> str:
|
|
"""Replace explicit class nasals before same-class consonants with anusvāra.
|
|
|
|
Only normalize ङ् and ञ् — these are almost always written with anusvāra.
|
|
Keep ण्ड, न्त, म्ब etc. as explicit conjuncts (standard spelling).
|
|
"""
|
|
replacements = [
|
|
# ङ् before velars (क, ख, ग, घ)
|
|
(r"ङ्([कखगघ])", r"ं\1"),
|
|
# ञ् before palatals (च, छ, ज, झ)
|
|
(r"ञ्([चछजझ])", r"ं\1"),
|
|
]
|
|
for pattern, repl in replacements:
|
|
text = re.sub(pattern, repl, text)
|
|
return text
|
|
|
|
|
|
def join_virama_words(text: str) -> str:
|
|
"""Join Devanagari words where the previous word ends with virāma (्).
|
|
|
|
Handles both virāma+consonant and virāma+vowel cases.
|
|
Uses [ \\t]+ instead of \\s+ to avoid joining across newlines.
|
|
"""
|
|
# Join virāma + space + consonant (क U+0915 through ह U+0939)
|
|
text = re.sub(r"्[ \t]+([क-ह])", r"्\1", text)
|
|
|
|
# Join virāma + space + independent vowel
|
|
_VOWEL_TO_MATRA = {
|
|
"अ": "", # inherent vowel — just drop virāma and the अ
|
|
"आ": "ा",
|
|
"इ": "ि",
|
|
"ई": "ी",
|
|
"उ": "ु",
|
|
"ऊ": "ू",
|
|
"ऋ": "ृ",
|
|
"ॠ": "ॄ",
|
|
"ऌ": "ॢ",
|
|
"ॡ": "ॣ",
|
|
"ए": "े",
|
|
"ऐ": "ै",
|
|
"ओ": "ो",
|
|
"औ": "ौ",
|
|
}
|
|
|
|
def _virama_vowel_replace(m: re.Match[str]) -> str:
|
|
vowel = m.group(1)
|
|
return _VOWEL_TO_MATRA.get(vowel, vowel)
|
|
|
|
text = re.sub(r"्[ \t]+([अ-औॠॡ])", _virama_vowel_replace, text)
|
|
return text
|
|
|
|
|
|
def iast_to_devanagari(iast_text: str) -> str:
|
|
"""Convert IAST transliteration to Devanagari script."""
|
|
lines = iast_text.split("\n")
|
|
result_lines = []
|
|
for line in lines:
|
|
# Remove hyphens between words for Devanagari
|
|
clean = line.replace("-", "")
|
|
# Normalize ṁ (U+1E41) to ṃ (U+1E43) for proper anusvāra
|
|
clean = clean.replace("ṁ", "ṃ")
|
|
deva = transliterate(clean, sanscript.IAST, sanscript.DEVANAGARI)
|
|
deva = normalize_devanagari_anusvara(deva)
|
|
deva = join_virama_words(deva)
|
|
result_lines.append(deva)
|
|
return "\n".join(result_lines)
|
|
|
|
|
|
def parse_synonyms(text: str) -> list:
|
|
"""Parse synonym text into structured array.
|
|
|
|
Input format: "word—meaning; word—meaning; word—meaning"
|
|
"""
|
|
result = []
|
|
entries = text.split(";")
|
|
for entry in entries:
|
|
entry = entry.strip()
|
|
if not entry:
|
|
continue
|
|
if "—" in entry:
|
|
parts = entry.split("—", 1)
|
|
elif "--" in entry:
|
|
parts = entry.split("--", 1)
|
|
else:
|
|
continue
|
|
|
|
if len(parts) == 2:
|
|
word = parts[0].strip()
|
|
meaning = parts[1].strip()
|
|
# Clean word: remove trailing/leading punctuation
|
|
word = word.strip("., ")
|
|
if word and meaning:
|
|
result.append({"word": word, "meanings": [meaning]})
|
|
|
|
return result
|
|
|
|
|
|
def pair_padas(padas: list) -> str:
|
|
"""Pair pādas into verse lines (2 pādas per line).
|
|
|
|
A śloka has 4 pādas displayed as 2 lines. If there's an odd number,
|
|
the first pāda stands alone (usually a speaker attribution).
|
|
"""
|
|
if not padas:
|
|
return ""
|
|
|
|
lines = []
|
|
i = 0
|
|
|
|
# If odd number of pādas, first one is standalone
|
|
if len(padas) % 2 == 1:
|
|
lines.append(padas[0])
|
|
i = 1
|
|
|
|
# Pair remaining pādas
|
|
while i < len(padas) - 1:
|
|
lines.append(padas[i] + " " + padas[i + 1])
|
|
i += 2
|
|
|
|
# Handle any trailing single pāda
|
|
if i < len(padas):
|
|
lines.append(padas[i])
|
|
|
|
return "\n".join(lines)
|
|
|
|
|
|
def get_css_class(tag: Tag) -> str:
|
|
"""Get the primary CSS class of a tag."""
|
|
classes = tag.get("class", [])
|
|
return classes[0] if classes else ""
|
|
|
|
|
|
def is_empty(tag: Tag) -> bool:
|
|
"""Check if a tag is empty or whitespace-only."""
|
|
text = tag.get_text().strip()
|
|
return not text or text == "\xa0"
|
|
|
|
|
|
def is_dashed_line(text: str) -> bool:
|
|
"""Check if text is a dashed-line separator."""
|
|
clean = text.strip().replace("-", "").replace("—", "").strip()
|
|
return clean == ""
|
|
|
|
|
|
def parse_verse_bookmark(text: str) -> int | None:
|
|
"""Parse a verse bookmark like 'NoI: verse 1' through 'NoI: verse 11'.
|
|
|
|
Returns text number (1-11), or None if not a verse bookmark.
|
|
"""
|
|
text = text.strip()
|
|
m = re.match(r"NoI:\s*verse\s+(\d+)", text)
|
|
if m:
|
|
return int(m.group(1))
|
|
return None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Boundary detection
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def find_noi_boundaries(all_paragraphs: list) -> tuple[int, int]:
|
|
"""Find the start and end paragraph indices of the NoI content.
|
|
|
|
Start: <p class="p12"> containing "Nectar of Instruction" and "1976"
|
|
End: next <p class="p12"> after start (start of next book), or end of file
|
|
"""
|
|
start_idx = None
|
|
end_idx = len(all_paragraphs)
|
|
|
|
for i, p in enumerate(all_paragraphs):
|
|
cls = get_css_class(p)
|
|
if cls != BOOK_HEADER_CLASS:
|
|
continue
|
|
text = p.get_text().strip()
|
|
if "Nectar of Instruction" in text and "1976" in text:
|
|
start_idx = i
|
|
elif start_idx is not None:
|
|
# Next p12 after NoI start = end of NoI
|
|
end_idx = i
|
|
break
|
|
|
|
if start_idx is None:
|
|
print(
|
|
"ERROR: Could not find 'The Nectar of Instruction - 1976 Edition' header in other.html!"
|
|
)
|
|
sys.exit(1)
|
|
|
|
return start_idx, end_idx
|
|
|
|
|
|
def find_section_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> dict:
|
|
"""Find the key p64 bookmarks for NoI section boundaries.
|
|
|
|
Returns dict with bookmark text -> paragraph index.
|
|
"""
|
|
bookmarks = {}
|
|
for i in range(start_idx, end_idx):
|
|
p = all_paragraphs[i]
|
|
cls = get_css_class(p)
|
|
if cls == SECTION_BOOKMARK_CLASS:
|
|
text = decode_text(p)
|
|
if "NoI" in text:
|
|
bookmarks[text] = i
|
|
return bookmarks
|
|
|
|
|
|
def find_verse_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
|
|
"""Find all p49 verse bookmark tags in the NoI range.
|
|
|
|
Returns list of (paragraph_index, text_number) for actual verse bookmarks.
|
|
Also returns appendixes bookmark index separately.
|
|
"""
|
|
verse_bookmarks = []
|
|
appendixes_idx = None
|
|
for i in range(start_idx, end_idx):
|
|
p = all_paragraphs[i]
|
|
cls = get_css_class(p)
|
|
if cls == VERSE_BOOKMARK_CLASS:
|
|
text = decode_text(p)
|
|
text_num = parse_verse_bookmark(text)
|
|
if text_num is not None:
|
|
verse_bookmarks.append((i, text_num))
|
|
elif text.strip() == "NoI Appendixes":
|
|
appendixes_idx = i
|
|
return verse_bookmarks, appendixes_idx
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Intro segment extraction
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def extract_intro_segment(
|
|
all_paragraphs: list, start_idx: int, end_idx: int
|
|
) -> list[str]:
|
|
"""Extract paragraphs for an intro segment (front pages, preface, or appendixes).
|
|
|
|
Decodes ScaGoudy, preserves HTML formatting, skips dashed lines and empty paragraphs.
|
|
"""
|
|
result = []
|
|
|
|
for i in range(start_idx, end_idx):
|
|
p = all_paragraphs[i]
|
|
cls = get_css_class(p)
|
|
|
|
if is_empty(p):
|
|
continue
|
|
|
|
# Skip bookmark and header classes
|
|
if cls in (SECTION_BOOKMARK_CLASS, BOOK_HEADER_CLASS, VERSE_BOOKMARK_CLASS):
|
|
continue
|
|
|
|
# Skip noise classes
|
|
if cls in SKIP_CLASSES:
|
|
continue
|
|
|
|
html = decode_html_content(p)
|
|
if not html:
|
|
continue
|
|
|
|
# Skip dashed-line separators
|
|
plain = re.sub(r"<[^>]+>", "", html).strip()
|
|
if is_dashed_line(plain):
|
|
continue
|
|
|
|
result.append(html)
|
|
|
|
return result
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Verse extraction state machine
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def extract_verses(all_paragraphs: list, verses_start: int, verses_end: int) -> list:
|
|
"""Extract verses from the NoI section using a state machine.
|
|
|
|
Returns list of verse dicts.
|
|
"""
|
|
verses = []
|
|
current_verse = None
|
|
state = "IDLE"
|
|
|
|
for i in range(verses_start, verses_end):
|
|
p = all_paragraphs[i]
|
|
cls = get_css_class(p)
|
|
|
|
# --- Verse bookmark (p49) ---
|
|
if cls == VERSE_BOOKMARK_CLASS:
|
|
# Finalize previous verse if any
|
|
if current_verse and state != "IDLE":
|
|
_finalize_verse(current_verse)
|
|
verses.append(current_verse)
|
|
|
|
text = decode_text(p)
|
|
text_num = parse_verse_bookmark(text)
|
|
if text_num is not None:
|
|
title = TEXT_TITLES.get(text_num, f"Text {text_num}")
|
|
reference = f"NoI {text_num}"
|
|
|
|
current_verse = {
|
|
"book": "noi",
|
|
"book_title": "The Nectar of Instruction",
|
|
"text": text_num,
|
|
"title": title,
|
|
"reference": reference,
|
|
"devanagari": "",
|
|
"transliteration": "",
|
|
"_padas": [],
|
|
"synonyms": [],
|
|
"translation": "",
|
|
"purport": [],
|
|
}
|
|
state = "VERSE_START"
|
|
else:
|
|
# Non-verse bookmark (e.g., "NoI Appendixes") — stop processing
|
|
current_verse = None
|
|
state = "IDLE"
|
|
continue
|
|
|
|
if not current_verse:
|
|
continue
|
|
|
|
# --- Verse heading (p50) ---
|
|
if cls == VERSE_HEADING_CLASS:
|
|
# Just confirms the text title; skip
|
|
continue
|
|
|
|
# --- RM Devanagari (p51) — SKIP ---
|
|
if cls == DEVANAGARI_CLASS:
|
|
continue
|
|
|
|
# --- TEXT label (p52) — skip ---
|
|
if cls == TEXT_LABEL_CLASS:
|
|
state = "TRANSLITERATION"
|
|
continue
|
|
|
|
# --- Pāda transliteration lines (p5) ---
|
|
if cls == PADA_TRANSLIT_CLASS and state in (
|
|
"VERSE_START",
|
|
"TRANSLITERATION",
|
|
):
|
|
state = "TRANSLITERATION"
|
|
line = decode_text(p)
|
|
# Remove trailing period
|
|
line = line.rstrip(".")
|
|
# Strip leading/trailing whitespace
|
|
line = line.strip()
|
|
if line:
|
|
current_verse["_padas"].append(line)
|
|
continue
|
|
|
|
# --- Synonyms (p54) ---
|
|
if cls == SYNONYM_CLASS and current_verse:
|
|
state = "SYNONYMS"
|
|
syn_text = decode_text(p)
|
|
syn_text = syn_text.strip()
|
|
parsed = parse_synonyms(syn_text)
|
|
current_verse["synonyms"].extend(parsed)
|
|
continue
|
|
|
|
# --- TRANSLATION / PURPORT label (p55) ---
|
|
if cls == TRANSLATION_LABEL_CLASS and current_verse:
|
|
label_text = decode_text(p).strip().upper()
|
|
if "TRANSLATION" in label_text:
|
|
state = "TRANSLATION"
|
|
elif "PURPORT" in label_text:
|
|
state = "PURPORT"
|
|
else:
|
|
# Other labels within purport (e.g., "The Author", "Glossary")
|
|
if state == "PURPORT":
|
|
html = decode_html_content(p)
|
|
if html:
|
|
current_verse["purport"].append(html)
|
|
continue
|
|
|
|
# --- Translation text (p56) ---
|
|
if cls == TRANSLATION_TEXT_CLASS and current_verse and state == "TRANSLATION":
|
|
html = decode_html_content(p)
|
|
# Remove bold wrapper if present (the whole translation is bold)
|
|
html = re.sub(r"^<b>(.*)</b>$", r"\1", html, flags=re.DOTALL)
|
|
html = html.strip()
|
|
current_verse["translation"] = html
|
|
state = "POST_TRANSLATION"
|
|
continue
|
|
|
|
# --- PURPORT label (p57) ---
|
|
if cls == PURPORT_LABEL_CLASS and current_verse:
|
|
state = "PURPORT"
|
|
continue
|
|
|
|
# --- Purport body paragraphs (p48) ---
|
|
if state == "PURPORT" and current_verse and cls == PURPORT_BODY_CLASS:
|
|
html = decode_html_content(p)
|
|
if html:
|
|
plain = re.sub(r"<[^>]+>", "", html).strip()
|
|
if not is_dashed_line(plain):
|
|
current_verse["purport"].append(html)
|
|
continue
|
|
|
|
# --- Quoted Sanskrit/Bengali verses within purport (p33, p53, p66) ---
|
|
if state == "PURPORT" and current_verse and cls in QUOTED_VERSE_CLASSES:
|
|
html = decode_html_content(p)
|
|
if html:
|
|
# Strip inner <i>/<\/i> tags to avoid double wrapping
|
|
stripped = re.sub(r"</?i>", "", html)
|
|
current_verse["purport"].append(
|
|
f'<div style="text-align:center"><i>{stripped}</i></div>'
|
|
)
|
|
continue
|
|
|
|
# --- Body text / verse translations within purport (p44) ---
|
|
if state == "PURPORT" and current_verse and cls == BODY_TEXT_CLASS:
|
|
html = decode_html_content(p)
|
|
if html:
|
|
plain = re.sub(r"<[^>]+>", "", html).strip()
|
|
if not is_dashed_line(plain):
|
|
current_verse["purport"].append(html)
|
|
continue
|
|
|
|
# --- Miscellaneous text (p67) — treat as body text within purport ---
|
|
if state == "PURPORT" and current_verse and cls == MISC_TEXT_CLASS:
|
|
html = decode_html_content(p)
|
|
if html:
|
|
current_verse["purport"].append(html)
|
|
continue
|
|
|
|
# --- Section heading within purport (p31) ---
|
|
if state == "PURPORT" and current_verse and cls == SECTION_HEADING_CLASS:
|
|
html = decode_html_content(p)
|
|
if html:
|
|
current_verse["purport"].append(html)
|
|
continue
|
|
|
|
# --- Body text classes that might appear in purport (p6, p45) ---
|
|
if state == "PURPORT" and current_verse and cls in BODY_CLASSES:
|
|
html = decode_html_content(p)
|
|
if html:
|
|
plain = re.sub(r"<[^>]+>", "", html).strip()
|
|
if not is_dashed_line(plain):
|
|
current_verse["purport"].append(html)
|
|
continue
|
|
|
|
# Finalize last verse
|
|
if current_verse and state != "IDLE":
|
|
_finalize_verse(current_verse)
|
|
verses.append(current_verse)
|
|
|
|
return verses
|
|
|
|
|
|
def _finalize_verse(verse: dict):
|
|
"""Build transliteration and Devanagari from pādas. Remove internal fields."""
|
|
padas = verse.get("_padas", [])
|
|
|
|
# Transliteration: one pāda per line
|
|
verse["transliteration"] = "\n".join(padas)
|
|
|
|
# Devanagari: pair pādas into 2-line verse structure, then convert
|
|
if padas:
|
|
paired = pair_padas(padas)
|
|
verse["devanagari"] = iast_to_devanagari(paired)
|
|
|
|
# Clean up internal field
|
|
if "_padas" in verse:
|
|
del verse["_padas"]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# JSON writing
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def write_json(data, filepath):
|
|
"""Write JSON with proper Unicode and formatting."""
|
|
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
|
with open(filepath, "w", encoding="utf-8") as f:
|
|
json.dump(data, f, ensure_ascii=False, indent=2)
|
|
print(f" Written: {filepath}")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def main():
|
|
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
html_path = os.path.join(base_dir, "public", "html", "other.html")
|
|
output_base = os.path.join(base_dir, "lib", "jsons", "books", "noi")
|
|
|
|
if not os.path.exists(html_path):
|
|
print(f"ERROR: {html_path} not found!")
|
|
sys.exit(1)
|
|
|
|
print(f"Reading {html_path}...")
|
|
with open(html_path, "r", encoding="utf-8") as f:
|
|
soup = BeautifulSoup(f, "html.parser")
|
|
|
|
all_paragraphs = soup.find_all("p")
|
|
print(f"Total paragraphs in file: {len(all_paragraphs)}")
|
|
|
|
# --- Find NoI boundaries ---
|
|
noi_start, noi_end = find_noi_boundaries(all_paragraphs)
|
|
print(f"NoI content: paragraphs {noi_start} to {noi_end}")
|
|
|
|
# --- Find section bookmarks (p64) ---
|
|
section_bookmarks = find_section_bookmarks(all_paragraphs, noi_start, noi_end)
|
|
print(f"\nSection bookmarks (p64) found:")
|
|
for bm_text, bm_idx in sorted(section_bookmarks.items(), key=lambda x: x[1]):
|
|
print(f" [{bm_idx}] {bm_text}")
|
|
|
|
# --- Find verse bookmarks (p49) ---
|
|
verse_bookmarks, appendixes_idx = find_verse_bookmarks(
|
|
all_paragraphs, noi_start, noi_end
|
|
)
|
|
print(f"\nVerse bookmarks (p49) found: {len(verse_bookmarks)}")
|
|
for bm_idx, text_num in verse_bookmarks:
|
|
print(f" [{bm_idx}] NoI: verse {text_num}")
|
|
if appendixes_idx is not None:
|
|
print(f" [{appendixes_idx}] NoI Appendixes")
|
|
|
|
# Key bookmark indices
|
|
front_pages_idx = section_bookmarks.get("NoI Front pages")
|
|
preface_idx = section_bookmarks.get("NoI Preface")
|
|
|
|
# First verse bookmark
|
|
first_verse_idx = verse_bookmarks[0][0] if verse_bookmarks else None
|
|
|
|
if first_verse_idx is None:
|
|
print("ERROR: Could not find any verse bookmarks!")
|
|
sys.exit(1)
|
|
|
|
# End boundary: the Glossary <p class="p55"><b>Glossary</b></p> or noi_end
|
|
end_boundary = noi_end
|
|
# Look for the Glossary marker just before the next book
|
|
for i in range(noi_start, noi_end):
|
|
p = all_paragraphs[i]
|
|
cls = get_css_class(p)
|
|
if cls == TRANSLATION_LABEL_CLASS:
|
|
text = decode_text(p).strip()
|
|
if text.upper() == "GLOSSARY":
|
|
end_boundary = i
|
|
break
|
|
|
|
# --- Extract intro segments ---
|
|
print("\n--- Extracting Intro Segments ---")
|
|
|
|
intro_segments = []
|
|
|
|
# 1. Front pages segment — from NoI Front pages (p64) to NoI Preface (p64)
|
|
front_end = preface_idx if preface_idx is not None else first_verse_idx
|
|
if front_pages_idx is not None and front_end is not None:
|
|
front_paras = extract_intro_segment(all_paragraphs, front_pages_idx, front_end)
|
|
front_segment = {
|
|
"slug": "front-pages",
|
|
"title": "Front Pages",
|
|
"type": "intro-segment",
|
|
"book": "noi",
|
|
"order": 1,
|
|
"paragraphs": front_paras,
|
|
}
|
|
intro_segments.append(front_segment)
|
|
print(f" Front Pages: {len(front_paras)} paragraphs")
|
|
|
|
# 2. Preface — from NoI Preface (p64) to first verse bookmark
|
|
if preface_idx is not None and first_verse_idx is not None:
|
|
preface_paras = extract_intro_segment(
|
|
all_paragraphs, preface_idx, first_verse_idx
|
|
)
|
|
preface_segment = {
|
|
"slug": "preface",
|
|
"title": "Preface",
|
|
"type": "intro-segment",
|
|
"book": "noi",
|
|
"order": 2,
|
|
"paragraphs": preface_paras,
|
|
}
|
|
intro_segments.append(preface_segment)
|
|
print(f" Preface: {len(preface_paras)} paragraphs")
|
|
|
|
# 3. Appendixes — from NoI Appendixes (p49) to end boundary
|
|
if appendixes_idx is not None:
|
|
appendixes_paras = extract_intro_segment(
|
|
all_paragraphs, appendixes_idx, end_boundary
|
|
)
|
|
appendixes_segment = {
|
|
"slug": "appendixes",
|
|
"title": "Appendixes",
|
|
"type": "intro-segment",
|
|
"book": "noi",
|
|
"order": 3,
|
|
"paragraphs": appendixes_paras,
|
|
}
|
|
intro_segments.append(appendixes_segment)
|
|
print(f" Appendixes: {len(appendixes_paras)} paragraphs")
|
|
|
|
# --- Extract verses ---
|
|
print("\n--- Extracting Verses ---")
|
|
|
|
# Verse section runs from first verse bookmark to appendixes (or end boundary)
|
|
verse_end = appendixes_idx if appendixes_idx is not None else end_boundary
|
|
verses = extract_verses(all_paragraphs, first_verse_idx, verse_end)
|
|
print(f"Extracted {len(verses)} texts (expected 11: Texts 1-11)")
|
|
|
|
# --- Write intro.json ---
|
|
texts_listing = []
|
|
for num in range(1, 12):
|
|
title = TEXT_TITLES[num]
|
|
slug = str(num).zfill(2)
|
|
texts_listing.append({"number": num, "title": title, "slug": slug})
|
|
|
|
intro_meta = {
|
|
"book": "noi",
|
|
"title": "The Nectar of Instruction",
|
|
"subtitle": "Śrī Upadeśāmṛta",
|
|
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
|
"year": 1976,
|
|
"description": "Eleven essential instructions on the science of bhakti-yoga by Śrīla Rūpa Gosvāmī.",
|
|
"totalTexts": 11,
|
|
"texts": texts_listing,
|
|
}
|
|
write_json(intro_meta, os.path.join(output_base, "intro.json"))
|
|
|
|
# --- Write intro segments ---
|
|
print("\n--- Writing Intro Segments ---")
|
|
for seg in intro_segments:
|
|
filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
|
|
write_json(seg, filepath)
|
|
|
|
# --- Write verse JSON files ---
|
|
print("\n--- Writing Verse Files ---")
|
|
for verse in verses:
|
|
text_num = verse["text"]
|
|
filename = f"{str(text_num).zfill(2)}.json"
|
|
filepath = os.path.join(output_base, "verses", filename)
|
|
write_json(verse, filepath)
|
|
|
|
# --- Summary ---
|
|
print(f"\n{'=' * 60}")
|
|
print(f" EXTRACTION SUMMARY")
|
|
print(f"{'=' * 60}")
|
|
print(f" Book: The Nectar of Instruction (1976 Edition)")
|
|
print(f" Intro segments: {len(intro_segments)}")
|
|
for seg in intro_segments:
|
|
print(
|
|
f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs"
|
|
)
|
|
print(f" Texts extracted: {len(verses)}")
|
|
texts_found = sorted([v["text"] for v in verses])
|
|
if texts_found:
|
|
print(
|
|
f" Texts: {texts_found[0]}-{texts_found[-1]} ({len(texts_found)} total)"
|
|
)
|
|
else:
|
|
print(" Texts: None")
|
|
|
|
total_purport_paras = sum(len(v["purport"]) for v in verses)
|
|
total_synonyms = sum(len(v["synonyms"]) for v in verses)
|
|
print(f" Total purport ¶s: {total_purport_paras}")
|
|
print(f" Total synonyms: {total_synonyms}")
|
|
print(f" Output directory: {output_base}/")
|
|
|
|
# --- Quality checks ---
|
|
print(f"\n--- Quality Checks ---")
|
|
|
|
# Check all verses have expected fields
|
|
issues = []
|
|
for verse in verses:
|
|
ref = verse["reference"]
|
|
if not verse["transliteration"]:
|
|
issues.append(f" {ref}: missing transliteration")
|
|
if not verse["devanagari"]:
|
|
issues.append(f" {ref}: missing devanagari")
|
|
if not verse["synonyms"]:
|
|
issues.append(f" {ref}: missing synonyms")
|
|
if not verse["translation"]:
|
|
issues.append(f" {ref}: missing translation")
|
|
if not verse["purport"]:
|
|
issues.append(f" {ref}: missing purport")
|
|
|
|
if issues:
|
|
print(" ISSUES FOUND:")
|
|
for issue in issues:
|
|
print(issue)
|
|
else:
|
|
print(
|
|
" All texts have transliteration, devanagari, synonyms, translation, and purport ✓"
|
|
)
|
|
|
|
# Check expected text numbers are present
|
|
expected = set(range(1, 12))
|
|
found = set(texts_found)
|
|
missing = expected - found
|
|
extra = found - expected
|
|
if missing:
|
|
print(f" MISSING texts: {sorted(missing)}")
|
|
if extra:
|
|
print(f" UNEXPECTED texts: {sorted(extra)}")
|
|
if not missing and not extra:
|
|
print(f" All 11 texts present ✓")
|
|
|
|
# --- Samples ---
|
|
print(f"\n--- Sample: Text One ---")
|
|
t1 = [v for v in verses if v["text"] == 1]
|
|
if t1:
|
|
v = t1[0]
|
|
print(f" Reference: {v['reference']}")
|
|
print(f" Title: {v['title']}")
|
|
print(f" Transliteration: {v['transliteration'][:80]}...")
|
|
print(f" Devanagari: {v['devanagari'][:80]}...")
|
|
print(f" Synonyms: {len(v['synonyms'])} entries")
|
|
if v["synonyms"]:
|
|
print(f" First: {v['synonyms'][0]}")
|
|
print(f" Last: {v['synonyms'][-1]}")
|
|
trans_preview = re.sub(r"<[^>]+>", "", v["translation"])[:100]
|
|
print(f" Translation: {trans_preview}...")
|
|
print(f" Purport ¶s: {len(v['purport'])}")
|
|
|
|
print(f"\n--- Sample: Text Six (middle) ---")
|
|
t6 = [v for v in verses if v["text"] == 6]
|
|
if t6:
|
|
v = t6[0]
|
|
print(f" Reference: {v['reference']}")
|
|
print(f" Title: {v['title']}")
|
|
print(f" Transliteration: {v['transliteration'][:80]}...")
|
|
print(f" Devanagari: {v['devanagari'][:80]}...")
|
|
print(f" Synonyms: {len(v['synonyms'])} entries")
|
|
trans_preview = re.sub(r"<[^>]+>", "", v["translation"])[:100]
|
|
print(f" Translation: {trans_preview}...")
|
|
print(f" Purport ¶s: {len(v['purport'])}")
|
|
|
|
print(f"\n--- Sample: Text Eleven (last) ---")
|
|
t11 = [v for v in verses if v["text"] == 11]
|
|
if t11:
|
|
v = t11[0]
|
|
print(f" Reference: {v['reference']}")
|
|
print(f" Title: {v['title']}")
|
|
print(f" Transliteration: {v['transliteration'][:80]}...")
|
|
print(f" Devanagari: {v['devanagari'][:80]}...")
|
|
print(f" Synonyms: {len(v['synonyms'])} entries")
|
|
trans_preview = re.sub(r"<[^>]+>", "", v["translation"])[:100]
|
|
print(f" Translation: {trans_preview}...")
|
|
print(f" Purport ¶s: {len(v['purport'])}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|