#!/usr/bin/env python3 """Extract Songs of the Vaiṣṇava Ācāryas (1972) from other.html. Hierarchical sub-chapter extraction: each song within a part becomes its OWN chapter JSON file with a parent-child relationship. Parent chapters (Parts 1–4) are containers only — no content of their own. Numbering: Part 1, Song 1 → slug "1-1", Part 2, Song 3 → slug "2-3", etc. Song boundaries are delimited by p49 bookmarks with text like: "SVA 1: Śrī Guru Praṇāma" "SVA 2: Aruṇodaya-kīrtana" Output: lib/jsons/books/sva/intro.json — Book metadata with hierarchical chapters lib/jsons/books/sva/intro/*.json — Intro segments (front-pages, foreword, introduction) lib/jsons/books/sva/chapters/*.json — Sub-chapter files (1-1.json, 1-2.json, 2-1.json, ...) Usage: python3 scripts/sva_extract.py """ import json import os import re import sys from collections import defaultdict from bs4 import BeautifulSoup, NavigableString, Tag # --------------------------------------------------------------------------- # ScaGoudy → IAST Unicode mapping # --------------------------------------------------------------------------- SCAGOUDY_TO_IAST = { "ä": "ā", "Ä": "Ā", "å": "ṛ", "Å": "Ṛ", "ñ": "ṣ", "Ñ": "Ṣ", "ö": "ṭ", "Ö": "Ṭ", "ë": "ṇ", "Ë": "Ṇ", "ò": "ḍ", "Ò": "Ḍ", "ç": "ś", "Ç": "Ś", "ì": "ṅ", "Ì": "Ṅ", "ï": "ñ", "Ï": "Ñ", "ü": "ū", "Ü": "Ū", "é": "ī", "É": "Ī", "à": "ṁ", "À": "Ṁ", "ù": "ḥ", "Ù": "Ḥ", "è": "ṝ", "û": "ṁl", "ÿ": "ḷ", } def decode_scagoudy(text: str) -> str: """Replace ScaGoudy-encoded characters with proper IAST Unicode.""" for src, dst in SCAGOUDY_TO_IAST.items(): text = text.replace(src, dst) return text def decode_html_content(tag: Tag) -> str: """Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags.""" inner = "".join(str(child) for child in tag.children) inner = decode_scagoudy(inner) # Remove span tags with CSS classes (RTF conversion artifacts) inner = re.sub( r']*class="[^"]*"[^>]*>(.*?)', r"\1", inner, flags=re.DOTALL ) # Clean up Apple-tab-span leftovers (tab characters) inner = inner.replace("\t", " ") # Force valid UTF-8 by round-tripping through encode/decode with replacement inner = inner.encode("utf-8", errors="replace").decode("utf-8", errors="replace") # Strip null bytes and other control characters that break BSON inner = re.sub(r"[\x00-\x08\x0b\x0c\x0e-\x1f]", "", inner) # Remove lone surrogates (U+D800–U+DFFF) that are invalid in BSON/MongoDB inner = re.sub(r"[\ud800-\udfff]", "", inner) # Merge adjacent bold tags: → (nothing) for _ in range(5): inner = re.sub(r"\s*", "", inner) # Remove empty bold and italic tags inner = re.sub(r"\s*", "", inner) inner = re.sub(r"\s*", "", inner) # Normalize whitespace inner = re.sub(r"\s+", " ", inner).strip() return inner def decode_text(tag: Tag) -> str: """Get plain text from a tag with ScaGoudy decoding.""" return decode_scagoudy(tag.get_text().strip()) def get_css_class(tag: Tag) -> str: """Get the first CSS class of a tag, or empty string.""" classes = tag.get("class", []) return classes[0] if classes else "" def is_empty(tag: Tag) -> bool: """Check if a tag is empty or whitespace-only.""" text = tag.get_text().strip() return not text or text == "\xa0" def is_dash_line(text: str) -> bool: """Return True if text is only dashes, spaces, and/or whitespace.""" cleaned = text.strip() if not cleaned: return False return all(c in "-—–\u2014\u2013 \t" for c in cleaned) def is_inside_table(tag: Tag) -> bool: """Check if a tag is nested inside a element.""" parent = tag.parent while parent: if parent.name == "table": return True parent = parent.parent return False # --------------------------------------------------------------------------- # CSS class categorization for SVA # --------------------------------------------------------------------------- # Book title header (bold, p12) BOOK_HEADER_CLASS = "p12" # Bookmark class — section boundaries BOOKMARK_CLASS = "p13" # Song bookmark / section bookmark within a part SONG_BOOKMARK_CLASS = "p49" # Front pages text FRONT_PAGES_CLASS = "p18" # Body paragraph classes BODY_CLASSES = frozenset(["p6", "p48", "p140", "p204", "p44"]) # Section headings (bold) SECTION_HEADING_CLASS = "p31" # Song title headings (bold) SONG_TITLE_CLASS = "p27" # Song title variant (bold italic with author) SONG_TITLE_VARIANT_CLASS = "p169" # Verse line classes (italic, centered) VERSE_PADA_CLASSES = frozenset(["p5", "p53", "p24", "p66", "p255", "p134"]) # Verse number classes (e.g. "3)", "4)", "[1]") — centered but NOT verse text VERSE_NUMBER_CLASSES = frozenset(["p248", "p30"]) # Translation variant classes (bold, centered) — in addition to p56 TRANSLATION_VARIANT_CLASSES = frozenset(["p86", "p240", "p242"]) # Reference/purport note classes that should be centered CENTERED_NOTE_CLASSES = frozenset(["p241", "p22"]) # Synonyms text SYNONYMS_CLASS = "p54" # Translation text (bold) TRANSLATION_CLASS = "p56" # "SYNONYMS" header SYNONYMS_HEADER_CLASS = "p165" # "TRANSLATION" header TRANSLATION_HEADER_CLASS = "p55" # Mantra text MANTRA_CLASS = "p19" # Note text NOTE_CLASS = "p204" # Attribution ATTRIBUTION_CLASS = "p129" # Date/location DATE_CLASS = "p67" # Signature block SIGNATURE_CLASS = "p222" # Bold section header BOLD_HEADER_CLASS = "p227" # Bracketed notes BRACKET_NOTE_CLASS = "p228" # Disciplic succession cell content DISC_SUCC_CLASS = "p229" # Reference notes REFERENCE_CLASSES = frozenset( [ "p187", "p231", "p233", "p237", "p32", "p251", "p203", "p246", "p249", "p254", "p241", ] ) # Bold text classes BOLD_TEXT_CLASSES = frozenset(["p236", "p238"]) # Synonym notes SYNONYM_NOTES_CLASS = "p247" # Bold headings BOLD_HEADING_CLASSES = frozenset(["p224", "p223"]) # Part heading (bold, for song subsections) PART_HEADING_CLASS = "p22" # Sub-translation / inline bold classes INLINE_BOLD_CLASSES = frozenset(["p86", "p240", "p242", "p243", "p245", "p258"]) # Verse number marker VERSE_NUMBER_CLASS = "p30" # Song commentary / purport headings PURPORT_HEADING_CLASS = "p96" # Purport body text PURPORT_BODY_CLASS = "p140" # Additional verse line classes in tables TABLE_VERSE_CLASSES = frozenset(["p248", "p255", "p5", "p6"]) # Verse numbering markers VERSE_MARKER_CLASSES = frozenset(["p252", "p253"]) # Classes to skip entirely (empty/break/decorative) SKIP_CLASSES = frozenset( [ "p15", "p120", "p225", "p226", "p219", "p230", "p232", "p234", "p135", "p183", "p172", "p179", "p260", ] ) # --------------------------------------------------------------------------- # Clean parent titles (pre-decoded IAST) # --------------------------------------------------------------------------- CLEAN_PARENT_TITLES = { 1: "Prayers and Praṇāmas", 2: "Songs of Śrīla Bhaktivinoda Ṭhākura", 3: "Songs of Śrīla Narottama dāsa Ṭhākura", 4: "Songs by other Ācāryas", } # --------------------------------------------------------------------------- # Extraction helpers # --------------------------------------------------------------------------- def find_sva_boundaries(all_elements: list) -> tuple: """Find the start and end element indices of the SVA content. Start:

containing "Songs of the Vaiñëava Äcäryas" End:

containing "The Hare" (Cook Book) """ start_idx = None end_idx = len(all_elements) for i, elem in enumerate(all_elements): if elem.name != "p": continue cls = get_css_class(elem) text = decode_text(elem) # Find start if start_idx is None: if cls == BOOK_HEADER_CLASS and "Songs of the Vai" in text: start_idx = i continue # Find end: the next p12 header (Hare Kṛṣṇa Cook Book) if cls == BOOK_HEADER_CLASS: end_idx = i break if start_idx is None: print("ERROR: Could not find 'Songs of the Vaiṣṇava Ācāryas' header!") sys.exit(1) return start_idx, end_idx def find_sva_bookmarks(all_elements: list, start_idx: int, end_idx: int) -> list: """Find all SVA bookmark/section tags in the SVA range. Looks for p13 class with "SVA" text, and p49 class with "SVA" text. Returns list of (element_index, bookmark_text, bookmark_type). bookmark_type is "p13" or "p49". """ bookmarks = [] for i in range(start_idx, end_idx): elem = all_elements[i] if elem.name != "p": continue cls = get_css_class(elem) if cls not in (BOOKMARK_CLASS, SONG_BOOKMARK_CLASS): continue text = decode_text(elem).strip() if text.startswith("SVA"): bookmarks.append((i, text, cls)) return bookmarks def process_table_t3(table: Tag) -> list: """Process a t3 table (side-by-side Bengali verse + English translation). Returns a list of HTML paragraph strings. """ result = [] for tr in table.find_all("tr"): tds = tr.find_all("td") if len(tds) < 2: # Single-cell row — just extract content for td in tds: for p in td.find_all("p"): html = decode_html_content(p) if html and html.strip() and html.strip() != "\xa0": result.append(html) continue # Left cell: Bengali verse lines left_ps = tds[0].find_all("p") verse_lines = [] for p in left_ps: html = decode_html_content(p) if html and html.strip() and html.strip() != "\xa0": # Strip tags — we'll re-wrap stripped = re.sub(r"", "", html) if stripped.strip(): verse_lines.append(stripped.strip()) if verse_lines: joined = "
".join(verse_lines) result.append(f'

{joined}
') # Right cell: English translation — centered right_ps = tds[1].find_all("p") for p in right_ps: html = decode_html_content(p) if html and html.strip() and html.strip() != "\xa0": result.append(f'
{html}
') return result def process_table_t2(table: Tag) -> list: """Process a t2 table (disciplic succession table). Extract cell content as regular paragraphs. """ result = [] for tr in table.find_all("tr"): for td in tr.find_all("td"): for p in td.find_all("p"): html = decode_html_content(p) if html and html.strip() and html.strip() != "\xa0": result.append(html) return result def extract_paragraphs( all_elements: list, start_idx: int, end_idx: int, ) -> list: """Extract paragraphs for a section (song, intro segment). Handles all CSS classes within the SVA content. Handles
and
elements. Groups consecutive verse pāda classes into centered italic divs. """ result = [] verse_accumulator = [] # Track which

tags are inside tables we've already processed processed_table_ps = set() def flush_verse(): if verse_accumulator: joined = "
".join(verse_accumulator) result.append(f'

{joined}
') verse_accumulator.clear() for i in range(start_idx, end_idx): elem = all_elements[i] # Handle tables if elem.name == "table": flush_verse() table_classes = elem.get("class", []) # Mark all

inside this table as processed for p in elem.find_all("p"): processed_table_ps.add(id(p)) if "t3" in table_classes: table_paras = process_table_t3(elem) result.extend(table_paras) elif "t2" in table_classes: table_paras = process_table_t2(elem) result.extend(table_paras) else: # Generic table — extract all paragraph content for p in elem.find_all("p"): html = decode_html_content(p) if html and html.strip(): result.append(html) continue if elem.name != "p": continue # Skip if this

was already processed as part of a table if id(elem) in processed_table_ps: continue # Skip if inside a table (safety check) if is_inside_table(elem): continue cls = get_css_class(elem) # Skip empty paragraphs if is_empty(elem): continue # Skip noise classes if cls in SKIP_CLASSES: continue # Skip bookmark tags (p13, p49) — these are navigation markers if cls == BOOKMARK_CLASS: continue if cls == SONG_BOOKMARK_CLASS: continue # Skip book title header (p12) if cls == BOOK_HEADER_CLASS: continue text = decode_text(elem) # Check for dash lines — skip them if is_dash_line(text): continue # Sanskrit verse pādas (p5, p53, p24, p248, p255) — accumulate if cls in VERSE_PADA_CLASSES: html = decode_html_content(elem) if html: stripped = re.sub(r"", "", html) if stripped.strip(): verse_accumulator.append(stripped.strip()) continue # Non-verse paragraph — flush any accumulated verse first flush_verse() # Short p6 content (verse numbers like "1)", "2)") should be centered # Long p6 content (body paragraphs) stays left-aligned if cls == "p6": html = decode_html_content(elem) if html: plain = re.sub(r"<[^>]+>", "", html).strip() # Short content that looks like a verse number or bracketed ref if len(plain) <= 10 and re.match(r"^[\[\(]?\d+[\]\)]?\.?$", plain): result.append(f'

{html}
') else: result.append(html) continue # Front pages text (p18) if cls == FRONT_PAGES_CLASS: html = decode_html_content(elem) if html: result.append(html) continue # Section heading (p31) — bold, centered if cls == SECTION_HEADING_CLASS: html = decode_html_content(elem) if html: if not html.strip().startswith(""): html = f"{html}" result.append(f'
{html}
') continue # Song title (p27) — bold, centered if cls == SONG_TITLE_CLASS: html = decode_html_content(elem) if html: if not html.strip().startswith(""): html = f"{html}" result.append(f'
{html}
') continue # Song title variant (p169) — bold italic with author, centered if cls == SONG_TITLE_VARIANT_CLASS: html = decode_html_content(elem) if html: result.append(f'
{html}
') continue # Synonyms (p54) if cls == SYNONYMS_CLASS: html = decode_html_content(elem) if html: result.append(html) continue # Verse number heading (p248, p30) — e.g. "3)", "4)", "[1]" — center it if cls in VERSE_NUMBER_CLASSES: flush_verse() html = decode_html_content(elem) if html: result.append(f'
{html}
') continue # Translation variant classes (p86, p240, p242) — bold, centered if cls in TRANSLATION_VARIANT_CLASSES: flush_verse() html = decode_html_content(elem) if html: result.append(f'
{html}
') continue # Centered note classes (p241) if cls in CENTERED_NOTE_CLASSES: flush_verse() html = decode_html_content(elem) if html: result.append(f'
{html}
') continue # Translation (p56) — bold, centered if cls == TRANSLATION_CLASS: html = decode_html_content(elem) if html: result.append(f'
{html}
') continue # SYNONYMS header (p165) — centered if cls == SYNONYMS_HEADER_CLASS: html = decode_html_content(elem) if html: result.append(f'
{html}
') continue # TRANSLATION header (p55) — centered if cls == TRANSLATION_HEADER_CLASS: html = decode_html_content(elem) if html: result.append(f'
{html}
') continue # Mantra text (p19) if cls == MANTRA_CLASS: html = decode_html_content(elem) if html: result.append(html) continue # Attribution (p129) if cls == ATTRIBUTION_CLASS: html = decode_html_content(elem) if html: result.append(html) continue # Date/location (p67) if cls == DATE_CLASS: html = decode_html_content(elem) if html: result.append(html) continue # Signature block (p222) if cls == SIGNATURE_CLASS: html = decode_html_content(elem) if html: result.append(html) continue # Bold section header (p227) if cls == BOLD_HEADER_CLASS: html = decode_html_content(elem) if html: if not html.strip().startswith(""): html = f"{html}" result.append(html) continue # Bracketed notes (p228) if cls == BRACKET_NOTE_CLASS: html = decode_html_content(elem) if html: result.append(html) continue # Disciplic succession cell content (p229) if cls == DISC_SUCC_CLASS: html = decode_html_content(elem) if html: result.append(html) continue # Reference notes (p187, p231, p233, p237, p32, p251) — centered if cls in REFERENCE_CLASSES: html = decode_html_content(elem) if html: result.append(f'
{html}
') continue # Bold text classes if cls in BOLD_TEXT_CLASSES: html = decode_html_content(elem) if html: result.append(html) continue # Synonym note (p247) — centered if cls == SYNONYM_NOTES_CLASS: html = decode_html_content(elem) if html: result.append(f'
{html}
') continue # Bold headings (p224, p223) if cls in BOLD_HEADING_CLASSES: html = decode_html_content(elem) if html: if not html.strip().startswith(""): html = f"{html}" result.append(html) continue # Part heading (p22) — e.g. "part 1", "part 2" if cls == PART_HEADING_CLASS: html = decode_html_content(elem) if html: result.append(html) continue # Inline bold classes (p86, p240, p242, p243, p245, p258) if cls in INLINE_BOLD_CLASSES: html = decode_html_content(elem) if html: result.append(html) continue # Verse number (p30) if cls == VERSE_NUMBER_CLASS: html = decode_html_content(elem) if html: result.append(html) continue # Purport heading (p96) if cls == PURPORT_HEADING_CLASS: html = decode_html_content(elem) if html: result.append(html) continue # Purport body (p140) if cls == PURPORT_BODY_CLASS: html = decode_html_content(elem) if html: result.append(html) continue # Verse markers (p252, p253) if cls in VERSE_MARKER_CLASSES: html = decode_html_content(elem) if html: result.append(html) continue # Body paragraphs (p6, p48, p140, p204, p44) if cls in BODY_CLASSES: html = decode_html_content(elem) if html: # Detect verses disguised as body: starts with and has
stripped_html = html.strip() if stripped_html.startswith("") and "", "", stripped_html) result.append( f'
{inner}
' ) else: result.append(html) continue # Any other class — try to extract (be permissive) html = decode_html_content(elem) if html and not is_dash_line(text): result.append(html) # Flush any trailing verse flush_verse() return result # --------------------------------------------------------------------------- # Main extraction logic # --------------------------------------------------------------------------- def extract_sva(soup: BeautifulSoup) -> dict: """Extract the entire Songs of the Vaiṣṇava Ācāryas from the parsed HTML. Returns a dict with: "intro_segments": list of intro segment dicts "sub_chapters": list of sub-chapter dicts (one per song) "parent_chapters": list of parent chapter info for intro.json """ # Get all top-level

and

elements all_elements = list(soup.find_all(["p", "table"])) # Step 1: Find SVA boundaries sva_start, sva_end = find_sva_boundaries(all_elements) print(f" SVA content: elements {sva_start} to {sva_end}") print(f" Total elements in range: {sva_end - sva_start}") # Step 2: Find all SVA bookmarks bookmarks = find_sva_bookmarks(all_elements, sva_start, sva_end) print(f" Found {len(bookmarks)} SVA bookmarks") for pos, text, cls in bookmarks[:30]: print(f" Elem ~{pos} [{cls}]: {text[:80]}") if len(bookmarks) > 30: print(f" ... and {len(bookmarks) - 30} more") # Step 3: Categorize bookmarks # p13 bookmarks define major sections: # "SVA: Foreword" → foreword # "SVA: Introduction" → introduction # "SVA 2: ..." → Part 2 start # "SVA 3: ..." → Part 3 start # "SVA 4: ..." → Part 4 start # p49 bookmarks define songs within parts: # "SVA 1: ..." → songs in Part 1 # "SVA 2: ..." → songs in Part 2 # etc. foreword_pos = None introduction_pos = None part2_pos = None part3_pos = None part4_pos = None first_part1_pos = None for pos, text, cls in bookmarks: if cls == BOOKMARK_CLASS: # p13 if text.strip() == "SVA: Foreword": foreword_pos = pos elif text.strip() == "SVA: Introduction": introduction_pos = pos elif text.strip().startswith("SVA 2:"): part2_pos = pos elif text.strip().startswith("SVA 3:"): part3_pos = pos elif text.strip().startswith("SVA 4:"): part4_pos = pos if cls == SONG_BOOKMARK_CLASS: # p49 if text.strip().startswith("SVA 1:") and first_part1_pos is None: first_part1_pos = pos print(f"\n Section boundaries:") print(f" Book start: elem {sva_start}") print( f" Foreword: {'elem ' + str(foreword_pos) if foreword_pos else 'NOT FOUND'}" ) print( f" Introduction: {'elem ' + str(introduction_pos) if introduction_pos else 'NOT FOUND'}" ) print( f" Part 1 start: {'elem ' + str(first_part1_pos) if first_part1_pos else 'NOT FOUND'}" ) print( f" Part 2 start: {'elem ' + str(part2_pos) if part2_pos else 'NOT FOUND'}" ) print( f" Part 3 start: {'elem ' + str(part3_pos) if part3_pos else 'NOT FOUND'}" ) print( f" Part 4 start: {'elem ' + str(part4_pos) if part4_pos else 'NOT FOUND'}" ) print(f" Book end: elem {sva_end}") # Step 4: Extract intro segments # Front pages: from book header to Foreword bookmark front_pages_start = sva_start front_pages_end = ( foreword_pos if foreword_pos else (introduction_pos or first_part1_pos or sva_end) ) fp_paras = extract_paragraphs(all_elements, front_pages_start, front_pages_end) front_pages = { "slug": "front-pages", "title": "Front Pages", "type": "intro-segment", "book": "sva", "order": 1, "paragraphs": fp_paras, } # Foreword: from Foreword bookmark to Introduction bookmark foreword = None if foreword_pos is not None: fw_start = foreword_pos + 1 fw_end = introduction_pos if introduction_pos else (first_part1_pos or sva_end) fw_paras = extract_paragraphs(all_elements, fw_start, fw_end) foreword = { "slug": "foreword", "title": "Foreword", "type": "intro-segment", "book": "sva", "order": 2, "paragraphs": fw_paras, } # Introduction: from Introduction bookmark to Part 1 start introduction = None if introduction_pos is not None: intro_start = introduction_pos + 1 intro_end = first_part1_pos if first_part1_pos else (part2_pos or sva_end) intro_paras = extract_paragraphs(all_elements, intro_start, intro_end) introduction = { "slug": "introduction", "title": "Introduction", "type": "intro-segment", "book": "sva", "order": 3, "paragraphs": intro_paras, } intro_segments = [front_pages] if foreword: intro_segments.append(foreword) if introduction: intro_segments.append(introduction) # Step 5: Collect all p49 song bookmarks grouped by part number song_bookmarks_by_part = defaultdict(list) for pos, text, cls in bookmarks: if cls != SONG_BOOKMARK_CLASS: continue # Parse: "SVA 1: Śrī Guru Praṇāma" → part_num=1, title="Śrī Guru Praṇāma" m = re.match(r"SVA\s+(\d+):\s+(.+)", text.strip()) if m: part_num = int(m.group(1)) song_title = m.group(2).strip() song_bookmarks_by_part[part_num].append( { "pos": pos, "title": song_title, "raw_text": text.strip(), } ) # Step 6: Define part boundaries part_boundaries = [] if first_part1_pos is not None: ch1_end = part2_pos if part2_pos else sva_end part_boundaries.append((1, first_part1_pos, ch1_end)) if part2_pos is not None: ch2_end = part3_pos if part3_pos else sva_end part_boundaries.append((2, part2_pos, ch2_end)) if part3_pos is not None: ch3_end = part4_pos if part4_pos else sva_end part_boundaries.append((3, part3_pos, ch3_end)) if part4_pos is not None: part_boundaries.append((4, part4_pos, sva_end)) # Step 7: Extract sub-chapters (songs) for each part sub_chapters = [] parent_chapters = [] for part_num, part_start, part_end in part_boundaries: parent_title = CLEAN_PARENT_TITLES.get(part_num, f"Part {part_num}") songs = song_bookmarks_by_part.get(part_num, []) # Sort by position songs.sort(key=lambda s: s["pos"]) sub_entries = [] song_counter = 0 for si, song in enumerate(songs): song_title = decode_scagoudy(song["title"]) # Detect "Purport to..." bookmarks — merge into preceding song is_purport = song_title.lower().startswith("purport to") if ( is_purport and sub_chapters and sub_chapters[-1]["parentChapter"] == part_num ): # Merge this purport's content into the last song content_start = song["pos"] + 1 if si + 1 < len(songs): content_end = songs[si + 1]["pos"] else: content_end = part_end purport_paragraphs = extract_paragraphs( all_elements, content_start, content_end ) sub_chapters[-1]["paragraphs"].extend(purport_paragraphs) continue song_counter += 1 slug = f"{part_num}-{song_counter}" # Content range: from this bookmark to next song bookmark (or part end) # We start AFTER the bookmark element itself content_start = song["pos"] + 1 if si + 1 < len(songs): content_end = songs[si + 1]["pos"] else: content_end = part_end paragraphs = extract_paragraphs(all_elements, content_start, content_end) sub_chapters.append( { "book": "sva", "slug": slug, "parentChapter": part_num, "subChapter": song_counter, "title": song_title, "parentTitle": parent_title, "type": "sub-chapter", "paragraphs": paragraphs, } ) sub_entries.append({"slug": slug, "title": song_title}) parent_chapters.append( { "number": part_num, "title": parent_title, "subChapters": sub_entries, } ) print(f"\n Part {part_num}: {parent_title} → {len(sub_entries)} songs") for entry in sub_entries: matching = [sc for sc in sub_chapters if sc["slug"] == entry["slug"]] para_count = len(matching[0]["paragraphs"]) if matching else 0 print( f" {entry['slug']:>5}: {para_count:>3} paragraphs — {entry['title']}" ) return { "intro_segments": intro_segments, "sub_chapters": sub_chapters, "parent_chapters": parent_chapters, } # --------------------------------------------------------------------------- # Output # --------------------------------------------------------------------------- def write_json(data, filepath): """Write data as formatted JSON to filepath, creating directories as needed.""" os.makedirs(os.path.dirname(filepath), exist_ok=True) with open(filepath, "w", encoding="utf-8") as f: json.dump(data, f, ensure_ascii=False, indent=2) def main(): base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) html_path = os.path.join(base_dir, "public", "html", "other.html") output_base = os.path.join(base_dir, "lib", "jsons", "books", "sva") if not os.path.exists(html_path): print(f"ERROR: {html_path} not found!") sys.exit(1) print(f"Reading {html_path}...") with open(html_path, "r", encoding="utf-8") as f: soup = BeautifulSoup(f, "html.parser") print("Extracting Songs of the Vaiṣṇava Ācāryas (1972)...\n") result = extract_sva(soup) intro_segments = result["intro_segments"] sub_chapters = result["sub_chapters"] parent_chapters = result["parent_chapters"] # --- Write intro.json --- intro_meta = { "book": "sva", "title": "Songs of the Vaiṣṇava Ācāryas", "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda", "year": 1972, "description": "Bengali devotional songs by the great Vaiṣṇava teachers, with English translations and commentary.", "totalChapters": 4, "chapters": parent_chapters, } intro_path = os.path.join(output_base, "intro.json") write_json(intro_meta, intro_path) print(f"\n Written: {intro_path}") # --- Write intro segments --- print(f"\n--- Intro Segments ---") for seg in intro_segments: filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json") write_json(seg, filepath) print( f" {seg['order']:>2}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}" ) # --- Write sub-chapter files (songs) --- print(f"\n--- Songs (Sub-Chapters) ---") for sc in sub_chapters: slug = sc["slug"] filepath = os.path.join(output_base, "chapters", f"{slug}.json") write_json(sc, filepath) print( f" {slug:>5}: {len(sc['paragraphs']):>3} paragraphs — " f"[Part {sc['parentChapter']}] {sc['title']}" ) # --- Summary --- total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments) total_chapter_paras = sum(len(sc["paragraphs"]) for sc in sub_chapters) total_paras = total_intro_paras + total_chapter_paras print(f"\n{'=' * 60}") print(f" EXTRACTION SUMMARY") print(f"{'=' * 60}") print(f" Intro segments: {len(intro_segments)}") print(f" Parent chapters: {len(parent_chapters)}") print(f" Songs (sub-ch): {len(sub_chapters)}") print(f" Total paragraphs: {total_paras}") print(f" Intro: {total_intro_paras}") print(f" Songs: {total_chapter_paras}") if sub_chapters: print(f" Avg paras/song: {total_chapter_paras / len(sub_chapters):.1f}") print(f" Output directory: {output_base}/") # --- Per-parent breakdown --- print(f"\n--- Part Breakdown ---") for pc in parent_chapters: part_num = pc["number"] sub_count = len(pc["subChapters"]) part_paras = sum( len(sc["paragraphs"]) for sc in sub_chapters if sc["parentChapter"] == part_num ) print( f" Part {part_num}: {sub_count:>2} songs, {part_paras:>4} paragraphs — {pc['title']}" ) # --- Samples --- print(f"\n--- Sample: Front Pages ---") if intro_segments: front = intro_segments[0] for i, p in enumerate(front["paragraphs"][:5]): preview = re.sub(r"<[^>]+>", "", p)[:100] print(f" [{i}] {preview}...") if sub_chapters: first_sc = sub_chapters[0] print(f"\n--- Sample: {first_sc['slug']} ({first_sc['title']}) ---") for i, p in enumerate(first_sc["paragraphs"][:5]): preview = re.sub(r"<[^>]+>", "", p)[:100] print(f" [{i}] {preview}...") if len(sub_chapters) > 1: last_sc = sub_chapters[-1] print(f"\n--- Sample: {last_sc['slug']} ({last_sc['title']}) ---") for i, p in enumerate(last_sc["paragraphs"][:5]): preview = re.sub(r"<[^>]+>", "", p)[:100] print(f" [{i}] {preview}...") # --- Quality Checks --- print(f"\n--- Quality Checks ---") # Check all 4 parts present expected_parents = set(range(1, 5)) actual_parents = {pc["number"] for pc in parent_chapters} missing = expected_parents - actual_parents extra = actual_parents - expected_parents if missing: print(f" ⚠ Missing parts: {sorted(missing)}") if extra: print(f" ⚠ Extra parts: {sorted(extra)}") if not missing and not extra: print(f" ✅ All 4 parts present") # Check all 3 intro segments present expected_intros = {"front-pages", "foreword", "introduction"} actual_intros = {s["slug"] for s in intro_segments} missing_intros = expected_intros - actual_intros if missing_intros: print(f" ⚠ Missing intro segments: {missing_intros}") else: print(f" ✅ All 3 intro segments present") # Check empty sub-chapters empty_scs = [sc["slug"] for sc in sub_chapters if len(sc["paragraphs"]) == 0] if empty_scs: print(f" ⚠ Empty songs (0 paragraphs): {empty_scs}") else: print(f" ✅ No empty songs") # Check empty intro segments empty_intros = [s["slug"] for s in intro_segments if len(s["paragraphs"]) == 0] if empty_intros: print(f" ⚠ Empty intro segments: {empty_intros}") else: print(f" ✅ No empty intro segments") # Check ScaGoudy decoding encoded_chars = set("äåöëòçüéàùè") suspect_count = 0 all_sections = sub_chapters + intro_segments for section in all_sections: for p in section["paragraphs"]: plain = re.sub(r"<[^>]+>", "", p) for c in encoded_chars: if c in plain: suspect_count += 1 break if suspect_count == 0: print(f" ✅ No suspected undecoded ScaGoudy characters found") else: print( f" ℹ {suspect_count} paragraphs contain characters that may be undecoded ScaGoudy" ) # Check verse blocks verse_block_count = 0 for sc in sub_chapters: for p in sc["paragraphs"]: if '
' in p: verse_block_count += 1 print(f" ℹ {verse_block_count} verse blocks in songs") # Check table-derived content table_verse_count = 0 for sc in sub_chapters: for p in sc["paragraphs"]: if '
' in p and "
" in p: table_verse_count += 1 print(f" ℹ {table_verse_count} multi-line verse blocks (likely from tables)") # ScaGoudy decoding spot check print(f"\n--- ScaGoudy Decoding Spot Check ---") if intro_segments: fw_segs = [s for s in intro_segments if s["slug"] == "foreword"] if fw_segs: fw_text = " ".join(fw_segs[0]["paragraphs"][:10]) if "Prabhupāda" in fw_text: print(f" ✅ 'Prabhupāda' correctly decoded in Foreword") elif "Prabhupäda" in fw_text: print(f" ⚠ ScaGoudy decoding incomplete for 'Prabhupäda'") if "Kṛṣṇa" in fw_text: print(f" ✅ 'Kṛṣṇa' correctly decoded in Foreword") elif "Kåñëa" in fw_text: print(f" ⚠ 'Kåñëa' still encoded in Foreword") # Sample verse blocks print(f"\n--- Sample Verse Blocks ---") shown = 0 for sc in sub_chapters: for p in sc["paragraphs"]: if '
' in p: preview = re.sub(r"<[^>]+>", "", p)[:120] print(f" {sc['slug']} ({sc['title']}): {preview}...") shown += 1 if shown >= 5: break if shown >= 5: break # Song title check print(f"\n--- Song Titles ---") for pc in parent_chapters: print(f" Part {pc['number']}: {pc['title']}") for sub in pc["subChapters"]: print(f" {sub['slug']:>5}: {sub['title']}") # Verify slug numbering continuity within each parent print(f"\n--- Slug Numbering Check ---") for pc in parent_chapters: part_num = pc["number"] slugs = [s["slug"] for s in pc["subChapters"]] expected_slugs = [f"{part_num}-{i + 1}" for i in range(len(slugs))] if slugs == expected_slugs: print( f" ✅ Part {part_num}: {len(slugs)} songs, slugs {slugs[0]}..{slugs[-1]}" ) else: print( f" ⚠ Part {part_num}: slug mismatch! expected {expected_slugs}, got {slugs}" ) # Sub-chapter size check print(f"\n--- Song Size Check ---") very_small = [] very_large = [] for sc in sub_chapters: count = len(sc["paragraphs"]) if count < 1: very_small.append((sc["slug"], count, sc["title"])) elif count > 200: very_large.append((sc["slug"], count, sc["title"])) if very_small: print(f" ⚠ Very small songs (0 paragraphs):") for slug, count, title in very_small: print(f" {slug}: {count} paragraphs — {title}") else: print(f" ✅ All songs have ≥ 1 paragraph") if very_large: print(f" ⚠ Very large songs (> 200 paragraphs):") for slug, count, title in very_large: print(f" {slug}: {count} paragraphs — {title}") else: print(f" ✅ All songs have ≤ 200 paragraphs") # Sub-chapter title decoding check print(f"\n--- Song Title Decoding Check ---") bad_titles = 0 for sc in sub_chapters: title = sc["title"] if "å" in title or "ë" in title or "ñ" in title or "ä" in title: print(f" ⚠ {sc['slug']} title has un-decoded ScaGoudy: {title}") bad_titles += 1 if bad_titles == 0: print(f" ✅ All {len(sub_chapters)} song titles properly decoded") # Check for residual span tags span_count = 0 for sc in sub_chapters: for p in sc["paragraphs"]: if 'class="' in p and "]+>", "", p) if is_dash_line(plain): dash_count += 1 for seg in intro_segments: for p in seg["paragraphs"]: plain = re.sub(r"<[^>]+>", "", p) if is_dash_line(plain): dash_count += 1 if dash_count == 0: print(f" ✅ No dashed-line paragraphs in output") else: print(f" ⚠ {dash_count} dashed-line paragraphs found in output") print(f"\nDone! ✨") if __name__ == "__main__": main()