#!/usr/bin/env python3 """Extract Songs of the Vaiṣṇava Ācāryas (1972) from other.html. The book has 3 intro segments (front-pages, foreword, introduction), and 4 chapters (Parts 1–4). Output: lib/jsons/books/sva/intro.json — Book metadata lib/jsons/books/sva/intro/*.json — Intro segments lib/jsons/books/sva/chapters/*.json — Chapters (01–04) Usage: python3 scripts/sva_extract.py """ import json import os import re import sys from bs4 import BeautifulSoup, NavigableString, Tag # --------------------------------------------------------------------------- # ScaGoudy → IAST Unicode mapping # --------------------------------------------------------------------------- SCAGOUDY_TO_IAST = { "ä": "ā", "Ä": "Ā", "å": "ṛ", "Å": "Ṛ", "ñ": "ṣ", "Ñ": "Ṣ", "ö": "ṭ", "Ö": "Ṭ", "ë": "ṇ", "Ë": "Ṇ", "ò": "ḍ", "Ò": "Ḍ", "ç": "ś", "Ç": "Ś", "ì": "ṅ", "ï": "ñ", "ü": "ū", "Ü": "Ū", "é": "ī", "É": "Ī", "à": "ṁ", "ù": "ḥ", "è": "ṝ", "û": "ṁl", "ÿ": "ḷ", } def decode_scagoudy(text: str) -> str: """Replace ScaGoudy-encoded characters with proper IAST Unicode.""" for src, dst in SCAGOUDY_TO_IAST.items(): text = text.replace(src, dst) return text def decode_html_content(tag: Tag) -> str: """Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags.""" inner = "".join(str(child) for child in tag.children) inner = decode_scagoudy(inner) # Remove span tags with CSS classes (RTF conversion artifacts) inner = re.sub( r']*class="[^"]*"[^>]*>(.*?)', r"\1", inner, flags=re.DOTALL ) # Clean up Apple-tab-span leftovers (tab characters) inner = inner.replace("\t", " ") # Normalize whitespace inner = re.sub(r"\s+", " ", inner).strip() return inner def decode_text(tag: Tag) -> str: """Get plain text from a tag with ScaGoudy decoding.""" return decode_scagoudy(tag.get_text().strip()) def get_css_class(tag: Tag) -> str: """Get the first CSS class of a tag, or empty string.""" classes = tag.get("class", []) return classes[0] if classes else "" def is_empty(tag: Tag) -> bool: """Check if a tag is empty or whitespace-only.""" text = tag.get_text().strip() return not text or text == "\xa0" def is_dash_line(text: str) -> bool: """Return True if text is only dashes, spaces, and/or whitespace.""" cleaned = text.strip() if not cleaned: return False return all(c in "-—–\u2014\u2013 \t" for c in cleaned) def is_inside_table(tag: Tag) -> bool: """Check if a tag is nested inside a element.""" parent = tag.parent while parent: if parent.name == "table": return True parent = parent.parent return False # --------------------------------------------------------------------------- # CSS class categorization for SVA # --------------------------------------------------------------------------- # Book title header (bold, p12) BOOK_HEADER_CLASS = "p12" # Bookmark class — section boundaries BOOKMARK_CLASS = "p13" # Song bookmark / section bookmark within a part SONG_BOOKMARK_CLASS = "p49" # Front pages text FRONT_PAGES_CLASS = "p18" # Body paragraph classes BODY_CLASSES = frozenset(["p6", "p48", "p140", "p204", "p44"]) # Section headings (bold) SECTION_HEADING_CLASS = "p31" # Song title headings (bold) SONG_TITLE_CLASS = "p27" # Song title variant (bold italic with author) SONG_TITLE_VARIANT_CLASS = "p169" # Verse line classes (italic, centered) VERSE_PADA_CLASSES = frozenset(["p5", "p53", "p24", "p66", "p255"]) # Verse number classes (e.g. "3)", "4)", "[1]") — centered but NOT verse text VERSE_NUMBER_CLASS = "p248" # Synonyms text SYNONYMS_CLASS = "p54" # Translation text (bold) TRANSLATION_CLASS = "p56" # "SYNONYMS" header SYNONYMS_HEADER_CLASS = "p165" # "TRANSLATION" header TRANSLATION_HEADER_CLASS = "p55" # Mantra text MANTRA_CLASS = "p19" # Note text NOTE_CLASS = "p204" # Attribution ATTRIBUTION_CLASS = "p129" # Date/location DATE_CLASS = "p67" # Signature block SIGNATURE_CLASS = "p222" # Bold section header BOLD_HEADER_CLASS = "p227" # Bracketed notes BRACKET_NOTE_CLASS = "p228" # Disciplic succession cell content DISC_SUCC_CLASS = "p229" # Reference notes REFERENCE_CLASSES = frozenset( [ "p187", "p231", "p233", "p237", "p32", "p251", "p203", "p246", "p249", "p254", "p241", ] ) # Bold text classes BOLD_TEXT_CLASSES = frozenset(["p236", "p238"]) # Synonym notes SYNONYM_NOTES_CLASS = "p247" # Bold headings BOLD_HEADING_CLASSES = frozenset(["p224", "p223"]) # Part heading (bold, for song subsections) PART_HEADING_CLASS = "p22" # Sub-translation / inline bold classes INLINE_BOLD_CLASSES = frozenset(["p86", "p240", "p242", "p243", "p245", "p258"]) # Verse number marker VERSE_NUMBER_CLASS = "p30" # Song commentary / purport headings PURPORT_HEADING_CLASS = "p96" # Purport body text PURPORT_BODY_CLASS = "p140" # Additional verse line classes in tables TABLE_VERSE_CLASSES = frozenset(["p248", "p255", "p5", "p6"]) # Verse numbering markers VERSE_MARKER_CLASSES = frozenset(["p252", "p253"]) # Classes to skip entirely (empty/break/decorative) SKIP_CLASSES = frozenset( [ "p15", "p120", "p225", "p226", "p219", "p230", "p232", "p234", "p135", "p183", "p172", "p179", "p260", ] ) # --------------------------------------------------------------------------- # Extraction helpers # --------------------------------------------------------------------------- def find_sva_boundaries(all_elements: list) -> tuple: """Find the start and end element indices of the SVA content. Start:

containing "Songs of the Vaiñëava Äcäryas" End:

containing "The Hare" (Cook Book) """ start_idx = None end_idx = len(all_elements) for i, elem in enumerate(all_elements): if elem.name != "p": continue cls = get_css_class(elem) text = decode_text(elem) # Find start if start_idx is None: if cls == BOOK_HEADER_CLASS and "Songs of the Vai" in text: start_idx = i continue # Find end: the next p12 header (Hare Kṛṣṇa Cook Book) if cls == BOOK_HEADER_CLASS: end_idx = i break if start_idx is None: print("ERROR: Could not find 'Songs of the Vaiṣṇava Ācāryas' header!") sys.exit(1) return start_idx, end_idx def find_sva_bookmarks(all_elements: list, start_idx: int, end_idx: int) -> list: """Find all SVA bookmark/section tags in the SVA range. Looks for p13 class with "SVA" text, and p49 class with "SVA" text. Returns list of (element_index, bookmark_text, bookmark_type). bookmark_type is "p13" or "p49". """ bookmarks = [] for i in range(start_idx, end_idx): elem = all_elements[i] if elem.name != "p": continue cls = get_css_class(elem) if cls not in (BOOKMARK_CLASS, SONG_BOOKMARK_CLASS): continue text = decode_text(elem).strip() if text.startswith("SVA"): bookmarks.append((i, text, cls)) return bookmarks def process_table_t3(table: Tag) -> list: """Process a t3 table (side-by-side Bengali verse + English translation). Returns a list of HTML paragraph strings. """ result = [] for tr in table.find_all("tr"): tds = tr.find_all("td") if len(tds) < 2: # Single-cell row — just extract content for td in tds: for p in td.find_all("p"): html = decode_html_content(p) if html and html.strip() and html.strip() != "\xa0": result.append(html) continue # Left cell: Bengali verse lines left_ps = tds[0].find_all("p") verse_lines = [] for p in left_ps: html = decode_html_content(p) if html and html.strip() and html.strip() != "\xa0": # Strip tags — we'll re-wrap stripped = re.sub(r"", "", html) if stripped.strip(): verse_lines.append(stripped.strip()) if verse_lines: joined = "
".join(verse_lines) result.append(f'

{joined}
') # Right cell: English translation right_ps = tds[1].find_all("p") for p in right_ps: html = decode_html_content(p) if html and html.strip() and html.strip() != "\xa0": result.append(html) return result def process_table_t2(table: Tag) -> list: """Process a t2 table (disciplic succession table). Extract cell content as regular paragraphs. """ result = [] for tr in table.find_all("tr"): for td in tr.find_all("td"): for p in td.find_all("p"): html = decode_html_content(p) if html and html.strip() and html.strip() != "\xa0": result.append(html) return result def extract_paragraphs( all_elements: list, start_idx: int, end_idx: int, ) -> list: """Extract paragraphs for a section (chapter, intro segment). Handles all CSS classes within the SVA content. Handles
and
elements. Groups consecutive verse pāda classes into centered italic divs. """ result = [] verse_accumulator = [] # Track which

tags are inside tables we've already processed processed_table_ps = set() def flush_verse(): if verse_accumulator: joined = "
".join(verse_accumulator) result.append(f'

{joined}
') verse_accumulator.clear() for i in range(start_idx, end_idx): elem = all_elements[i] # Handle tables if elem.name == "table": flush_verse() table_classes = elem.get("class", []) # Mark all

inside this table as processed for p in elem.find_all("p"): processed_table_ps.add(id(p)) if "t3" in table_classes: table_paras = process_table_t3(elem) result.extend(table_paras) elif "t2" in table_classes: table_paras = process_table_t2(elem) result.extend(table_paras) else: # Generic table — extract all paragraph content for p in elem.find_all("p"): html = decode_html_content(p) if html and html.strip(): result.append(html) continue if elem.name != "p": continue # Skip if this

was already processed as part of a table if id(elem) in processed_table_ps: continue # Skip if inside a table (safety check) if is_inside_table(elem): continue cls = get_css_class(elem) # Skip empty paragraphs if is_empty(elem): continue # Skip noise classes if cls in SKIP_CLASSES: continue # Skip bookmark tags (p13, p49) — these are navigation markers if cls == BOOKMARK_CLASS: continue if cls == SONG_BOOKMARK_CLASS: continue # Skip book title header (p12) if cls == BOOK_HEADER_CLASS: continue text = decode_text(elem) # Check for dash lines — skip them if is_dash_line(text): continue # Sanskrit verse pādas (p5, p53, p24, p248, p255) — accumulate if cls in VERSE_PADA_CLASSES: html = decode_html_content(elem) if html: stripped = re.sub(r"", "", html) if stripped.strip(): verse_accumulator.append(stripped.strip()) continue # Non-verse paragraph — flush any accumulated verse first flush_verse() # Front pages text (p18) if cls == FRONT_PAGES_CLASS: html = decode_html_content(elem) if html: result.append(html) continue # Section heading (p31) — bold if cls == SECTION_HEADING_CLASS: html = decode_html_content(elem) if html: if not html.strip().startswith(""): html = f"{html}" result.append(html) continue # Song title (p27) — bold if cls == SONG_TITLE_CLASS: html = decode_html_content(elem) if html: if not html.strip().startswith(""): html = f"{html}" result.append(html) continue # Song title variant (p169) — bold italic with author if cls == SONG_TITLE_VARIANT_CLASS: html = decode_html_content(elem) if html: result.append(html) continue # Synonyms (p54) if cls == SYNONYMS_CLASS: html = decode_html_content(elem) if html: result.append(html) continue # Verse number heading (p248) — e.g. "3)", "4)", "[1]" — center it if cls == VERSE_NUMBER_CLASS: html = decode_html_content(elem) if html: result.append(f'

{html}
') continue # Translation (p56) — bold, centered if cls == TRANSLATION_CLASS: html = decode_html_content(elem) if html: result.append(f'
{html}
') continue # SYNONYMS header (p165) if cls == SYNONYMS_HEADER_CLASS: html = decode_html_content(elem) if html: result.append(html) continue # TRANSLATION header (p55) if cls == TRANSLATION_HEADER_CLASS: html = decode_html_content(elem) if html: result.append(html) continue # Mantra text (p19) if cls == MANTRA_CLASS: html = decode_html_content(elem) if html: result.append(html) continue # Attribution (p129) if cls == ATTRIBUTION_CLASS: html = decode_html_content(elem) if html: result.append(html) continue # Date/location (p67) if cls == DATE_CLASS: html = decode_html_content(elem) if html: result.append(html) continue # Signature block (p222) if cls == SIGNATURE_CLASS: html = decode_html_content(elem) if html: result.append(html) continue # Bold section header (p227) if cls == BOLD_HEADER_CLASS: html = decode_html_content(elem) if html: if not html.strip().startswith(""): html = f"{html}" result.append(html) continue # Bracketed notes (p228) if cls == BRACKET_NOTE_CLASS: html = decode_html_content(elem) if html: result.append(html) continue # Disciplic succession cell content (p229) if cls == DISC_SUCC_CLASS: html = decode_html_content(elem) if html: result.append(html) continue # Reference notes if cls in REFERENCE_CLASSES: html = decode_html_content(elem) if html: result.append(html) continue # Bold text classes if cls in BOLD_TEXT_CLASSES: html = decode_html_content(elem) if html: result.append(html) continue # Synonym notes (p247) if cls == SYNONYM_NOTES_CLASS: html = decode_html_content(elem) if html: result.append(html) continue # Bold headings (p224, p223) if cls in BOLD_HEADING_CLASSES: html = decode_html_content(elem) if html: if not html.strip().startswith(""): html = f"{html}" result.append(html) continue # Part heading (p22) — e.g. "part 1", "part 2" if cls == PART_HEADING_CLASS: html = decode_html_content(elem) if html: result.append(html) continue # Inline bold classes (p86, p240, p242, p243, p245, p258) if cls in INLINE_BOLD_CLASSES: html = decode_html_content(elem) if html: result.append(html) continue # Verse number (p30) if cls == VERSE_NUMBER_CLASS: html = decode_html_content(elem) if html: result.append(html) continue # Purport heading (p96) if cls == PURPORT_HEADING_CLASS: html = decode_html_content(elem) if html: result.append(html) continue # Purport body (p140) if cls == PURPORT_BODY_CLASS: html = decode_html_content(elem) if html: result.append(html) continue # Verse markers (p252, p253) if cls in VERSE_MARKER_CLASSES: html = decode_html_content(elem) if html: result.append(html) continue # Body paragraphs (p6, p48, p140, p204, p44) if cls in BODY_CLASSES: html = decode_html_content(elem) if html: # Detect verses disguised as body: starts with and has
stripped_html = html.strip() if stripped_html.startswith("") and "", "", stripped_html) result.append( f'
{inner}
' ) else: result.append(html) continue # Any other class — try to extract (be permissive) html = decode_html_content(elem) if html and not is_dash_line(text): result.append(html) # Flush any trailing verse flush_verse() return result # --------------------------------------------------------------------------- # Main extraction logic # --------------------------------------------------------------------------- def extract_sva(soup: BeautifulSoup) -> dict: """Extract the entire Songs of the Vaiṣṇava Ācāryas from the parsed HTML. Returns a dict with: "intro_segments": list of intro segment dicts "chapters": list of chapter dicts "chapter_listing": list of {number, title} for intro.json """ # Get all top-level

and

elements all_elements = list(soup.find_all(["p", "table"])) # Step 1: Find SVA boundaries sva_start, sva_end = find_sva_boundaries(all_elements) print(f" SVA content: elements {sva_start} to {sva_end}") print(f" Total elements in range: {sva_end - sva_start}") # Step 2: Find all SVA bookmarks bookmarks = find_sva_bookmarks(all_elements, sva_start, sva_end) print(f" Found {len(bookmarks)} SVA bookmarks") for pos, text, cls in bookmarks[:20]: print(f" Elem ~{pos} [{cls}]: {text[:80]}") if len(bookmarks) > 20: print(f" ... and {len(bookmarks) - 20} more") # Step 3: Categorize bookmarks into sections # p13 bookmarks define major sections: # "SVA: Foreword" → foreword # "SVA: Introduction" → introduction # "SVA 2: ..." → Part 2 start # "SVA 3: ..." → Part 3 start # "SVA 4: ..." → Part 4 start foreword_pos = None introduction_pos = None part2_pos = None part3_pos = None part4_pos = None # p49 bookmarks define songs within parts: # "SVA 1: ..." → songs in Part 1 # "SVA 2: ..." → songs in Part 2 # "SVA 3: ..." → songs in Part 3 # "SVA 4: ..." → songs in Part 4 # Also find the first SVA 1: bookmark (start of Part 1) first_part1_pos = None # Find "SVA: A Glimpse..." bookmark in intro glimpse_pos = None for pos, text, cls in bookmarks: if cls == BOOKMARK_CLASS: # p13 if text.strip() == "SVA: Foreword": foreword_pos = pos elif text.strip() == "SVA: Introduction": introduction_pos = pos elif text.strip().startswith("SVA 2:"): part2_pos = pos elif text.strip().startswith("SVA 3:"): part3_pos = pos elif text.strip().startswith("SVA 4:"): part4_pos = pos if cls == SONG_BOOKMARK_CLASS: # p49 if text.strip().startswith("SVA 1:") and first_part1_pos is None: first_part1_pos = pos if "Glimpse" in text or "Life of" in text: glimpse_pos = pos print(f"\n Section boundaries:") print(f" Book start: elem {sva_start}") print( f" Foreword: {'elem ' + str(foreword_pos) if foreword_pos else 'NOT FOUND'}" ) print( f" Introduction: {'elem ' + str(introduction_pos) if introduction_pos else 'NOT FOUND'}" ) print( f" Part 1 start: {'elem ' + str(first_part1_pos) if first_part1_pos else 'NOT FOUND'}" ) print( f" Part 2 start: {'elem ' + str(part2_pos) if part2_pos else 'NOT FOUND'}" ) print( f" Part 3 start: {'elem ' + str(part3_pos) if part3_pos else 'NOT FOUND'}" ) print( f" Part 4 start: {'elem ' + str(part4_pos) if part4_pos else 'NOT FOUND'}" ) print(f" Book end: elem {sva_end}") # Step 4: Extract intro segments # Front pages: from book header to Foreword bookmark front_pages_start = sva_start front_pages_end = ( foreword_pos if foreword_pos else (introduction_pos or first_part1_pos or sva_end) ) fp_paras = extract_paragraphs(all_elements, front_pages_start, front_pages_end) front_pages = { "slug": "front-pages", "title": "Front Pages", "type": "intro-segment", "book": "sva", "order": 1, "paragraphs": fp_paras, } # Foreword: from Foreword bookmark to Introduction bookmark foreword = None if foreword_pos is not None: fw_start = foreword_pos + 1 fw_end = introduction_pos if introduction_pos else (first_part1_pos or sva_end) fw_paras = extract_paragraphs(all_elements, fw_start, fw_end) foreword = { "slug": "foreword", "title": "Foreword", "type": "intro-segment", "book": "sva", "order": 2, "paragraphs": fw_paras, } # Introduction: from Introduction bookmark to Part 1 start introduction = None if introduction_pos is not None: intro_start = introduction_pos + 1 intro_end = first_part1_pos if first_part1_pos else (part2_pos or sva_end) intro_paras = extract_paragraphs(all_elements, intro_start, intro_end) introduction = { "slug": "introduction", "title": "Introduction", "type": "intro-segment", "book": "sva", "order": 3, "paragraphs": intro_paras, } # Step 5: Extract chapters (Parts 1–4) chapter_boundaries = [] if first_part1_pos is not None: ch1_end = part2_pos if part2_pos else sva_end chapter_boundaries.append((1, "Prayers and Praṇāmas", first_part1_pos, ch1_end)) if part2_pos is not None: ch2_end = part3_pos if part3_pos else sva_end chapter_boundaries.append( (2, "Songs of Śrīla Bhaktivinoda Ṭhākura", part2_pos, ch2_end) ) if part3_pos is not None: ch3_end = part4_pos if part4_pos else sva_end chapter_boundaries.append( (3, "Songs of Śrīla Narottama dāsa Ṭhākura", part3_pos, ch3_end) ) if part4_pos is not None: chapter_boundaries.append((4, "Songs by other Ācāryas", part4_pos, sva_end)) chapters = [] for ch_num, ch_title, ch_start, ch_end in chapter_boundaries: # Start extraction from the element after the p13 bookmark (for parts 2-4) # or from the p49 bookmark itself for part 1 content_start = ch_start if ch_num >= 2: # For parts 2-4, skip the p13 bookmark and the p31 heading that follows content_start = ch_start + 1 paragraphs = extract_paragraphs(all_elements, content_start, ch_end) ch = { "book": "sva", "chapter": ch_num, "title": ch_title, "type": "chapter", "paragraphs": paragraphs, } chapters.append(ch) # Step 6: Build results intro_segments = [front_pages] if foreword: intro_segments.append(foreword) if introduction: intro_segments.append(introduction) chapter_listing = [ {"number": ch["chapter"], "title": ch["title"]} for ch in chapters ] return { "intro_segments": intro_segments, "chapters": chapters, "chapter_listing": chapter_listing, } # --------------------------------------------------------------------------- # Output # --------------------------------------------------------------------------- def write_json(data, filepath): """Write data as formatted JSON to filepath, creating directories as needed.""" os.makedirs(os.path.dirname(filepath), exist_ok=True) with open(filepath, "w", encoding="utf-8") as f: json.dump(data, f, ensure_ascii=False, indent=2) def main(): base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) html_path = os.path.join(base_dir, "public", "html", "other.html") output_base = os.path.join(base_dir, "lib", "jsons", "books", "sva") if not os.path.exists(html_path): print(f"ERROR: {html_path} not found!") sys.exit(1) print(f"Reading {html_path}...") with open(html_path, "r", encoding="utf-8") as f: soup = BeautifulSoup(f, "html.parser") print("Extracting Songs of the Vaiṣṇava Ācāryas (1972)...\n") result = extract_sva(soup) intro_segments = result["intro_segments"] chapters = result["chapters"] chapter_listing = result["chapter_listing"] # --- Write intro.json --- intro_meta = { "book": "sva", "title": "Songs of the Vaiṣṇava Ācāryas", "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda", "year": 1972, "description": "Bengali devotional songs by the great Vaiṣṇava teachers, with English translations and commentary.", "totalChapters": 4, "chapters": chapter_listing, } intro_path = os.path.join(output_base, "intro.json") write_json(intro_meta, intro_path) print(f"\n Written: {intro_path}") # --- Write intro segments --- print(f"\n--- Intro Segments ---") for seg in intro_segments: filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json") write_json(seg, filepath) print( f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}" ) # --- Write chapters --- print(f"\n--- Chapters ---") for ch in chapters: ch_num_str = str(ch["chapter"]).zfill(2) filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json") write_json(ch, filepath) print( f" Part {ch['chapter']}: {len(ch['paragraphs']):>3} paragraphs — " f"{ch['title']}" ) # --- Summary --- total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments) total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters) total_paras = total_intro_paras + total_chapter_paras print(f"\n{'=' * 60}") print(f" EXTRACTION SUMMARY") print(f"{'=' * 60}") print(f" Intro segments: {len(intro_segments)}") for seg in intro_segments: print( f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs" ) print(f" Chapters (Parts): {len(chapters)}") for ch in chapters: print( f" Part {ch['chapter']}: {len(ch['paragraphs'])} paragraphs — {ch['title']}" ) print(f" Total paragraphs: {total_paras}") print(f" Intro: {total_intro_paras}") print(f" Chapters: {total_chapter_paras}") if chapters: print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}") print(f" Output directory: {output_base}/") # --- Samples --- print(f"\n--- Sample: Part 1 (Prayers and Praṇāmas) ---") if chapters: ch1 = chapters[0] for i, p in enumerate(ch1["paragraphs"][:5]): preview = re.sub(r"<[^>]+>", "", p)[:120] print(f" [{i}] {preview}...") if len(chapters) >= 2: print(f"\n--- Sample: Part 2 (Songs of Bhaktivinoda Ṭhākura) ---") ch2 = chapters[1] for i, p in enumerate(ch2["paragraphs"][:5]): preview = re.sub(r"<[^>]+>", "", p)[:120] print(f" [{i}] {preview}...") if len(chapters) >= 4: print(f"\n--- Sample: Part 4 (Songs by other Ācāryas) ---") ch4 = chapters[3] for i, p in enumerate(ch4["paragraphs"][:5]): preview = re.sub(r"<[^>]+>", "", p)[:120] print(f" [{i}] {preview}...") # --- Quality Checks --- print(f"\n--- Quality Checks ---") # Check all 4 parts present expected_chapters = set(range(1, 5)) actual_chapters = set(ch["chapter"] for ch in chapters) missing = expected_chapters - actual_chapters extra = actual_chapters - expected_chapters if missing: print(f" ⚠ Missing parts: {sorted(missing)}") if extra: print(f" ⚠ Extra parts: {sorted(extra)}") if not missing and not extra: print(f" ✅ All 4 parts present") # Check intro segments expected_intros = {"front-pages", "foreword", "introduction"} actual_intros = {s["slug"] for s in intro_segments} missing_intros = expected_intros - actual_intros if missing_intros: print(f" ⚠ Missing intro segments: {missing_intros}") else: print(f" ✅ All 3 intro segments present") # Check empty chapters empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0] if empty_chapters: print(f" ⚠ Empty parts (0 paragraphs): {empty_chapters}") else: print(f" ✅ No empty parts") # Check ScaGoudy decoding encoded_chars = set("äåöëòçüéàùè") suspect_count = 0 all_sections = chapters + intro_segments for section in all_sections: for p in section["paragraphs"]: plain = re.sub(r"<[^>]+>", "", p) for c in encoded_chars: if c in plain: suspect_count += 1 break if suspect_count == 0: print(f" ✅ No suspected undecoded ScaGoudy characters found") else: print( f" ℹ {suspect_count} paragraphs contain characters that may be undecoded ScaGoudy" ) # Check verse blocks verse_block_count = 0 for ch in chapters: for p in ch["paragraphs"]: if '
' in p: verse_block_count += 1 print(f" ℹ {verse_block_count} verse blocks in chapters") # Check table-derived content table_verse_count = 0 for ch in chapters: for p in ch["paragraphs"]: if '
' in p and "
" in p: table_verse_count += 1 print(f" ℹ {table_verse_count} multi-line verse blocks (likely from tables)") # ScaGoudy decoding spot check print(f"\n--- ScaGoudy Decoding Spot Check ---") if intro_segments: fw_segs = [s for s in intro_segments if s["slug"] == "foreword"] if fw_segs: fw_text = " ".join(fw_segs[0]["paragraphs"][:10]) if "Prabhupāda" in fw_text: print(f" ✅ 'Prabhupāda' correctly decoded in Foreword") elif "Prabhupäda" in fw_text: print(f" ⚠ ScaGoudy decoding incomplete for 'Prabhupäda'") if "Kṛṣṇa" in fw_text: print(f" ✅ 'Kṛṣṇa' correctly decoded in Foreword") elif "Kåñëa" in fw_text: print(f" ⚠ 'Kåñëa' still encoded in Foreword") # Sample verse blocks print(f"\n--- Sample Verse Blocks ---") shown = 0 for ch in chapters: for p in ch["paragraphs"]: if '
' in p: preview = re.sub(r"<[^>]+>", "", p)[:120] print(f" Part {ch['chapter']}: {preview}...") shown += 1 if shown >= 5: break if shown >= 5: break # Song title check print(f"\n--- Song Titles in Chapters ---") song_count = 0 sample_songs = [] for ch in chapters: ch_songs = 0 for p in ch["paragraphs"]: # Bold italic song titles if "" in p and "" in p: plain = re.sub(r"<[^>]+>", "", p).strip() if len(plain) < 100: ch_songs += 1 if len(sample_songs) < 8: sample_songs.append(f" Part {ch['chapter']}: {plain}") song_count += ch_songs print(f" Part {ch['chapter']}: ~{ch_songs} song titles") print(f" Total song titles: ~{song_count}") for s in sample_songs: print(s) print(f"\nDone! ✨") if __name__ == "__main__": main()