#!/usr/bin/env python3 """Extract Teachings of Lord Kapila, the Son of Devahūti (1977) from other.html. The book has 5 intro/back-matter segments and 18 chapters. Intro segments: front-pages (order 1) foreword (order 2) about-the-author (order 97) credits (order 98) back-flap (order 99) Output: lib/jsons/books/tlk/intro.json — Book metadata lib/jsons/books/tlk/intro/*.json — Intro segments lib/jsons/books/tlk/chapters/*.json — Chapters (01–18) Usage: python3 scripts/tlk_extract.py """ import json import os import re import sys from bs4 import BeautifulSoup, Tag # --------------------------------------------------------------------------- # ScaGoudy → IAST Unicode mapping # --------------------------------------------------------------------------- SCAGOUDY_TO_IAST = { "ä": "ā", "Ä": "Ā", "å": "ṛ", "Å": "Ṛ", "ñ": "ṣ", "Ñ": "Ṣ", "ö": "ṭ", "Ö": "Ṭ", "ë": "ṇ", "Ë": "Ṇ", "ò": "ḍ", "Ò": "Ḍ", "ç": "ś", "Ç": "Ś", "ì": "ṅ", "ï": "ñ", "ü": "ū", "Ü": "Ū", "é": "ī", "É": "Ī", "à": "ṁ", "ù": "ḥ", "è": "ṝ", "û": "ṁl", "ÿ": "ḷ", } def decode_scagoudy(text: str) -> str: """Replace ScaGoudy-encoded characters with proper IAST Unicode.""" for src, dst in SCAGOUDY_TO_IAST.items(): text = text.replace(src, dst) return text def decode_html_content(tag: Tag) -> str: """Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags.""" inner = "".join(str(child) for child in tag.children) inner = decode_scagoudy(inner) # Remove span tags with CSS classes (RTF conversion artifacts) inner = re.sub( r']*class="[^"]*"[^>]*>(.*?)', r"\1", inner, flags=re.DOTALL ) # Clean up Apple-tab-span leftovers (tab characters) inner = inner.replace("\t", " ") # Normalize whitespace inner = re.sub(r"\s+", " ", inner).strip() return inner def decode_text(tag: Tag) -> str: """Get plain text from a tag with ScaGoudy decoding.""" return decode_scagoudy(tag.get_text().strip()) def get_css_class(tag: Tag) -> str: """Get the first CSS class of a tag, or empty string.""" classes = tag.get("class", []) return classes[0] if classes else "" def is_empty(tag: Tag) -> bool: """Check if a tag is empty or whitespace-only.""" text = tag.get_text().strip() return not text or text == "\xa0" def is_dash_line(text: str) -> bool: """Return True if text is only dashes, spaces, and/or whitespace.""" cleaned = text.strip() if not cleaned: return False return all(c in "-—–\u2014\u2013 \t" for c in cleaned) # --------------------------------------------------------------------------- # CSS class categorization for TLK # --------------------------------------------------------------------------- # Book title header class (bold,

) BOOK_HEADER_CLASS = "p12" # Bookmark class — used for section boundary detection BOOKMARK_CLASS = "p13" # Chapter heading (bold, e.g. "CHAPTER ONE\nThe Purpose of Lord Kapila's Advent") CHAPTER_HEADING_CLASS = "p27" # Section heading (bold, e.g. "Foreword", "Credits") SECTION_HEADING_CLASS = "p31" # Body paragraphs — THE main content class BODY_CLASS = "p6" # Sanskrit verse text (italic, multi-line with
) SANSKRIT_VERSE_CLASS = "p53" # Verse translation/reference text (after Sanskrit verses, mid-purport quotes) VERSE_TRANSLATION_CLASS = "p44" # Internal verse bookmark (e.g. "TLK Vs 1", "TLK Vs 2") VERSE_BOOKMARK_CLASS = "p49" # TEXT N / TRANSLATION / PURPORT headers (bold) STRUCTURED_HEADER_CLASS = "p55" # Translation text (bold — verse translation after TRANSLATION header) TRANSLATION_TEXT_CLASS = "p211" # Verse reference (e.g. [Śrīmad-Bhāgavatam 3.25.1]) VERSE_REFERENCE_CLASS = "p212" # Publication info text PUBLICATION_INFO_CLASS = "p128" # Attribution line (e.g. "—The Publishers") ATTRIBUTION_CLASS = "p129" # Block quote (bold, testimonials in Credits section) BLOCKQUOTE_CLASS = "p47" # Attribution for block quotes QUOTE_ATTRIBUTION_CLASS = "p42" # Jacket/design credits JACKET_CREDITS_CLASS = "p3" # Classes to skip entirely (empty/break) SKIP_CLASSES = frozenset(["p15"]) # --------------------------------------------------------------------------- # Extraction helpers # --------------------------------------------------------------------------- def find_tlk_boundaries(all_paragraphs: list) -> tuple: """Find the start and end paragraph indices of the TLK content. Start:

containing "Teachings of Lord Kapila" End: next

which is "Teachings of Queen Kunté" (or similar) """ start_idx = None end_idx = len(all_paragraphs) for i, p in enumerate(all_paragraphs): cls = get_css_class(p) text = decode_text(p) # Find start if start_idx is None: if cls == BOOK_HEADER_CLASS and "Teachings of Lord Kapila" in text: start_idx = i continue # Find end: the next p12 book header after our start if cls == BOOK_HEADER_CLASS: end_idx = i break if start_idx is None: print("ERROR: Could not find 'Teachings of Lord Kapila' header in other.html!") sys.exit(1) return start_idx, end_idx def find_all_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list: """Find all TLK bookmark tags in the TLK range. Bookmarks are p13 class with text starting with "TLK". Returns list of (paragraph_index, bookmark_text). """ bookmarks = [] for i in range(start_idx, end_idx): p = all_paragraphs[i] cls = get_css_class(p) if cls != BOOKMARK_CLASS: continue text = decode_text(p).strip() if text.startswith("TLK"): bookmarks.append((i, text)) return bookmarks def parse_chapter_bookmark(text: str): """Parse a chapter bookmark like 'TLK 1: The Purpose of Lord Kapila's Advent'. Returns (chapter_number, title) or None if not a chapter bookmark. """ m = re.match(r"TLK\s+(\d+):\s+(.+)", text.strip()) if m: num = int(m.group(1)) title = m.group(2).strip() return num, title return None def parse_intro_bookmark(text: str): """Check if this is a front-pages or foreword bookmark. Returns the slug or None. """ stripped = text.strip() # "TLK Into" is the front-pages bookmark (note: "Into" not "Intro") if stripped == "TLK Into": return "front-pages" if stripped == "TLK Foreword": return "foreword" return None def parse_backmatter_bookmark(text: str): """Check if this is a back-matter bookmark. Returns the slug or None. """ stripped = text.strip() if stripped == "TLK The Author": return "about-the-author" if stripped == "TLK Credits": return "credits" if stripped == "TLK Back Flap": return "back-flap" return None INTRO_SLUG_ORDER = { "front-pages": 1, "foreword": 2, } BACKMATTER_SLUG_ORDER = { "about-the-author": 97, "credits": 98, "back-flap": 99, } INTRO_SLUG_TITLE = { "front-pages": "Front Pages", "foreword": "Foreword", "about-the-author": "About the Author", "credits": "Credits", "back-flap": "Back Flap", } def extract_paragraphs( all_paragraphs: list, start_idx: int, end_idx: int, ) -> list: """Extract paragraphs for a section (chapter, intro segment). Handles all CSS classes within the TLK content. Handles p53 Sanskrit verses as centered italic divs. Handles structured verse headers (TEXT N, TRANSLATION, PURPORT). """ result = [] for i in range(start_idx, end_idx): p = all_paragraphs[i] cls = get_css_class(p) # Skip empty paragraphs if is_empty(p): continue # Skip noise/break classes if cls in SKIP_CLASSES: continue # Skip bookmark tags (p13) if cls == BOOKMARK_CLASS: continue # Skip book title header (p12) if cls == BOOK_HEADER_CLASS: continue # Skip internal verse bookmarks (p49 containing "TLK Vs") if cls == VERSE_BOOKMARK_CLASS: text = decode_text(p) if "TLK Vs" in text or "TLK" in text: continue # Check for dash lines — skip them text = decode_text(p) if is_dash_line(text): continue # Sanskrit verse text (p53) — italic, multi-line with
if cls == SANSKRIT_VERSE_CLASS: html = decode_html_content(p) if html: # Strip outer tags, then re-wrap in centered italic div inner = re.sub(r"", "", html) result.append(f'

{inner}
') continue # Structured headers: TEXT N, TRANSLATION, PURPORT (p55) if cls == STRUCTURED_HEADER_CLASS: html = decode_html_content(p) if html: # These are already bold in the source; ensure bold wrapping if not html.strip().startswith(""): html = f"{html}" result.append(html) continue # Translation text (p211) — bold verse translation if cls == TRANSLATION_TEXT_CLASS: html = decode_html_content(p) if html: if not html.strip().startswith(""): html = f"{html}" result.append(html) continue # Verse reference (p212) — e.g. [Śrīmad-Bhāgavatam 3.25.1] if cls == VERSE_REFERENCE_CLASS: html = decode_html_content(p) if html: result.append(html) continue # Body paragraphs (p6) — the main content class if cls == BODY_CLASS: html = decode_html_content(p) if html: # Detect Sanskrit verses disguised as body paragraphs: # they start with and contain
(multi-line italic = verse) stripped_html = html.strip() if stripped_html.startswith("") and "", "", stripped_html) result.append( f'
{inner}
' ) else: result.append(html) continue # Verse translation text (p44) — after Sanskrit verses mid-purport if cls == VERSE_TRANSLATION_CLASS: html = decode_html_content(p) if html: result.append(html) continue # Chapter heading (p27) — bold chapter title if cls == CHAPTER_HEADING_CLASS: html = decode_html_content(p) if html: if not html.strip().startswith(""): html = f"{html}" result.append(html) continue # Section heading (p31) — bold section title if cls == SECTION_HEADING_CLASS: html = decode_html_content(p) if html: if not html.strip().startswith(""): html = f"{html}" result.append(html) continue # Publication info (p128) if cls == PUBLICATION_INFO_CLASS: html = decode_html_content(p) if html: result.append(html) continue # Attribution (p129) — e.g. "—The Publishers" if cls == ATTRIBUTION_CLASS: html = decode_html_content(p) if html: result.append(html) continue # Block quotes (p47) — testimonials in Credits section if cls == BLOCKQUOTE_CLASS: html = decode_html_content(p) if html: result.append(html) continue # Quote attribution (p42) if cls == QUOTE_ATTRIBUTION_CLASS: html = decode_html_content(p) if html: result.append(html) continue # Jacket/design credits (p3) if cls == JACKET_CREDITS_CLASS: html = decode_html_content(p) if html: result.append(html) continue # Any other class — try to extract content (be permissive) html = decode_html_content(p) if html and not is_dash_line(text): result.append(html) return result # --------------------------------------------------------------------------- # Main extraction logic # --------------------------------------------------------------------------- def extract_tlk(soup: BeautifulSoup) -> dict: """Extract the entire Teachings of Lord Kapila from the parsed HTML. Returns a dict with: "intro_segments": list of intro segment dicts "chapters": list of chapter dicts "chapter_listing": list of {number, title} for intro.json """ all_paragraphs = list(soup.find_all("p")) # Step 1: Find TLK boundaries tlk_start, tlk_end = find_tlk_boundaries(all_paragraphs) print(f" TLK content: paragraphs {tlk_start} to {tlk_end}") print(f" Total paragraphs in range: {tlk_end - tlk_start}") # Step 2: Find all bookmarks bookmarks = find_all_bookmarks(all_paragraphs, tlk_start, tlk_end) print(f" Found {len(bookmarks)} TLK bookmarks") for pos, text in bookmarks: print(f" Para ~{pos}: {text}") # Step 3: Categorize bookmarks intro_bookmarks = {} # slug -> paragraph index backmatter_bookmarks = {} # slug -> paragraph index chapter_bookmarks = [] # (para_idx, chapter_num, title) for pos, text in bookmarks: # Check chapter ch = parse_chapter_bookmark(text) if ch: chapter_bookmarks.append((pos, ch[0], ch[1])) continue # Check intro (front-pages, foreword) intro_slug = parse_intro_bookmark(text) if intro_slug: intro_bookmarks[intro_slug] = pos continue # Check back matter (about-the-author, credits, back-flap) bm_slug = parse_backmatter_bookmark(text) if bm_slug: backmatter_bookmarks[bm_slug] = pos continue print(f" WARNING: Unrecognized bookmark: {text}") print(f" Intro bookmarks: {list(intro_bookmarks.keys())}") print(f" Chapter bookmarks: {len(chapter_bookmarks)}") print(f" Back matter bookmarks: {list(backmatter_bookmarks.keys())}") # Build an ordered list of all section start positions for boundary computation all_section_starts = sorted( list(intro_bookmarks.values()) + [pos for pos, _, _ in chapter_bookmarks] + list(backmatter_bookmarks.values()) ) def next_section_after(pos): """Return the next section start after `pos`, or tlk_end.""" for s in all_section_starts: if s > pos: return s return tlk_end # Step 4: Extract intro segments (front-pages, foreword) intro_segments = [] for slug in ["front-pages", "foreword"]: if slug in intro_bookmarks: bm_pos = intro_bookmarks[slug] content_start = bm_pos + 1 content_end = next_section_after(bm_pos) paras = extract_paragraphs(all_paragraphs, content_start, content_end) seg = { "slug": slug, "title": INTRO_SLUG_TITLE[slug], "type": "intro-segment", "book": "tlk", "order": INTRO_SLUG_ORDER[slug], "paragraphs": paras, } intro_segments.append(seg) # Step 5: Extract chapters 1–18 chapters = [] for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_bookmarks): content_start = ch_pos + 1 content_end = next_section_after(ch_pos) paragraphs = extract_paragraphs(all_paragraphs, content_start, content_end) ch = { "book": "tlk", "chapter": ch_num, "title": ch_title, "type": "chapter", "paragraphs": paragraphs, } chapters.append(ch) # Step 6: Extract back matter segments (about-the-author, credits, back-flap) backmatter_segments = [] for slug in ["about-the-author", "credits", "back-flap"]: if slug in backmatter_bookmarks: bm_pos = backmatter_bookmarks[slug] content_start = bm_pos + 1 content_end = next_section_after(bm_pos) paras = extract_paragraphs(all_paragraphs, content_start, content_end) seg = { "slug": slug, "title": INTRO_SLUG_TITLE[slug], "type": "intro-segment", "book": "tlk", "order": BACKMATTER_SLUG_ORDER[slug], "paragraphs": paras, } backmatter_segments.append(seg) # Combine all intro segments (front + back matter) all_intro_segments = intro_segments + backmatter_segments # Step 7: Build chapter listing for intro.json chapter_listing = [ {"number": ch["chapter"], "title": ch["title"]} for ch in chapters ] return { "intro_segments": all_intro_segments, "chapters": chapters, "chapter_listing": chapter_listing, } # --------------------------------------------------------------------------- # Output # --------------------------------------------------------------------------- def write_json(data, filepath): """Write data as formatted JSON to filepath, creating directories as needed.""" os.makedirs(os.path.dirname(filepath), exist_ok=True) with open(filepath, "w", encoding="utf-8") as f: json.dump(data, f, ensure_ascii=False, indent=2) def main(): base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) html_path = os.path.join(base_dir, "public", "html", "other.html") output_base = os.path.join(base_dir, "lib", "jsons", "books", "tlk") if not os.path.exists(html_path): print(f"ERROR: {html_path} not found!") sys.exit(1) print(f"Reading {html_path}...") with open(html_path, "r", encoding="utf-8") as f: soup = BeautifulSoup(f, "html.parser") print("Extracting Teachings of Lord Kapila, the Son of Devahūti (1977)...\n") result = extract_tlk(soup) intro_segments = result["intro_segments"] chapters = result["chapters"] chapter_listing = result["chapter_listing"] # --- Write intro.json --- intro_meta = { "book": "tlk", "title": "Teachings of Lord Kapila, the Son of Devahūti", "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda", "year": 1977, "description": "Lord Kapila's teachings on Sāṅkhya philosophy and bhakti-yoga to His mother Devahūti.", "totalChapters": 18, "chapters": chapter_listing, } intro_path = os.path.join(output_base, "intro.json") write_json(intro_meta, intro_path) print(f"\n Written: {intro_path}") # --- Write intro segments --- print(f"\n--- Intro / Back-Matter Segments ---") for seg in intro_segments: filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json") write_json(seg, filepath) print( f" {seg['order']:>2}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}" ) # --- Write chapters --- print(f"\n--- Chapters ---") for ch in chapters: ch_num_str = str(ch["chapter"]).zfill(2) filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json") write_json(ch, filepath) print( f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — " f"{ch['title']}" ) # --- Summary --- total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments) total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters) total_paras = total_intro_paras + total_chapter_paras print(f"\n{'=' * 60}") print(f" EXTRACTION SUMMARY") print(f"{'=' * 60}") print(f" Intro/back-matter segments: {len(intro_segments)}") for seg in intro_segments: print( f" {seg['order']:>2}. {seg['title']}: {len(seg['paragraphs'])} paragraphs" ) print(f" Chapters: {len(chapters)}") print(f" Total paragraphs: {total_paras}") print(f" Intro/back: {total_intro_paras}") print(f" Chapters: {total_chapter_paras}") if chapters: print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}") print(f" Output directory: {output_base}/") # --- Samples --- print(f"\n--- Sample: Chapter 1 ---") if chapters: ch1 = chapters[0] for i, p in enumerate(ch1["paragraphs"][:5]): preview = re.sub(r"<[^>]+>", "", p)[:120] print(f" [{i}] {preview}...") if len(chapters) >= 9: print(f"\n--- Sample: Chapter 9 ---") ch9 = [c for c in chapters if c["chapter"] == 9] if ch9: for i, p in enumerate(ch9[0]["paragraphs"][:3]): preview = re.sub(r"<[^>]+>", "", p)[:120] print(f" [{i}] {preview}...") if len(chapters) >= 18: print(f"\n--- Sample: Chapter 18 (last) ---") last = chapters[-1] for i, p in enumerate(last["paragraphs"][:3]): preview = re.sub(r"<[^>]+>", "", p)[:120] print(f" [{i}] {preview}...") # --- Quality Checks --- print(f"\n--- Quality Checks ---") # Check all 18 chapters present expected_chapters = set(range(1, 19)) actual_chapters = set(ch["chapter"] for ch in chapters) missing = expected_chapters - actual_chapters extra = actual_chapters - expected_chapters if missing: print(f" ⚠ Missing chapters: {sorted(missing)}") if extra: print(f" ⚠ Extra chapters: {sorted(extra)}") if not missing and not extra: print(f" ✅ All 18 chapters present") # Check intro segments expected_intros = { "front-pages", "foreword", "about-the-author", "credits", "back-flap", } actual_intros = {s["slug"] for s in intro_segments} missing_intros = expected_intros - actual_intros extra_intros = actual_intros - expected_intros if missing_intros: print(f" ⚠ Missing intro segments: {missing_intros}") if extra_intros: print(f" ⚠ Extra intro segments: {extra_intros}") if not missing_intros and not extra_intros: print(f" ✅ All 5 intro/back-matter segments present") # Check empty chapters empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0] if empty_chapters: print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}") else: print(f" ✅ No empty chapters") # Check empty intro segments empty_intros = [s["slug"] for s in intro_segments if len(s["paragraphs"]) == 0] if empty_intros: print(f" ⚠ Empty intro segments (0 paragraphs): {empty_intros}") else: print(f" ✅ No empty intro segments") # Check ScaGoudy decoding — look for common encoded characters encoded_chars = set("äåöëòçüéàùè") suspect_count = 0 all_sections = chapters + intro_segments for section in all_sections: for p in section["paragraphs"]: plain = re.sub(r"<[^>]+>", "", p) for c in encoded_chars: if c in plain: suspect_count += 1 break if suspect_count == 0: print(f" ✅ No suspected undecoded ScaGoudy characters found") else: print( f" ℹ {suspect_count} paragraphs contain characters that may be undecoded ScaGoudy" ) # Check verse blocks (centered italic divs) verse_block_count = 0 for ch in chapters: for p in ch["paragraphs"]: if '
' in p: verse_block_count += 1 print(f" ℹ {verse_block_count} Sanskrit verse blocks in chapters") # Count structured verse headers (TEXT N, TRANSLATION, PURPORT) text_n_count = 0 translation_header_count = 0 purport_header_count = 0 for ch in chapters: for p in ch["paragraphs"]: plain = re.sub(r"<[^>]+>", "", p).strip() if re.match(r"TEXT\s+\d+", plain): text_n_count += 1 elif plain == "TRANSLATION": translation_header_count += 1 elif plain == "PURPORT": purport_header_count += 1 print(f" ℹ TEXT N headers: {text_n_count}") print(f" ℹ TRANSLATION headers: {translation_header_count}") print(f" ℹ PURPORT headers: {purport_header_count}") # ScaGoudy decoding spot check print(f"\n--- ScaGoudy Decoding Spot Check ---") if chapters: ch1_text = " ".join(chapters[0]["paragraphs"][:20]) if "Prabhupāda" in ch1_text: print(f" ✅ 'Prabhupāda' correctly decoded") elif "Prabhupäda" in ch1_text: print(f" ⚠ ScaGoudy decoding incomplete for 'Prabhupäda'") else: print(f" ℹ 'Prabhupāda' not found in Chapter 1 first 20 paragraphs") if "Kṛṣṇa" in ch1_text: print(f" ✅ 'Kṛṣṇa' correctly decoded in Chapter 1") elif "Kåñëa" in ch1_text: print(f" ⚠ 'Kåñëa' still encoded — ScaGoudy decoding may have failed!") else: print( f" ℹ 'Kṛṣṇa' not found in Chapter 1 first 20 paragraphs (may be normal)" ) # Check for Kapila if "Kapila" in ch1_text: print(f" ✅ 'Kapila' found in Chapter 1") # Check for Devahūti if "Devahūti" in ch1_text: print(f" ✅ 'Devahūti' correctly decoded") elif "Devahüti" in ch1_text: print(f" ⚠ 'Devahüti' still encoded") # Check for Sāṅkhya if "Sāṅkhya" in ch1_text or "sāṅkhya" in ch1_text or "Sāṇkhya" in ch1_text: print(f" ✅ 'Sāṅkhya' correctly decoded") # Sample verse blocks print(f"\n--- Sample Sanskrit Verse Blocks ---") shown = 0 for ch in chapters: for p in ch["paragraphs"]: if '
' in p: preview = re.sub(r"<[^>]+>", "", p)[:120] print(f" Ch {ch['chapter']}: {preview}...") shown += 1 if shown >= 3: break if shown >= 3: break # Sample structured verse sections (TEXT N + TRANSLATION) print(f"\n--- Sample Structured Verse Sections ---") shown = 0 for ch in chapters[:3]: for i, p in enumerate(ch["paragraphs"]): plain = re.sub(r"<[^>]+>", "", p).strip() if re.match(r"TEXT\s+\d+", plain) and shown < 3: # Show TEXT header and next few paragraphs print(f" Ch {ch['chapter']}:") for j in range(i, min(i + 5, len(ch["paragraphs"]))): pj_preview = re.sub(r"<[^>]+>", "", ch["paragraphs"][j])[:100] print(f" [{j}] {pj_preview}...") shown += 1 print() if shown >= 3: break # Check intro segments content print(f"--- Intro Segment Samples ---") for seg in intro_segments: print(f"\n {seg['title']} ({seg['slug']}, order {seg['order']}):") for i, p in enumerate(seg["paragraphs"][:2]): preview = re.sub(r"<[^>]+>", "", p)[:120] print(f" [{i}] {preview}...") print(f"\nDone! ✨") if __name__ == "__main__": main()