#!/usr/bin/env python3 """Extract Perfect Questions Perfect Answers (1977) from other.html. The book has 3 intro segments (front-pages, introduction, about-the-author), 9 chapters, and a Concluding Words section. Output: lib/jsons/books/pqpa/intro.json — Book metadata lib/jsons/books/pqpa/intro/*.json — Intro segments lib/jsons/books/pqpa/chapters/*.json — Chapters + concluding words Usage: python3 scripts/pqpa_extract.py """ import json import os import re import sys from bs4 import BeautifulSoup, Tag # --------------------------------------------------------------------------- # ScaGoudy → IAST Unicode mapping # --------------------------------------------------------------------------- SCAGOUDY_TO_IAST = { "ä": "ā", "Ä": "Ā", "å": "ṛ", "Å": "Ṛ", "ñ": "ṣ", "Ñ": "Ṣ", "ö": "ṭ", "Ö": "Ṭ", "ë": "ṇ", "Ë": "Ṇ", "ò": "ḍ", "Ò": "Ḍ", "ç": "ś", "Ç": "Ś", "ì": "ṅ", "ï": "ñ", "ü": "ū", "Ü": "Ū", "é": "ī", "É": "Ī", "à": "ṁ", "ù": "ḥ", "è": "ṝ", "û": "ṁl", "ÿ": "ḷ", } def decode_scagoudy(text: str) -> str: """Replace ScaGoudy-encoded characters with proper IAST Unicode.""" for src, dst in SCAGOUDY_TO_IAST.items(): text = text.replace(src, dst) return text def decode_html_content(tag: Tag) -> str: """Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags.""" inner = "".join(str(child) for child in tag.children) inner = decode_scagoudy(inner) # Remove span tags with CSS classes (RTF conversion artifacts) inner = re.sub( r']*class="[^"]*"[^>]*>(.*?)', r"\1", inner, flags=re.DOTALL ) # Clean up Apple-tab-span leftovers (tab characters) inner = inner.replace("\t", " ") # Normalize whitespace inner = re.sub(r"\s+", " ", inner).strip() return inner def decode_text(tag: Tag) -> str: """Get plain text from a tag with ScaGoudy decoding.""" return decode_scagoudy(tag.get_text().strip()) def get_css_class(tag: Tag) -> str: """Get the first CSS class of a tag, or empty string.""" classes = tag.get("class", []) return classes[0] if classes else "" def is_empty(tag: Tag) -> bool: """Check if a tag is empty or whitespace-only.""" text = tag.get_text().strip() return not text or text == "\xa0" def is_dash_line(text: str) -> bool: """Return True if text is only dashes, spaces, and/or whitespace.""" cleaned = text.strip() if not cleaned: return False return all(c in "-—–\u2014\u2013 \t" for c in cleaned) # --------------------------------------------------------------------------- # CSS class categorization for PQPA # --------------------------------------------------------------------------- # Book title header class BOOK_HEADER_CLASS = "p12" # Bookmark class — used for section boundary detection BOOKMARK_CLASS = "p13" # Body paragraph classes (primary content) BODY_CLASSES = frozenset(["p6", "p48"]) # Dialogue line class — the main content class (714 occurrences) DIALOGUE_CLASS = "p92" # Sanskrit verse pāda classes (italic, centered) VERSE_PADA_CLASSES = frozenset(["p53", "p66"]) # Verse translations / body text TRANSLATION_CLASS = "p44" # Chapter heading (bold) CHAPTER_HEADING_CLASS = "p27" # Section heading (bold) SECTION_HEADING_CLASS = "p31" # Front pages text FRONT_PAGES_CLASS = "p18" # Attribution text ATTRIBUTION_CLASS = "p42" # Numbered items / verse quotes NUMBERED_CLASS = "p24" # Mantra text MANTRA_CLASS = "p19" # Contents heading CONTENTS_CLASS = "p69" # Title page text TITLE_PAGE_CLASS = "p3" # Misc classes to include MISC_CLASSES = frozenset(["p28", "p76", "p93", "p94", "p95", "p96"]) # Classes to skip entirely (empty/decorative) SKIP_CLASSES = frozenset(["p9", "p15", "p70"]) # --------------------------------------------------------------------------- # Extraction helpers # --------------------------------------------------------------------------- def find_pqpa_boundaries(all_paragraphs: list) -> tuple: """Find the start and end paragraph indices of the PQPA content. Start:

containing "Perfect Questions Perfect Answers" End: next

after start, or end of file """ start_idx = None end_idx = len(all_paragraphs) for i, p in enumerate(all_paragraphs): cls = get_css_class(p) if cls != BOOK_HEADER_CLASS: continue text = decode_text(p) if "Perfect Questions Perfect Answers" in text: start_idx = i elif start_idx is not None: # Next p12 after PQPA start = end of PQPA end_idx = i break if start_idx is None: print( "ERROR: Could not find 'Perfect Questions Perfect Answers' header in other.html!" ) sys.exit(1) # Also check for Glossary marker as end boundary for i in range(start_idx + 1, end_idx): p = all_paragraphs[i] cls = get_css_class(p) if cls == TITLE_PAGE_CLASS: text = p.get_text().strip() if text == "Glossary": end_idx = i break return start_idx, end_idx def find_all_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list: """Find all p13 bookmark tags in the PQPA range. Returns list of (paragraph_index, bookmark_text). The bookmark text is stripped of any span noise like . """ bookmarks = [] for i in range(start_idx, end_idx): p = all_paragraphs[i] cls = get_css_class(p) if cls == BOOKMARK_CLASS: # Get text, stripping span noise text = decode_text(p).strip() if text.startswith("PQPA"): bookmarks.append((i, text)) return bookmarks def parse_chapter_bookmark(text: str): """Parse a chapter bookmark like 'PQPA 1: Kṛṣṇa, the All-Attractive'. Returns (chapter_number, title) or None if not a chapter bookmark. """ m = re.match(r"PQPA\s+(\d+):\s+(.+)", text.strip()) if m: return int(m.group(1)), m.group(2).strip() return None def parse_intro_bookmark(text: str) -> bool: """Check if this is the Introduction bookmark.""" return text.strip() == "PQPA Introduction" def parse_concluding_bookmark(text: str) -> bool: """Check if this is the Concluding Words bookmark.""" return text.strip() == "PQPA Concluding Words" def extract_paragraphs( all_paragraphs: list, start_idx: int, end_idx: int, is_front_pages: bool = False, ) -> list: """Extract paragraphs for a section (chapter, intro segment, or conclusion). Handles all CSS classes within the PQPA content. Groups consecutive p53/p66 verse pādas into centered italic divs. Detects verse patterns in p6/p92 (starts with , has
). """ result = [] verse_accumulator = [] def flush_verse(): if verse_accumulator: joined = "
".join(verse_accumulator) result.append(f'

{joined}
') verse_accumulator.clear() for i in range(start_idx, end_idx): p = all_paragraphs[i] cls = get_css_class(p) # Skip empty paragraphs if is_empty(p): continue # Skip noise classes if cls in SKIP_CLASSES: continue # Skip bookmark tags (p13) if cls == BOOKMARK_CLASS: continue # Skip book title header (p12) if cls == BOOK_HEADER_CLASS: continue # Check for dash lines — skip them text = decode_text(p) if is_dash_line(text): continue # Sanskrit verse pādas (p53, p66) — accumulate into centered block if cls in VERSE_PADA_CLASSES: html = decode_html_content(p) if html: # Remove inner tags before re-wrapping stripped = re.sub(r"", "", html) verse_accumulator.append(stripped) continue # Non-verse paragraph — flush any accumulated verse first flush_verse() # Dialogue lines (p92) — the main content class if cls == DIALOGUE_CLASS: html = decode_html_content(p) if html: # Detect verses disguised as dialogue: starts with and has
stripped_html = html.strip() if stripped_html.startswith("") and "", "", stripped_html) result.append( f'
{inner}
' ) else: result.append(html) continue # Chapter heading (p27) — include as paragraph if cls == CHAPTER_HEADING_CLASS: html = decode_html_content(p) if html: result.append(html) continue # Section heading (p31) — include as paragraph if cls == SECTION_HEADING_CLASS: html = decode_html_content(p) if html: result.append(html) continue # Front pages text (p18) — include if cls == FRONT_PAGES_CLASS: html = decode_html_content(p) if html: result.append(html) continue # Contents heading (p69) — include if cls == CONTENTS_CLASS: html = decode_html_content(p) if html: result.append(html) continue # Attribution text (p42) if cls == ATTRIBUTION_CLASS: html = decode_html_content(p) if html: result.append(html) continue # Numbered items / verse quotes (p24) if cls == NUMBERED_CLASS: html = decode_html_content(p) if html: # Detect verses: starts with and has
stripped_html = html.strip() if stripped_html.startswith("") and "", "", stripped_html) result.append( f'
{inner}
' ) else: result.append(html) continue # Mantra text (p19) — wrap in centered div if cls == MANTRA_CLASS: html = decode_html_content(p) if html: # Check if it's a multi-line verse if "", "", html) result.append( f'
{inner}
' ) else: result.append(html) continue # Verse translation / body text (p44) if cls == TRANSLATION_CLASS: html = decode_html_content(p) if html: result.append(html) continue # Title page text (p3) — include if cls == TITLE_PAGE_CLASS: html = decode_html_content(p) if html: result.append(html) continue # Body paragraphs (p6, p48) if cls in BODY_CLASSES: html = decode_html_content(p) if html: # Detect Sanskrit verses disguised as body paragraphs: # they start with and contain
(multi-line italic = verse) stripped_html = html.strip() if stripped_html.startswith("") and "", "", stripped_html) result.append( f'
{inner}
' ) else: result.append(html) continue # Misc classes — include as-is if cls in MISC_CLASSES: html = decode_html_content(p) if html: result.append(html) continue # Any other class — try to extract content (be permissive) html = decode_html_content(p) if html and not is_dash_line(text): result.append(html) # Flush any trailing verse flush_verse() return result # --------------------------------------------------------------------------- # Main extraction logic # --------------------------------------------------------------------------- def extract_pqpa(soup: BeautifulSoup) -> dict: """Extract the entire Perfect Questions Perfect Answers from the parsed HTML. Returns a dict with: "intro_segments": list of intro segment dicts "chapters": list of chapter dicts "concluding_words": concluding words dict "chapter_listing": list of {number, title} for intro.json """ all_paragraphs = list(soup.find_all("p")) # Step 1: Find PQPA boundaries pqpa_start, pqpa_end = find_pqpa_boundaries(all_paragraphs) print(f" PQPA content: paragraphs {pqpa_start} to {pqpa_end}") print(f" Total paragraphs in range: {pqpa_end - pqpa_start}") # Step 2: Find all bookmarks bookmarks = find_all_bookmarks(all_paragraphs, pqpa_start, pqpa_end) print(f" Found {len(bookmarks)} PQPA bookmarks") for pos, text in bookmarks: print(f" Line ~{pos}: {text}") # Step 3: Categorize bookmarks intro_bookmark = None # (para_idx,) chapter_bookmarks = [] # (para_idx, chapter_num, title) concluding_bookmark = None # (para_idx,) for pos, text in bookmarks: ch = parse_chapter_bookmark(text) if ch: chapter_bookmarks.append((pos, ch[0], ch[1])) elif parse_intro_bookmark(text): intro_bookmark = pos elif parse_concluding_bookmark(text): concluding_bookmark = pos print( f" Introduction bookmark: {'found' if intro_bookmark is not None else 'NOT FOUND'}" ) print(f" Chapter bookmarks: {len(chapter_bookmarks)}") print( f" Concluding Words bookmark: {'found' if concluding_bookmark is not None else 'NOT FOUND'}" ) # Collect all section starts for boundary computation all_section_starts = sorted( ([intro_bookmark] if intro_bookmark is not None else []) + [pos for pos, _, _ in chapter_bookmarks] + ([concluding_bookmark] if concluding_bookmark is not None else []) ) # Step 4: Extract front-pages (from p12 header to Introduction bookmark) front_pages_start = pqpa_start + 1 # after p12 header front_pages_end = ( intro_bookmark if intro_bookmark is not None else (chapter_bookmarks[0][0] if chapter_bookmarks else pqpa_end) ) front_pages_paras = extract_paragraphs( all_paragraphs, front_pages_start, front_pages_end, is_front_pages=True ) front_pages = { "slug": "front-pages", "title": "Front Pages", "type": "intro-segment", "book": "pqpa", "order": 1, "paragraphs": front_pages_paras, } # Step 5: Extract Introduction (from PQPA Introduction to PQPA 1) introduction = None if intro_bookmark is not None: intro_content_start = intro_bookmark + 1 # End at the first chapter bookmark intro_content_end = chapter_bookmarks[0][0] if chapter_bookmarks else pqpa_end intro_paras = extract_paragraphs( all_paragraphs, intro_content_start, intro_content_end ) introduction = { "slug": "introduction", "title": "Introduction", "type": "intro-segment", "book": "pqpa", "order": 2, "paragraphs": intro_paras, } # Step 6: Extract chapters 1-9 chapters = [] for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_bookmarks): content_start = ch_pos + 1 # Content ends at next chapter bookmark, or concluding words, or pqpa_end if idx + 1 < len(chapter_bookmarks): content_end = chapter_bookmarks[idx + 1][0] elif concluding_bookmark is not None: content_end = concluding_bookmark else: content_end = pqpa_end paragraphs = extract_paragraphs(all_paragraphs, content_start, content_end) ch = { "book": "pqpa", "chapter": ch_num, "title": ch_title, "type": "chapter", "paragraphs": paragraphs, } chapters.append(ch) # Step 7: Extract Concluding Words + About the Author concluding_words = None about_the_author = None if concluding_bookmark is not None: # Find the "About the Author" boundary: #

His Divine Grace A. C. Bhaktivedanta Swami Prabhupāda

about_author_pos = None for i in range(concluding_bookmark + 1, pqpa_end): p = all_paragraphs[i] cls = get_css_class(p) if cls == CHAPTER_HEADING_CLASS: text = decode_text(p) if "His Divine Grace" in text: about_author_pos = i break # Concluding Words: from bookmark to About the Author (or pqpa_end) cw_content_start = concluding_bookmark + 1 cw_content_end = about_author_pos if about_author_pos is not None else pqpa_end cw_paras = extract_paragraphs(all_paragraphs, cw_content_start, cw_content_end) concluding_words = { "book": "pqpa", "chapter": None, "title": "Concluding Words", "type": "conclusion", "slug": "concluding-words", "paragraphs": cw_paras, } # About the Author: from p27 heading to pqpa_end if about_author_pos is not None: ata_paras = extract_paragraphs(all_paragraphs, about_author_pos, pqpa_end) about_the_author = { "slug": "about-the-author", "title": "About the Author", "type": "intro-segment", "book": "pqpa", "order": 99, "paragraphs": ata_paras, } # Step 8: Build intro segments list intro_segments = [front_pages] if introduction: intro_segments.append(introduction) if about_the_author: intro_segments.append(about_the_author) # Step 9: Build chapter listing for intro.json chapter_listing = [ {"number": ch["chapter"], "title": ch["title"]} for ch in chapters ] return { "intro_segments": intro_segments, "chapters": chapters, "concluding_words": concluding_words, "chapter_listing": chapter_listing, } # --------------------------------------------------------------------------- # Output # --------------------------------------------------------------------------- def write_json(data, filepath): """Write data as formatted JSON to filepath, creating directories as needed.""" os.makedirs(os.path.dirname(filepath), exist_ok=True) with open(filepath, "w", encoding="utf-8") as f: json.dump(data, f, ensure_ascii=False, indent=2) def main(): base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) html_path = os.path.join(base_dir, "public", "html", "other.html") output_base = os.path.join(base_dir, "lib", "jsons", "books", "pqpa") if not os.path.exists(html_path): print(f"ERROR: {html_path} not found!") sys.exit(1) print(f"Reading {html_path}...") with open(html_path, "r", encoding="utf-8") as f: soup = BeautifulSoup(f, "html.parser") print("Extracting Perfect Questions Perfect Answers (1977)...\n") result = extract_pqpa(soup) intro_segments = result["intro_segments"] chapters = result["chapters"] concluding_words = result["concluding_words"] chapter_listing = result["chapter_listing"] # --- Write intro.json --- intro_meta = { "book": "pqpa", "title": "Perfect Questions Perfect Answers", "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda", "year": 1977, "description": "Conversations between Śrīla Prabhupāda and Bob Cohen, a Peace Corps worker in India.", "totalChapters": 9, "chapters": chapter_listing, } intro_path = os.path.join(output_base, "intro.json") write_json(intro_meta, intro_path) print(f"\n Written: {intro_path}") # --- Write intro segments --- print(f"\n--- Intro Segments ---") for seg in intro_segments: filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json") write_json(seg, filepath) print( f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}" ) # --- Write chapters --- print(f"\n--- Chapters ---") for ch in chapters: ch_num_str = str(ch["chapter"]).zfill(2) filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json") write_json(ch, filepath) print( f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — " f"{ch['title']}" ) # --- Write concluding words --- if concluding_words: filepath = os.path.join(output_base, "chapters", "concluding-words.json") write_json(concluding_words, filepath) print( f"\n Concluding Words: {len(concluding_words['paragraphs'])} paragraphs → {filepath}" ) # --- Summary --- total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments) total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters) total_concluding_paras = ( len(concluding_words["paragraphs"]) if concluding_words else 0 ) total_paras = total_intro_paras + total_chapter_paras + total_concluding_paras print(f"\n{'=' * 60}") print(f" EXTRACTION SUMMARY") print(f"{'=' * 60}") print(f" Intro segments: {len(intro_segments)}") for seg in intro_segments: print( f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs" ) print(f" Chapters: {len(chapters)}") print(f" Concluding Words: {'Yes' if concluding_words else 'No'}") print(f" Total paragraphs: {total_paras}") print(f" Intro: {total_intro_paras}") print(f" Chapters: {total_chapter_paras}") print(f" Concluding: {total_concluding_paras}") if chapters: print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}") print(f" Output directory: {output_base}/") # --- Samples --- print(f"\n--- Sample: Chapter 1 ---") if chapters: ch1 = chapters[0] for i, p in enumerate(ch1["paragraphs"][:4]): preview = re.sub(r"<[^>]+>", "", p)[:120] print(f" [{i}] {preview}...") if len(chapters) >= 9: print(f"\n--- Sample: Chapter 9 (last) ---") last = chapters[-1] for i, p in enumerate(last["paragraphs"][:3]): preview = re.sub(r"<[^>]+>", "", p)[:120] print(f" [{i}] {preview}...") if concluding_words: print(f"\n--- Sample: Concluding Words ---") for i, p in enumerate(concluding_words["paragraphs"][:3]): preview = re.sub(r"<[^>]+>", "", p)[:120] print(f" [{i}] {preview}...") # --- Quality Checks --- print(f"\n--- Quality Checks ---") # Check all 9 chapters present expected_chapters = set(range(1, 10)) actual_chapters = set(ch["chapter"] for ch in chapters) missing = expected_chapters - actual_chapters extra = actual_chapters - expected_chapters if missing: print(f" ⚠ Missing chapters: {sorted(missing)}") if extra: print(f" ⚠ Extra chapters: {sorted(extra)}") if not missing and not extra: print(f" ✅ All 9 chapters present") # Check intro segments expected_intros = {"front-pages", "introduction", "about-the-author"} actual_intros = {s["slug"] for s in intro_segments} missing_intros = expected_intros - actual_intros if missing_intros: print(f" ⚠ Missing intro segments: {missing_intros}") else: print(f" ✅ All 3 intro segments present") # Check concluding words if concluding_words: print(f" ✅ Concluding Words present") else: print(f" ⚠ Concluding Words NOT found") # Check empty chapters empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0] if empty_chapters: print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}") else: print(f" ✅ No empty chapters") # Check ScaGoudy decoding — look for common encoded characters encoded_chars = set("äåöëòçüéàùè") suspect_count = 0 all_sections = chapters + intro_segments if concluding_words: all_sections = all_sections + [concluding_words] for section in all_sections: for p in section["paragraphs"]: plain = re.sub(r"<[^>]+>", "", p) for c in encoded_chars: if c in plain: suspect_count += 1 break if suspect_count == 0: print(f" ✅ No suspected undecoded ScaGoudy characters found") else: print( f" ℹ {suspect_count} paragraphs contain characters that may be undecoded ScaGoudy" ) # Check verse blocks verse_block_count = 0 for ch in chapters: for p in ch["paragraphs"]: if '
' in p: verse_block_count += 1 print(f" ℹ {verse_block_count} verse blocks in chapters") # ScaGoudy decoding spot check print(f"\n--- ScaGoudy Decoding Spot Check ---") if chapters: ch1_text = " ".join(chapters[0]["paragraphs"][:10]) if "Kṛṣṇa" in ch1_text: print(f" ✅ 'Kṛṣṇa' correctly decoded in Chapter 1") elif "Kåñëa" in ch1_text: print(f" ⚠ 'Kåñëa' still encoded — ScaGoudy decoding may have failed!") else: print( f" ℹ 'Kṛṣṇa' not found in Chapter 1 first 10 paragraphs (may be normal)" ) # Check for "Śrīla Prabhupāda" if "Prabhupāda" in ch1_text: print(f" ✅ 'Prabhupāda' correctly decoded") elif "Prabhupäda" in ch1_text: print(f" ⚠ ScaGoudy decoding incomplete for 'Prabhupäda'") # Dialogue line check — look for BOB: and ŚRĪLA PRABHUPĀDA: speakers print(f"\n--- Dialogue Speaker Check ---") speaker_counts = {} for ch in chapters: for p in ch["paragraphs"]: # Look for bold speaker patterns m = re.match(r"([^<]+):", p) if m: speaker = m.group(1).strip() speaker_counts[speaker] = speaker_counts.get(speaker, 0) + 1 if speaker_counts: print(f" Speakers found:") for speaker, count in sorted(speaker_counts.items(), key=lambda x: -x[1]): print(f" {speaker}: {count} lines") else: print(f" ⚠ No dialogue speakers detected") # Sample verse blocks print(f"\n--- Sample Verse Blocks ---") shown = 0 for ch in chapters: for p in ch["paragraphs"]: if '
' in p: preview = re.sub(r"<[^>]+>", "", p)[:120] print(f" {preview}...") shown += 1 if shown >= 3: break if shown >= 3: break print(f"\nDone! ✨") if __name__ == "__main__": main()