#!/usr/bin/env python3 """Extract "An Analysis of Bhagwat Geeta" from others-before-1977.html. This document is a structured analysis organized into three numbered sections (topics) with numbered points and verse references. It is plain English text (no ScaGoudy decoding needed). Source: public/html/others-before-1977.html - Title:

An Analysis of Bhagwat Geeta

(line ~15182) - Section 1: "Mans eternal relation with Godhead" — 45

items - Section 2: "Man's Duty Toward Godhead" — 34

items - Section 3: "Man's ultimate necessity and perfection of life" — 3

items - End boundary:

Solution of Present Crisises... (line ~15276) CSS classes: p116 — Title (bold) p39 — Section headers ("1: Mans eternal relation with Godhead") p5 — Duplicate section title (bold, with ) — SKIP p6 — Body paragraphs (plain English) p34 — Math calculation at the end — SKIP p11 — Next document title (end boundary) Output (prose book structure): lib/jsons/other-documents/analysis-of-bhagavad-gita/ intro.json # metadata chapters/ 01.json # Chapter 1 02.json # Chapter 2 03.json # Chapter 3 Usage: python3 scripts/od_analysis_bg_extract.py """ import json import os import re import sys from bs4 import BeautifulSoup, Tag # --------------------------------------------------------------------------- # Constants # --------------------------------------------------------------------------- BOOK_CODE = "od-analysis-bg" # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- def get_css_class(tag: Tag) -> str: """Get the first CSS class of a tag, or empty string.""" classes = tag.get("class", []) return classes[0] if classes else "" def is_empty(tag: Tag) -> bool: """Check if a tag is empty or whitespace-only.""" text = tag.get_text().strip() return not text or text == "\xa0" def clean_text(tag: Tag) -> str: """Extract inner HTML from a tag, strip span class artifacts, normalize whitespace.""" inner = "".join(str(child) for child in tag.children) # Remove span tags with CSS classes (RTF conversion artifacts) but keep their content inner = re.sub( r']*class="[^"]*"[^>]*>(.*?)', r"\1", inner, flags=re.DOTALL ) # Remove any remaining bare or tags inner = re.sub(r"]*>", "", inner) # Remove and tags (not needed for body paragraphs) inner = re.sub(r"", "", inner) # Clean up Apple-tab-span leftovers (tab characters) inner = inner.replace("\t", " ") # Strip
tags inner = re.sub(r"", " ", inner) # Normalize whitespace inner = re.sub(r"\s+", " ", inner).strip() # Fix HTML entities inner = inner.replace("&", "&") return inner def write_json(data, filepath): """Write data as formatted JSON to filepath, creating directories as needed.""" os.makedirs(os.path.dirname(filepath), exist_ok=True) with open(filepath, "w", encoding="utf-8") as f: json.dump(data, f, ensure_ascii=False, indent=2) print(f" ✅ Wrote {filepath}") # --------------------------------------------------------------------------- # Extraction # --------------------------------------------------------------------------- def extract_chapters(soup: BeautifulSoup) -> list[dict]: """Extract the chapter data from 'An Analysis of Bhagwat Geeta'.""" # Find the title tag:

An Analysis of Bhagwat Geeta

title_tag = None for p in soup.find_all("p", class_="p116"): text = p.get_text().strip() if "Analysis of Bhagwat Geeta" in text: title_tag = p break if not title_tag: print("❌ Could not find title 'An Analysis of Bhagwat Geeta'") sys.exit(1) print(f" 📍 Found title: {title_tag.get_text().strip()}") # Collect all

tags after the title until we hit the end boundary chapters = [] current_chapter = None tag = title_tag.find_next_sibling() while tag: if not isinstance(tag, Tag) or tag.name != "p": tag = tag.find_next_sibling() continue css = get_css_class(tag) text = tag.get_text().strip() # End boundary: next document starts if css == "p11" and "Solution of Present Crisises" in text: break # Skip empty tags if is_empty(tag): tag = tag.find_next_sibling() continue # Section header:

1: Mans eternal relation with Godhead

if css == "p39": # Parse section number and title match = re.match(r"(\d+):\s*(.+)", text) if match: num = int(match.group(1)) title = match.group(2).strip() current_chapter = { "number": num, "title": title, "paragraphs": [], } chapters.append(current_chapter) print(f" 📖 Section {num}: {title}") tag = tag.find_next_sibling() continue # Skip p5 (duplicate section title in bold with span) if css == "p5": tag = tag.find_next_sibling() continue # Skip p34 (math calculation at the end) if css == "p34": tag = tag.find_next_sibling() continue # Body paragraph:

if css == "p6" and current_chapter is not None: cleaned = clean_text(tag) if cleaned: current_chapter["paragraphs"].append(cleaned) tag = tag.find_next_sibling() continue # Any other class — skip tag = tag.find_next_sibling() return chapters # --------------------------------------------------------------------------- # Quality check # --------------------------------------------------------------------------- def quality_check(chapters: list[dict]): """Validate the extracted chapters.""" print("\n🔍 Quality check:\n") print(f" Chapters: {len(chapters)}") assert len(chapters) == 3, f"Expected 3 chapters, got {len(chapters)}" total_paragraphs = 0 for ch in chapters: count = len(ch["paragraphs"]) total_paragraphs += count print(f' Chapter {ch["number"]}: "{ch["title"]}" — {count} paragraphs') # Spot-check first paragraph if ch["paragraphs"]: first = ch["paragraphs"][0] print(f" First: {first[:80]}...") # Spot-check last paragraph if len(ch["paragraphs"]) > 1: last = ch["paragraphs"][-1] print(f" Last: {last[:80]}...") print(f"\n Total paragraphs: {total_paragraphs}") # Expected counts (from manual inspection): # Section 1: 45 items (numbered 1-45) # Section 2: 34 items (numbered 1-34) # Section 3: 3 items (numbered 1-3) expected_total = 82 # 45 + 34 + 3 if total_paragraphs == expected_total: print(f" ✅ Paragraph count matches expected: {expected_total}") else: print(f" ⚠️ Expected {expected_total} paragraphs, got {total_paragraphs}") # This is a warning, not a hard failure — some items may have sub-items # or the HTML structure may slightly differ # Check that paragraphs are not empty for ch in chapters: for i, p in enumerate(ch["paragraphs"]): assert p.strip(), f"Empty paragraph in chapter {ch['number']}, index {i}" print("\n ✅ All checks passed!") # --------------------------------------------------------------------------- # Output # --------------------------------------------------------------------------- def write_output(chapters: list[dict], output_dir: str): """Write intro.json and per-chapter JSON files.""" # Build intro.json intro = { "book": BOOK_CODE, "title": "An Analysis of Bhagwat Geeta", "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda", "year": None, "description": "A structured analysis of Bhagavad-gītā organized into three topics with numbered points and verse references.", "totalChapters": len(chapters), "chapters": [{"number": ch["number"], "title": ch["title"]} for ch in chapters], } write_json(intro, os.path.join(output_dir, "intro.json")) # Write per-chapter files chapters_dir = os.path.join(output_dir, "chapters") for ch in chapters: chapter_data = { "book": BOOK_CODE, "chapter": ch["number"], "title": ch["title"], "type": "chapter", "paragraphs": ch["paragraphs"], } filename = f"{ch['number']:02d}.json" write_json(chapter_data, os.path.join(chapters_dir, filename)) # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- def main(): base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html") output_dir = os.path.join( base_dir, "lib", "jsons", "other-documents", "analysis-of-bhagavad-gita" ) if not os.path.exists(html_path): print(f"❌ HTML file not found: {html_path}") sys.exit(1) # Clean up old single-file JSON if it exists old_json = os.path.join( base_dir, "lib", "jsons", "other-documents", "analysis-of-bhagavad-gita.json" ) if os.path.exists(old_json): os.remove(old_json) print(f" 🗑️ Removed old single-file JSON: {old_json}") print(f"📂 Reading {html_path}...") with open(html_path, "r", encoding="utf-8") as f: soup = BeautifulSoup(f, "html.parser") print("\n📝 Extracting 'An Analysis of Bhagwat Geeta'...\n") chapters = extract_chapters(soup) quality_check(chapters) write_output(chapters, output_dir) print(f"\n🎉 Done! Output directory: {output_dir}") if __name__ == "__main__": main()