304 lines
10 KiB
Python
304 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""Extract "An Analysis of Bhagwat Geeta" from others-before-1977.html.
|
|
|
|
This document is a structured analysis organized into three numbered sections
|
|
(topics) with numbered points and verse references. It is plain English text
|
|
(no ScaGoudy decoding needed).
|
|
|
|
Source: public/html/others-before-1977.html
|
|
- Title: <p class="p116"><b>An Analysis of Bhagwat Geeta</b></p> (line ~15182)
|
|
- Section 1: "Mans eternal relation with Godhead" — 45 <p class="p6"> items
|
|
- Section 2: "Man's Duty Toward Godhead" — 34 <p class="p6"> items
|
|
- Section 3: "Man's ultimate necessity and perfection of life" — 3 <p class="p6"> items
|
|
- End boundary: <p class="p11">Solution of Present Crisises... (line ~15276)
|
|
|
|
CSS classes:
|
|
p116 — Title (bold)
|
|
p39 — Section headers ("1: Mans eternal relation with Godhead")
|
|
p5 — Duplicate section title (bold, with <span class="s3">) — SKIP
|
|
p6 — Body paragraphs (plain English)
|
|
p34 — Math calculation at the end — SKIP
|
|
p11 — Next document title (end boundary)
|
|
|
|
Output (prose book structure):
|
|
lib/jsons/other-documents/analysis-of-bhagavad-gita/
|
|
intro.json # metadata
|
|
chapters/
|
|
01.json # Chapter 1
|
|
02.json # Chapter 2
|
|
03.json # Chapter 3
|
|
|
|
Usage:
|
|
python3 scripts/od_analysis_bg_extract.py
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
from bs4 import BeautifulSoup, Tag
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Constants
|
|
# ---------------------------------------------------------------------------
|
|
|
|
BOOK_CODE = "od-analysis-bg"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def get_css_class(tag: Tag) -> str:
|
|
"""Get the first CSS class of a tag, or empty string."""
|
|
classes = tag.get("class", [])
|
|
return classes[0] if classes else ""
|
|
|
|
|
|
def is_empty(tag: Tag) -> bool:
|
|
"""Check if a tag is empty or whitespace-only."""
|
|
text = tag.get_text().strip()
|
|
return not text or text == "\xa0"
|
|
|
|
|
|
def clean_text(tag: Tag) -> str:
|
|
"""Extract inner HTML from a tag, strip span class artifacts, normalize whitespace."""
|
|
inner = "".join(str(child) for child in tag.children)
|
|
# Remove span tags with CSS classes (RTF conversion artifacts) but keep their content
|
|
inner = re.sub(
|
|
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
|
|
)
|
|
# Remove any remaining bare <span> or </span> tags
|
|
inner = re.sub(r"</?span[^>]*>", "", inner)
|
|
# Remove <b> and </b> tags (not needed for body paragraphs)
|
|
inner = re.sub(r"</?b>", "", inner)
|
|
# Clean up Apple-tab-span leftovers (tab characters)
|
|
inner = inner.replace("\t", " ")
|
|
# Strip <br> tags
|
|
inner = re.sub(r"<br\s*/?>", " ", inner)
|
|
# Normalize whitespace
|
|
inner = re.sub(r"\s+", " ", inner).strip()
|
|
# Fix HTML entities
|
|
inner = inner.replace("&", "&")
|
|
return inner
|
|
|
|
|
|
def write_json(data, filepath):
|
|
"""Write data as formatted JSON to filepath, creating directories as needed."""
|
|
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
|
with open(filepath, "w", encoding="utf-8") as f:
|
|
json.dump(data, f, ensure_ascii=False, indent=2)
|
|
print(f" ✅ Wrote {filepath}")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Extraction
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def extract_chapters(soup: BeautifulSoup) -> list[dict]:
|
|
"""Extract the chapter data from 'An Analysis of Bhagwat Geeta'."""
|
|
|
|
# Find the title tag: <p class="p116"><b>An Analysis of Bhagwat Geeta</b></p>
|
|
title_tag = None
|
|
for p in soup.find_all("p", class_="p116"):
|
|
text = p.get_text().strip()
|
|
if "Analysis of Bhagwat Geeta" in text:
|
|
title_tag = p
|
|
break
|
|
|
|
if not title_tag:
|
|
print("❌ Could not find title 'An Analysis of Bhagwat Geeta'")
|
|
sys.exit(1)
|
|
|
|
print(f" 📍 Found title: {title_tag.get_text().strip()}")
|
|
|
|
# Collect all <p> tags after the title until we hit the end boundary
|
|
chapters = []
|
|
current_chapter = None
|
|
|
|
tag = title_tag.find_next_sibling()
|
|
while tag:
|
|
if not isinstance(tag, Tag) or tag.name != "p":
|
|
tag = tag.find_next_sibling()
|
|
continue
|
|
|
|
css = get_css_class(tag)
|
|
text = tag.get_text().strip()
|
|
|
|
# End boundary: next document starts
|
|
if css == "p11" and "Solution of Present Crisises" in text:
|
|
break
|
|
|
|
# Skip empty tags
|
|
if is_empty(tag):
|
|
tag = tag.find_next_sibling()
|
|
continue
|
|
|
|
# Section header: <p class="p39">1: Mans eternal relation with Godhead</p>
|
|
if css == "p39":
|
|
# Parse section number and title
|
|
match = re.match(r"(\d+):\s*(.+)", text)
|
|
if match:
|
|
num = int(match.group(1))
|
|
title = match.group(2).strip()
|
|
current_chapter = {
|
|
"number": num,
|
|
"title": title,
|
|
"paragraphs": [],
|
|
}
|
|
chapters.append(current_chapter)
|
|
print(f" 📖 Section {num}: {title}")
|
|
tag = tag.find_next_sibling()
|
|
continue
|
|
|
|
# Skip p5 (duplicate section title in bold with span)
|
|
if css == "p5":
|
|
tag = tag.find_next_sibling()
|
|
continue
|
|
|
|
# Skip p34 (math calculation at the end)
|
|
if css == "p34":
|
|
tag = tag.find_next_sibling()
|
|
continue
|
|
|
|
# Body paragraph: <p class="p6">
|
|
if css == "p6" and current_chapter is not None:
|
|
cleaned = clean_text(tag)
|
|
if cleaned:
|
|
current_chapter["paragraphs"].append(cleaned)
|
|
tag = tag.find_next_sibling()
|
|
continue
|
|
|
|
# Any other class — skip
|
|
tag = tag.find_next_sibling()
|
|
|
|
return chapters
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Quality check
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def quality_check(chapters: list[dict]):
|
|
"""Validate the extracted chapters."""
|
|
print("\n🔍 Quality check:\n")
|
|
|
|
print(f" Chapters: {len(chapters)}")
|
|
assert len(chapters) == 3, f"Expected 3 chapters, got {len(chapters)}"
|
|
|
|
total_paragraphs = 0
|
|
for ch in chapters:
|
|
count = len(ch["paragraphs"])
|
|
total_paragraphs += count
|
|
print(f' Chapter {ch["number"]}: "{ch["title"]}" — {count} paragraphs')
|
|
|
|
# Spot-check first paragraph
|
|
if ch["paragraphs"]:
|
|
first = ch["paragraphs"][0]
|
|
print(f" First: {first[:80]}...")
|
|
|
|
# Spot-check last paragraph
|
|
if len(ch["paragraphs"]) > 1:
|
|
last = ch["paragraphs"][-1]
|
|
print(f" Last: {last[:80]}...")
|
|
|
|
print(f"\n Total paragraphs: {total_paragraphs}")
|
|
|
|
# Expected counts (from manual inspection):
|
|
# Section 1: 45 items (numbered 1-45)
|
|
# Section 2: 34 items (numbered 1-34)
|
|
# Section 3: 3 items (numbered 1-3)
|
|
expected_total = 82 # 45 + 34 + 3
|
|
if total_paragraphs == expected_total:
|
|
print(f" ✅ Paragraph count matches expected: {expected_total}")
|
|
else:
|
|
print(f" ⚠️ Expected {expected_total} paragraphs, got {total_paragraphs}")
|
|
# This is a warning, not a hard failure — some items may have sub-items
|
|
# or the HTML structure may slightly differ
|
|
|
|
# Check that paragraphs are not empty
|
|
for ch in chapters:
|
|
for i, p in enumerate(ch["paragraphs"]):
|
|
assert p.strip(), f"Empty paragraph in chapter {ch['number']}, index {i}"
|
|
|
|
print("\n ✅ All checks passed!")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Output
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def write_output(chapters: list[dict], output_dir: str):
|
|
"""Write intro.json and per-chapter JSON files."""
|
|
|
|
# Build intro.json
|
|
intro = {
|
|
"book": BOOK_CODE,
|
|
"title": "An Analysis of Bhagwat Geeta",
|
|
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
|
"year": None,
|
|
"description": "A structured analysis of Bhagavad-gītā organized into three topics with numbered points and verse references.",
|
|
"totalChapters": len(chapters),
|
|
"chapters": [{"number": ch["number"], "title": ch["title"]} for ch in chapters],
|
|
}
|
|
|
|
write_json(intro, os.path.join(output_dir, "intro.json"))
|
|
|
|
# Write per-chapter files
|
|
chapters_dir = os.path.join(output_dir, "chapters")
|
|
for ch in chapters:
|
|
chapter_data = {
|
|
"book": BOOK_CODE,
|
|
"chapter": ch["number"],
|
|
"title": ch["title"],
|
|
"type": "chapter",
|
|
"paragraphs": ch["paragraphs"],
|
|
}
|
|
filename = f"{ch['number']:02d}.json"
|
|
write_json(chapter_data, os.path.join(chapters_dir, filename))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def main():
|
|
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html")
|
|
output_dir = os.path.join(
|
|
base_dir, "lib", "jsons", "other-documents", "analysis-of-bhagavad-gita"
|
|
)
|
|
|
|
if not os.path.exists(html_path):
|
|
print(f"❌ HTML file not found: {html_path}")
|
|
sys.exit(1)
|
|
|
|
# Clean up old single-file JSON if it exists
|
|
old_json = os.path.join(
|
|
base_dir, "lib", "jsons", "other-documents", "analysis-of-bhagavad-gita.json"
|
|
)
|
|
if os.path.exists(old_json):
|
|
os.remove(old_json)
|
|
print(f" 🗑️ Removed old single-file JSON: {old_json}")
|
|
|
|
print(f"📂 Reading {html_path}...")
|
|
with open(html_path, "r", encoding="utf-8") as f:
|
|
soup = BeautifulSoup(f, "html.parser")
|
|
|
|
print("\n📝 Extracting 'An Analysis of Bhagwat Geeta'...\n")
|
|
chapters = extract_chapters(soup)
|
|
|
|
quality_check(chapters)
|
|
|
|
write_output(chapters, output_dir)
|
|
print(f"\n🎉 Done! Output directory: {output_dir}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|