445 lines
14 KiB
Python
445 lines
14 KiB
Python
#!/usr/bin/env python3
|
||
"""Extract Śrī Caitanya-bhāgavata Commentary from others-before-1977.html.
|
||
|
||
The book has 4 chapters (no intro segments):
|
||
1. Chapter 1
|
||
2. Lord Caitanya's Appearance
|
||
3. Chapter 3
|
||
4. Chapter 4
|
||
|
||
Source: public/html/others-before-1977.html, lines ~4139–4180.
|
||
Start: <p class="p3"><b>Çré Caitanya-bhägavata-Commentary</b></p>
|
||
End: <p class="p2">Bhagaväner kathä [Original Documents]</p> (stop before)
|
||
|
||
Chapter boundaries: <p class="p17"> tags containing "Caitanya-bhägavata"
|
||
|
||
CSS classes:
|
||
p3 — Book title header (skip)
|
||
p17 — Chapter bookmark headers (boundary detection, skip from output)
|
||
p5 — Chapter title headings (skip — we derive titles ourselves)
|
||
p6 — Body paragraphs (MAIN content)
|
||
p38 — Body paragraph variant
|
||
p55 — Body paragraph variant (last paragraph of chapter 4)
|
||
p9 — End-of-chapter markers like "(End Chapter One)" — include
|
||
|
||
Output:
|
||
lib/jsons/books/cbhc/intro.json — Book metadata
|
||
lib/jsons/books/cbhc/chapters/01.json — Chapter 1
|
||
lib/jsons/books/cbhc/chapters/02.json — Chapter 2
|
||
lib/jsons/books/cbhc/chapters/03.json — Chapter 3
|
||
lib/jsons/books/cbhc/chapters/04.json — Chapter 4
|
||
|
||
Usage:
|
||
python3 scripts/cbhc_extract.py
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import re
|
||
import sys
|
||
|
||
from bs4 import BeautifulSoup, Tag
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def decode_html_content(tag: Tag) -> str:
|
||
"""Extract inner HTML from a tag, preserve formatting tags, strip spans."""
|
||
inner = "".join(str(child) for child in tag.children)
|
||
# Remove span tags with CSS classes (RTF conversion artifacts)
|
||
inner = re.sub(
|
||
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
|
||
)
|
||
# Clean up Apple-tab-span leftovers (tab characters)
|
||
inner = inner.replace("\t", " ")
|
||
# Normalize whitespace
|
||
inner = re.sub(r"\s+", " ", inner).strip()
|
||
return inner
|
||
|
||
|
||
def decode_text(tag: Tag) -> str:
|
||
"""Get plain text from a tag."""
|
||
return tag.get_text().strip()
|
||
|
||
|
||
def get_css_class(tag: Tag) -> str:
|
||
"""Get the first CSS class of a tag, or empty string."""
|
||
classes = tag.get("class", [])
|
||
return classes[0] if classes else ""
|
||
|
||
|
||
def is_empty(tag: Tag) -> bool:
|
||
"""Check if a tag is empty or whitespace-only."""
|
||
text = tag.get_text().strip()
|
||
return not text or text == "\xa0"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# CSS class categorization for CBHC
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# Book title header
|
||
BOOK_TITLE_CLASS = "p3"
|
||
|
||
# Chapter bookmark headers (used for boundary detection)
|
||
CHAPTER_BOOKMARK_CLASS = "p17"
|
||
|
||
# Chapter title (bold, e.g., "Chapter 1") — skip from body
|
||
CHAPTER_TITLE_CLASS = "p5"
|
||
|
||
# Body paragraph classes
|
||
BODY_CLASSES = frozenset(["p6", "p38", "p55"])
|
||
|
||
# End-of-chapter markers — include as a paragraph
|
||
END_MARKER_CLASS = "p9"
|
||
|
||
# Classes to skip entirely
|
||
SKIP_CLASSES = frozenset(["p43", "p24"])
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Boundary detection
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def find_cbhc_boundaries(all_paragraphs: list) -> tuple:
|
||
"""Find the start and end paragraph indices of the CBHC content.
|
||
|
||
Start: <p class="p3"> containing "Caitanya-bhägavata-Commentary"
|
||
End: <p class="p2"> containing "Bhagaväner kathä" (next work)
|
||
"""
|
||
start_idx = None
|
||
end_idx = len(all_paragraphs)
|
||
|
||
for i, p in enumerate(all_paragraphs):
|
||
cls = get_css_class(p)
|
||
text = decode_text(p)
|
||
|
||
# Find start: p3 containing "Caitanya-bhägavata"
|
||
if start_idx is None:
|
||
if cls == BOOK_TITLE_CLASS and "Caitanya-bh" in text:
|
||
start_idx = i
|
||
continue
|
||
continue
|
||
|
||
# Find end: p2 containing "Bhagaväner" (next work boundary)
|
||
if cls == "p2" and "Bhag" in text and "kath" in text:
|
||
end_idx = i
|
||
break
|
||
|
||
if start_idx is None:
|
||
print(
|
||
"ERROR: Could not find 'Çré Caitanya-bhägavata-Commentary' "
|
||
"in others-before-1977.html!"
|
||
)
|
||
sys.exit(1)
|
||
|
||
return start_idx, end_idx
|
||
|
||
|
||
def find_chapter_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
|
||
"""Find all CBHC chapter bookmark tags (p17) in the CBHC range.
|
||
|
||
Returns list of (paragraph_index, raw_title_text).
|
||
"""
|
||
bookmarks = []
|
||
for i in range(start_idx, end_idx):
|
||
p = all_paragraphs[i]
|
||
cls = get_css_class(p)
|
||
if cls == CHAPTER_BOOKMARK_CLASS:
|
||
text = decode_text(p).strip()
|
||
if text and "Caitanya-bh" in text:
|
||
bookmarks.append((i, text))
|
||
return bookmarks
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Paragraph extraction
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def extract_paragraphs(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
|
||
"""Extract body paragraphs for a chapter section.
|
||
|
||
Collects p6, p38, p55, and p9 (end markers).
|
||
Skips p3 (book header), p17 (bookmarks), p5 (chapter titles), p2, empties.
|
||
"""
|
||
result = []
|
||
|
||
for i in range(start_idx, end_idx):
|
||
p = all_paragraphs[i]
|
||
cls = get_css_class(p)
|
||
|
||
# Skip empty paragraphs
|
||
if is_empty(p):
|
||
continue
|
||
|
||
# Skip known skip classes
|
||
if cls in SKIP_CLASSES:
|
||
continue
|
||
|
||
# Skip book title header
|
||
if cls == BOOK_TITLE_CLASS:
|
||
continue
|
||
|
||
# Skip chapter bookmark headers
|
||
if cls == CHAPTER_BOOKMARK_CLASS:
|
||
continue
|
||
|
||
# Skip chapter title headings (p5)
|
||
if cls == CHAPTER_TITLE_CLASS:
|
||
continue
|
||
|
||
# Skip p2 (TOC entries)
|
||
if cls == "p2":
|
||
continue
|
||
|
||
# Body paragraphs
|
||
if cls in BODY_CLASSES:
|
||
html = decode_html_content(p)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# End-of-chapter markers (p9) — include as a paragraph
|
||
if cls == END_MARKER_CLASS:
|
||
html = decode_html_content(p)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Fallback: try to extract content from unknown classes
|
||
html = decode_html_content(p)
|
||
if html:
|
||
print(f" ⚠ Unknown class '{cls}' at paragraph {i}: {html[:80]}...")
|
||
result.append(html)
|
||
|
||
return result
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Main extraction logic
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def extract_cbhc(soup: BeautifulSoup) -> dict:
|
||
"""Extract the entire Śrī Caitanya-bhāgavata Commentary from the parsed HTML.
|
||
|
||
Returns a dict with:
|
||
"chapters": list of chapter dicts
|
||
"chapter_listing": list of {number, title} for intro.json
|
||
"""
|
||
all_paragraphs = list(soup.find_all("p"))
|
||
|
||
# Step 1: Find CBHC boundaries
|
||
cbhc_start, cbhc_end = find_cbhc_boundaries(all_paragraphs)
|
||
print(f" CBHC content: paragraphs {cbhc_start} to {cbhc_end}")
|
||
print(f" Total paragraphs in range: {cbhc_end - cbhc_start}")
|
||
|
||
# Step 2: Find all chapter bookmarks (p17)
|
||
bookmarks = find_chapter_bookmarks(all_paragraphs, cbhc_start, cbhc_end)
|
||
print(f" Found {len(bookmarks)} CBHC chapter bookmarks:")
|
||
for pos, title in bookmarks:
|
||
print(f" Paragraph {pos}: {title}")
|
||
|
||
# Step 3: Map bookmarks to chapters with proper titles
|
||
# Expected: 4 chapters
|
||
# We derive proper titles from the p5 heading that follows each bookmark
|
||
chapter_titles_map = {
|
||
1: "Chapter 1",
|
||
2: "Lord Caitanya's Appearance",
|
||
3: "Chapter 3",
|
||
4: "Chapter 4",
|
||
}
|
||
|
||
chapter_bookmarks = [] # (para_idx, chapter_num, normalized_title)
|
||
for idx, (pos, _raw_title) in enumerate(bookmarks):
|
||
ch_num = idx + 1
|
||
ch_title = chapter_titles_map.get(ch_num, f"Chapter {ch_num}")
|
||
chapter_bookmarks.append((pos, ch_num, ch_title))
|
||
|
||
print(f"\n Mapped {len(chapter_bookmarks)} chapters:")
|
||
for pos, num, title in chapter_bookmarks:
|
||
print(f" Chapter {num}: {title} (paragraph {pos})")
|
||
|
||
# Step 4: Extract chapters
|
||
chapters = []
|
||
for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_bookmarks):
|
||
# Content starts after the bookmark line (skip p17 and p5 heading)
|
||
content_start = ch_pos + 1
|
||
|
||
# Content ends at next chapter bookmark or cbhc_end
|
||
if idx + 1 < len(chapter_bookmarks):
|
||
content_end = chapter_bookmarks[idx + 1][0]
|
||
else:
|
||
content_end = cbhc_end
|
||
|
||
paragraphs = extract_paragraphs(all_paragraphs, content_start, content_end)
|
||
|
||
ch = {
|
||
"book": "cbhc",
|
||
"chapter": ch_num,
|
||
"title": ch_title,
|
||
"type": "chapter",
|
||
"paragraphs": paragraphs,
|
||
}
|
||
chapters.append(ch)
|
||
|
||
# Step 5: Build chapter listing for intro.json
|
||
chapter_listing = [
|
||
{"number": ch["chapter"], "title": ch["title"]} for ch in chapters
|
||
]
|
||
|
||
return {
|
||
"chapters": chapters,
|
||
"chapter_listing": chapter_listing,
|
||
}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Output
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def write_json(data, filepath):
|
||
"""Write data as formatted JSON to filepath, creating directories as needed."""
|
||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||
with open(filepath, "w", encoding="utf-8") as f:
|
||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||
|
||
|
||
def main():
|
||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html")
|
||
output_base = os.path.join(base_dir, "lib", "jsons", "books", "cbhc")
|
||
|
||
if not os.path.exists(html_path):
|
||
print(f"ERROR: {html_path} not found!")
|
||
sys.exit(1)
|
||
|
||
print(f"Reading {html_path}...")
|
||
with open(html_path, "r", encoding="utf-8") as f:
|
||
soup = BeautifulSoup(f, "html.parser")
|
||
|
||
print("Extracting Śrī Caitanya-bhāgavata Commentary...\n")
|
||
result = extract_cbhc(soup)
|
||
|
||
chapters = result["chapters"]
|
||
chapter_listing = result["chapter_listing"]
|
||
|
||
# --- Write intro.json ---
|
||
intro_meta = {
|
||
"book": "cbhc",
|
||
"title": "Śrī Caitanya-bhāgavata Commentary",
|
||
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
||
"year": None,
|
||
"description": (
|
||
"Commentary on Śrī Caitanya-bhāgavata by Vṛndāvana dāsa Ṭhākura "
|
||
"— covering the first four chapters of Lord Caitanya's pastimes."
|
||
),
|
||
"totalChapters": 4,
|
||
"chapters": chapter_listing,
|
||
}
|
||
intro_path = os.path.join(output_base, "intro.json")
|
||
write_json(intro_meta, intro_path)
|
||
print(f"\n Written: {intro_path}")
|
||
|
||
# --- Write chapters ---
|
||
print(f"\n--- Chapters ---")
|
||
for ch in chapters:
|
||
ch_num_str = str(ch["chapter"]).zfill(2)
|
||
filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
|
||
write_json(ch, filepath)
|
||
print(
|
||
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
|
||
f"{ch['title']}"
|
||
)
|
||
|
||
# --- Summary ---
|
||
total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
|
||
|
||
print(f"\n{'=' * 60}")
|
||
print(f" EXTRACTION SUMMARY")
|
||
print(f"{'=' * 60}")
|
||
print(f" Chapters: {len(chapters)}")
|
||
for ch in chapters:
|
||
print(
|
||
f" Ch {ch['chapter']}: {len(ch['paragraphs'])} paragraphs — {ch['title']}"
|
||
)
|
||
print(f" Total paragraphs: {total_chapter_paras}")
|
||
if chapters:
|
||
print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
|
||
print(f" Output directory: {output_base}/")
|
||
|
||
# --- Samples ---
|
||
for ch in chapters:
|
||
ch_num = ch["chapter"]
|
||
print(f"\n--- Sample: Chapter {ch_num} ({ch['title']}) ---")
|
||
for i, p in enumerate(ch["paragraphs"][:3]):
|
||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
||
print(f" [{i}] {preview}...")
|
||
if len(ch["paragraphs"]) > 3:
|
||
print(f" ... ({len(ch['paragraphs']) - 3} more paragraphs)")
|
||
|
||
# --- Quality Checks ---
|
||
print(f"\n--- Quality Checks ---")
|
||
|
||
# Check all 4 chapters present
|
||
expected_chapters = set(range(1, 5))
|
||
actual_chapters = set(ch["chapter"] for ch in chapters)
|
||
missing = expected_chapters - actual_chapters
|
||
extra = actual_chapters - expected_chapters
|
||
if missing:
|
||
print(f" ⚠ Missing chapters: {sorted(missing)}")
|
||
if extra:
|
||
print(f" ⚠ Extra chapters: {sorted(extra)}")
|
||
if not missing and not extra:
|
||
print(f" ✅ All 4 chapters present")
|
||
|
||
# Check empty chapters
|
||
empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
|
||
if empty_chapters:
|
||
print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
|
||
else:
|
||
print(f" ✅ No empty chapters")
|
||
|
||
# Check chapter 1 has "(End Chapter One)"
|
||
if chapters:
|
||
ch1_text = " ".join(chapters[0]["paragraphs"])
|
||
if "End Chapter One" in ch1_text:
|
||
print(f" ✅ '(End Chapter One)' end marker found in Chapter 1")
|
||
else:
|
||
print(f" ⚠ '(End Chapter One)' end marker NOT found in Chapter 1")
|
||
|
||
# Check chapter 2 mentions Lord Caitanya's appearance
|
||
if len(chapters) >= 2:
|
||
ch2_text = " ".join(chapters[1]["paragraphs"])
|
||
if "appearance" in ch2_text.lower() or "born" in ch2_text.lower():
|
||
print(f" ✅ Chapter 2 contains appearance/birth content")
|
||
else:
|
||
print(f" ⚠ Chapter 2 may not contain appearance content")
|
||
|
||
# Check for content leak from next work (Bhagaväner kathä)
|
||
all_text = ""
|
||
for ch in chapters:
|
||
all_text += " ".join(ch["paragraphs"])
|
||
if "Original Documents" in all_text:
|
||
print(f" ⚠ Content from next work (Bhagaväner kathä) may have leaked!")
|
||
else:
|
||
print(f" ✅ No content leak from next work detected")
|
||
|
||
# Check last chapter ends cleanly
|
||
if chapters:
|
||
last_ch = chapters[-1]
|
||
last_para = last_ch["paragraphs"][-1] if last_ch["paragraphs"] else ""
|
||
last_preview = re.sub(r"<[^>]+>", "", last_para)[:100]
|
||
print(f' ℹ Last paragraph of Ch {last_ch["chapter"]}: "{last_preview}..."')
|
||
|
||
print(f"\nDone! ✨")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|