859 lines
28 KiB
Python
859 lines
28 KiB
Python
#!/usr/bin/env python3
|
||
"""Extract Teachings of Lord Kapila, the Son of Devahūti (1977) from other.html.
|
||
|
||
The book has 5 intro/back-matter segments and 18 chapters.
|
||
|
||
Intro segments:
|
||
front-pages (order 1)
|
||
foreword (order 2)
|
||
about-the-author (order 97)
|
||
credits (order 98)
|
||
back-flap (order 99)
|
||
|
||
Output:
|
||
lib/jsons/books/tlk/intro.json — Book metadata
|
||
lib/jsons/books/tlk/intro/*.json — Intro segments
|
||
lib/jsons/books/tlk/chapters/*.json — Chapters (01–18)
|
||
|
||
Usage:
|
||
python3 scripts/tlk_extract.py
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import re
|
||
import sys
|
||
|
||
from bs4 import BeautifulSoup, Tag
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# ScaGoudy → IAST Unicode mapping
|
||
# ---------------------------------------------------------------------------
|
||
SCAGOUDY_TO_IAST = {
|
||
"ä": "ā",
|
||
"Ä": "Ā",
|
||
"å": "ṛ",
|
||
"Å": "Ṛ",
|
||
"ñ": "ṣ",
|
||
"Ñ": "Ṣ",
|
||
"ö": "ṭ",
|
||
"Ö": "Ṭ",
|
||
"ë": "ṇ",
|
||
"Ë": "Ṇ",
|
||
"ò": "ḍ",
|
||
"Ò": "Ḍ",
|
||
"ç": "ś",
|
||
"Ç": "Ś",
|
||
"ì": "ṅ",
|
||
"ï": "ñ",
|
||
"ü": "ū",
|
||
"Ü": "Ū",
|
||
"é": "ī",
|
||
"É": "Ī",
|
||
"à": "ṁ",
|
||
"ù": "ḥ",
|
||
"è": "ṝ",
|
||
"û": "ṁl",
|
||
"ÿ": "ḷ",
|
||
}
|
||
|
||
|
||
def decode_scagoudy(text: str) -> str:
|
||
"""Replace ScaGoudy-encoded characters with proper IAST Unicode."""
|
||
for src, dst in SCAGOUDY_TO_IAST.items():
|
||
text = text.replace(src, dst)
|
||
return text
|
||
|
||
|
||
def decode_html_content(tag: Tag) -> str:
|
||
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
|
||
inner = "".join(str(child) for child in tag.children)
|
||
inner = decode_scagoudy(inner)
|
||
# Remove span tags with CSS classes (RTF conversion artifacts)
|
||
inner = re.sub(
|
||
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
|
||
)
|
||
# Clean up Apple-tab-span leftovers (tab characters)
|
||
inner = inner.replace("\t", " ")
|
||
# Normalize whitespace
|
||
inner = re.sub(r"\s+", " ", inner).strip()
|
||
return inner
|
||
|
||
|
||
def decode_text(tag: Tag) -> str:
|
||
"""Get plain text from a tag with ScaGoudy decoding."""
|
||
return decode_scagoudy(tag.get_text().strip())
|
||
|
||
|
||
def get_css_class(tag: Tag) -> str:
|
||
"""Get the first CSS class of a tag, or empty string."""
|
||
classes = tag.get("class", [])
|
||
return classes[0] if classes else ""
|
||
|
||
|
||
def is_empty(tag: Tag) -> bool:
|
||
"""Check if a tag is empty or whitespace-only."""
|
||
text = tag.get_text().strip()
|
||
return not text or text == "\xa0"
|
||
|
||
|
||
def is_dash_line(text: str) -> bool:
|
||
"""Return True if text is only dashes, spaces, and/or whitespace."""
|
||
cleaned = text.strip()
|
||
if not cleaned:
|
||
return False
|
||
return all(c in "-—–\u2014\u2013 \t" for c in cleaned)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# CSS class categorization for TLK
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# Book title header class (bold, <p class="p12">)
|
||
BOOK_HEADER_CLASS = "p12"
|
||
|
||
# Bookmark class — used for section boundary detection
|
||
BOOKMARK_CLASS = "p13"
|
||
|
||
# Chapter heading (bold, e.g. "CHAPTER ONE\nThe Purpose of Lord Kapila's Advent")
|
||
CHAPTER_HEADING_CLASS = "p27"
|
||
|
||
# Section heading (bold, e.g. "Foreword", "Credits")
|
||
SECTION_HEADING_CLASS = "p31"
|
||
|
||
# Body paragraphs — THE main content class
|
||
BODY_CLASS = "p6"
|
||
|
||
# Sanskrit verse text (italic, multi-line with <br>)
|
||
SANSKRIT_VERSE_CLASS = "p53"
|
||
|
||
# Verse translation/reference text (after Sanskrit verses, mid-purport quotes)
|
||
VERSE_TRANSLATION_CLASS = "p44"
|
||
|
||
# Internal verse bookmark (e.g. "TLK Vs 1", "TLK Vs 2")
|
||
VERSE_BOOKMARK_CLASS = "p49"
|
||
|
||
# TEXT N / TRANSLATION / PURPORT headers (bold)
|
||
STRUCTURED_HEADER_CLASS = "p55"
|
||
|
||
# Translation text (bold — verse translation after TRANSLATION header)
|
||
TRANSLATION_TEXT_CLASS = "p211"
|
||
|
||
# Verse reference (e.g. [Śrīmad-Bhāgavatam 3.25.1])
|
||
VERSE_REFERENCE_CLASS = "p212"
|
||
|
||
# Publication info text
|
||
PUBLICATION_INFO_CLASS = "p128"
|
||
|
||
# Attribution line (e.g. "—The Publishers")
|
||
ATTRIBUTION_CLASS = "p129"
|
||
|
||
# Block quote (bold, testimonials in Credits section)
|
||
BLOCKQUOTE_CLASS = "p47"
|
||
|
||
# Attribution for block quotes
|
||
QUOTE_ATTRIBUTION_CLASS = "p42"
|
||
|
||
# Jacket/design credits
|
||
JACKET_CREDITS_CLASS = "p3"
|
||
|
||
# Classes to skip entirely (empty/break)
|
||
SKIP_CLASSES = frozenset(["p15"])
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Extraction helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def find_tlk_boundaries(all_paragraphs: list) -> tuple:
|
||
"""Find the start and end paragraph indices of the TLK content.
|
||
|
||
Start: <p class="p12"> containing "Teachings of Lord Kapila"
|
||
End: next <p class="p12"> which is "Teachings of Queen Kunté" (or similar)
|
||
"""
|
||
start_idx = None
|
||
end_idx = len(all_paragraphs)
|
||
|
||
for i, p in enumerate(all_paragraphs):
|
||
cls = get_css_class(p)
|
||
text = decode_text(p)
|
||
|
||
# Find start
|
||
if start_idx is None:
|
||
if cls == BOOK_HEADER_CLASS and "Teachings of Lord Kapila" in text:
|
||
start_idx = i
|
||
continue
|
||
|
||
# Find end: the next p12 book header after our start
|
||
if cls == BOOK_HEADER_CLASS:
|
||
end_idx = i
|
||
break
|
||
|
||
if start_idx is None:
|
||
print("ERROR: Could not find 'Teachings of Lord Kapila' header in other.html!")
|
||
sys.exit(1)
|
||
|
||
return start_idx, end_idx
|
||
|
||
|
||
def find_all_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
|
||
"""Find all TLK bookmark tags in the TLK range.
|
||
|
||
Bookmarks are p13 class with text starting with "TLK".
|
||
Returns list of (paragraph_index, bookmark_text).
|
||
"""
|
||
bookmarks = []
|
||
for i in range(start_idx, end_idx):
|
||
p = all_paragraphs[i]
|
||
cls = get_css_class(p)
|
||
if cls != BOOKMARK_CLASS:
|
||
continue
|
||
text = decode_text(p).strip()
|
||
if text.startswith("TLK"):
|
||
bookmarks.append((i, text))
|
||
return bookmarks
|
||
|
||
|
||
def parse_chapter_bookmark(text: str):
|
||
"""Parse a chapter bookmark like 'TLK 1: The Purpose of Lord Kapila's Advent'.
|
||
|
||
Returns (chapter_number, title) or None if not a chapter bookmark.
|
||
"""
|
||
m = re.match(r"TLK\s+(\d+):\s+(.+)", text.strip())
|
||
if m:
|
||
num = int(m.group(1))
|
||
title = m.group(2).strip()
|
||
return num, title
|
||
return None
|
||
|
||
|
||
def parse_intro_bookmark(text: str):
|
||
"""Check if this is a front-pages or foreword bookmark.
|
||
|
||
Returns the slug or None.
|
||
"""
|
||
stripped = text.strip()
|
||
# "TLK Into" is the front-pages bookmark (note: "Into" not "Intro")
|
||
if stripped == "TLK Into":
|
||
return "front-pages"
|
||
if stripped == "TLK Foreword":
|
||
return "foreword"
|
||
return None
|
||
|
||
|
||
def parse_backmatter_bookmark(text: str):
|
||
"""Check if this is a back-matter bookmark.
|
||
|
||
Returns the slug or None.
|
||
"""
|
||
stripped = text.strip()
|
||
if stripped == "TLK The Author":
|
||
return "about-the-author"
|
||
if stripped == "TLK Credits":
|
||
return "credits"
|
||
if stripped == "TLK Back Flap":
|
||
return "back-flap"
|
||
return None
|
||
|
||
|
||
INTRO_SLUG_ORDER = {
|
||
"front-pages": 1,
|
||
"foreword": 2,
|
||
}
|
||
|
||
BACKMATTER_SLUG_ORDER = {
|
||
"about-the-author": 97,
|
||
"credits": 98,
|
||
"back-flap": 99,
|
||
}
|
||
|
||
INTRO_SLUG_TITLE = {
|
||
"front-pages": "Front Pages",
|
||
"foreword": "Foreword",
|
||
"about-the-author": "About the Author",
|
||
"credits": "Credits",
|
||
"back-flap": "Back Flap",
|
||
}
|
||
|
||
|
||
def extract_paragraphs(
|
||
all_paragraphs: list,
|
||
start_idx: int,
|
||
end_idx: int,
|
||
) -> list:
|
||
"""Extract paragraphs for a section (chapter, intro segment).
|
||
|
||
Handles all CSS classes within the TLK content.
|
||
Handles p53 Sanskrit verses as centered italic divs.
|
||
Handles structured verse headers (TEXT N, TRANSLATION, PURPORT).
|
||
"""
|
||
result = []
|
||
|
||
for i in range(start_idx, end_idx):
|
||
p = all_paragraphs[i]
|
||
cls = get_css_class(p)
|
||
|
||
# Skip empty paragraphs
|
||
if is_empty(p):
|
||
continue
|
||
|
||
# Skip noise/break classes
|
||
if cls in SKIP_CLASSES:
|
||
continue
|
||
|
||
# Skip bookmark tags (p13)
|
||
if cls == BOOKMARK_CLASS:
|
||
continue
|
||
|
||
# Skip book title header (p12)
|
||
if cls == BOOK_HEADER_CLASS:
|
||
continue
|
||
|
||
# Skip internal verse bookmarks (p49 containing "TLK Vs")
|
||
if cls == VERSE_BOOKMARK_CLASS:
|
||
text = decode_text(p)
|
||
if "TLK Vs" in text or "TLK" in text:
|
||
continue
|
||
|
||
# Check for dash lines — skip them
|
||
text = decode_text(p)
|
||
if is_dash_line(text):
|
||
continue
|
||
|
||
# Sanskrit verse text (p53) — italic, multi-line with <br>
|
||
if cls == SANSKRIT_VERSE_CLASS:
|
||
html = decode_html_content(p)
|
||
if html:
|
||
# Strip outer <i> tags, then re-wrap in centered italic div
|
||
inner = re.sub(r"</?i>", "", html)
|
||
result.append(f'<div style="text-align:center"><i>{inner}</i></div>')
|
||
continue
|
||
|
||
# Structured headers: TEXT N, TRANSLATION, PURPORT (p55)
|
||
if cls == STRUCTURED_HEADER_CLASS:
|
||
html = decode_html_content(p)
|
||
if html:
|
||
# These are already bold in the source; ensure bold wrapping
|
||
if not html.strip().startswith("<b>"):
|
||
html = f"<b>{html}</b>"
|
||
result.append(html)
|
||
continue
|
||
|
||
# Translation text (p211) — bold verse translation
|
||
if cls == TRANSLATION_TEXT_CLASS:
|
||
html = decode_html_content(p)
|
||
if html:
|
||
if not html.strip().startswith("<b>"):
|
||
html = f"<b>{html}</b>"
|
||
result.append(html)
|
||
continue
|
||
|
||
# Verse reference (p212) — e.g. [Śrīmad-Bhāgavatam 3.25.1]
|
||
if cls == VERSE_REFERENCE_CLASS:
|
||
html = decode_html_content(p)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Body paragraphs (p6) — the main content class
|
||
if cls == BODY_CLASS:
|
||
html = decode_html_content(p)
|
||
if html:
|
||
# Detect Sanskrit verses disguised as body paragraphs:
|
||
# they start with <i> and contain <br> (multi-line italic = verse)
|
||
stripped_html = html.strip()
|
||
if stripped_html.startswith("<i>") and "<br" in stripped_html:
|
||
inner = re.sub(r"</?i>", "", stripped_html)
|
||
result.append(
|
||
f'<div style="text-align:center"><i>{inner}</i></div>'
|
||
)
|
||
else:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Verse translation text (p44) — after Sanskrit verses mid-purport
|
||
if cls == VERSE_TRANSLATION_CLASS:
|
||
html = decode_html_content(p)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Chapter heading (p27) — bold chapter title
|
||
if cls == CHAPTER_HEADING_CLASS:
|
||
html = decode_html_content(p)
|
||
if html:
|
||
if not html.strip().startswith("<b>"):
|
||
html = f"<b>{html}</b>"
|
||
result.append(html)
|
||
continue
|
||
|
||
# Section heading (p31) — bold section title
|
||
if cls == SECTION_HEADING_CLASS:
|
||
html = decode_html_content(p)
|
||
if html:
|
||
if not html.strip().startswith("<b>"):
|
||
html = f"<b>{html}</b>"
|
||
result.append(html)
|
||
continue
|
||
|
||
# Publication info (p128)
|
||
if cls == PUBLICATION_INFO_CLASS:
|
||
html = decode_html_content(p)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Attribution (p129) — e.g. "—The Publishers"
|
||
if cls == ATTRIBUTION_CLASS:
|
||
html = decode_html_content(p)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Block quotes (p47) — testimonials in Credits section
|
||
if cls == BLOCKQUOTE_CLASS:
|
||
html = decode_html_content(p)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Quote attribution (p42)
|
||
if cls == QUOTE_ATTRIBUTION_CLASS:
|
||
html = decode_html_content(p)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Jacket/design credits (p3)
|
||
if cls == JACKET_CREDITS_CLASS:
|
||
html = decode_html_content(p)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Any other class — try to extract content (be permissive)
|
||
html = decode_html_content(p)
|
||
if html and not is_dash_line(text):
|
||
result.append(html)
|
||
|
||
return result
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Main extraction logic
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def extract_tlk(soup: BeautifulSoup) -> dict:
|
||
"""Extract the entire Teachings of Lord Kapila from the parsed HTML.
|
||
|
||
Returns a dict with:
|
||
"intro_segments": list of intro segment dicts
|
||
"chapters": list of chapter dicts
|
||
"chapter_listing": list of {number, title} for intro.json
|
||
"""
|
||
all_paragraphs = list(soup.find_all("p"))
|
||
|
||
# Step 1: Find TLK boundaries
|
||
tlk_start, tlk_end = find_tlk_boundaries(all_paragraphs)
|
||
print(f" TLK content: paragraphs {tlk_start} to {tlk_end}")
|
||
print(f" Total paragraphs in range: {tlk_end - tlk_start}")
|
||
|
||
# Step 2: Find all bookmarks
|
||
bookmarks = find_all_bookmarks(all_paragraphs, tlk_start, tlk_end)
|
||
print(f" Found {len(bookmarks)} TLK bookmarks")
|
||
for pos, text in bookmarks:
|
||
print(f" Para ~{pos}: {text}")
|
||
|
||
# Step 3: Categorize bookmarks
|
||
intro_bookmarks = {} # slug -> paragraph index
|
||
backmatter_bookmarks = {} # slug -> paragraph index
|
||
chapter_bookmarks = [] # (para_idx, chapter_num, title)
|
||
|
||
for pos, text in bookmarks:
|
||
# Check chapter
|
||
ch = parse_chapter_bookmark(text)
|
||
if ch:
|
||
chapter_bookmarks.append((pos, ch[0], ch[1]))
|
||
continue
|
||
|
||
# Check intro (front-pages, foreword)
|
||
intro_slug = parse_intro_bookmark(text)
|
||
if intro_slug:
|
||
intro_bookmarks[intro_slug] = pos
|
||
continue
|
||
|
||
# Check back matter (about-the-author, credits, back-flap)
|
||
bm_slug = parse_backmatter_bookmark(text)
|
||
if bm_slug:
|
||
backmatter_bookmarks[bm_slug] = pos
|
||
continue
|
||
|
||
print(f" WARNING: Unrecognized bookmark: {text}")
|
||
|
||
print(f" Intro bookmarks: {list(intro_bookmarks.keys())}")
|
||
print(f" Chapter bookmarks: {len(chapter_bookmarks)}")
|
||
print(f" Back matter bookmarks: {list(backmatter_bookmarks.keys())}")
|
||
|
||
# Build an ordered list of all section start positions for boundary computation
|
||
all_section_starts = sorted(
|
||
list(intro_bookmarks.values())
|
||
+ [pos for pos, _, _ in chapter_bookmarks]
|
||
+ list(backmatter_bookmarks.values())
|
||
)
|
||
|
||
def next_section_after(pos):
|
||
"""Return the next section start after `pos`, or tlk_end."""
|
||
for s in all_section_starts:
|
||
if s > pos:
|
||
return s
|
||
return tlk_end
|
||
|
||
# Step 4: Extract intro segments (front-pages, foreword)
|
||
intro_segments = []
|
||
for slug in ["front-pages", "foreword"]:
|
||
if slug in intro_bookmarks:
|
||
bm_pos = intro_bookmarks[slug]
|
||
content_start = bm_pos + 1
|
||
content_end = next_section_after(bm_pos)
|
||
paras = extract_paragraphs(all_paragraphs, content_start, content_end)
|
||
seg = {
|
||
"slug": slug,
|
||
"title": INTRO_SLUG_TITLE[slug],
|
||
"type": "intro-segment",
|
||
"book": "tlk",
|
||
"order": INTRO_SLUG_ORDER[slug],
|
||
"paragraphs": paras,
|
||
}
|
||
intro_segments.append(seg)
|
||
|
||
# Step 5: Extract chapters 1–18
|
||
chapters = []
|
||
for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_bookmarks):
|
||
content_start = ch_pos + 1
|
||
content_end = next_section_after(ch_pos)
|
||
|
||
paragraphs = extract_paragraphs(all_paragraphs, content_start, content_end)
|
||
|
||
ch = {
|
||
"book": "tlk",
|
||
"chapter": ch_num,
|
||
"title": ch_title,
|
||
"type": "chapter",
|
||
"paragraphs": paragraphs,
|
||
}
|
||
chapters.append(ch)
|
||
|
||
# Step 6: Extract back matter segments (about-the-author, credits, back-flap)
|
||
backmatter_segments = []
|
||
for slug in ["about-the-author", "credits", "back-flap"]:
|
||
if slug in backmatter_bookmarks:
|
||
bm_pos = backmatter_bookmarks[slug]
|
||
content_start = bm_pos + 1
|
||
content_end = next_section_after(bm_pos)
|
||
paras = extract_paragraphs(all_paragraphs, content_start, content_end)
|
||
seg = {
|
||
"slug": slug,
|
||
"title": INTRO_SLUG_TITLE[slug],
|
||
"type": "intro-segment",
|
||
"book": "tlk",
|
||
"order": BACKMATTER_SLUG_ORDER[slug],
|
||
"paragraphs": paras,
|
||
}
|
||
backmatter_segments.append(seg)
|
||
|
||
# Combine all intro segments (front + back matter)
|
||
all_intro_segments = intro_segments + backmatter_segments
|
||
|
||
# Step 7: Build chapter listing for intro.json
|
||
chapter_listing = [
|
||
{"number": ch["chapter"], "title": ch["title"]} for ch in chapters
|
||
]
|
||
|
||
return {
|
||
"intro_segments": all_intro_segments,
|
||
"chapters": chapters,
|
||
"chapter_listing": chapter_listing,
|
||
}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Output
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def write_json(data, filepath):
|
||
"""Write data as formatted JSON to filepath, creating directories as needed."""
|
||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||
with open(filepath, "w", encoding="utf-8") as f:
|
||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||
|
||
|
||
def main():
|
||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
html_path = os.path.join(base_dir, "public", "html", "other.html")
|
||
output_base = os.path.join(base_dir, "lib", "jsons", "books", "tlk")
|
||
|
||
if not os.path.exists(html_path):
|
||
print(f"ERROR: {html_path} not found!")
|
||
sys.exit(1)
|
||
|
||
print(f"Reading {html_path}...")
|
||
with open(html_path, "r", encoding="utf-8") as f:
|
||
soup = BeautifulSoup(f, "html.parser")
|
||
|
||
print("Extracting Teachings of Lord Kapila, the Son of Devahūti (1977)...\n")
|
||
result = extract_tlk(soup)
|
||
|
||
intro_segments = result["intro_segments"]
|
||
chapters = result["chapters"]
|
||
chapter_listing = result["chapter_listing"]
|
||
|
||
# --- Write intro.json ---
|
||
intro_meta = {
|
||
"book": "tlk",
|
||
"title": "Teachings of Lord Kapila, the Son of Devahūti",
|
||
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
||
"year": 1977,
|
||
"description": "Lord Kapila's teachings on Sāṅkhya philosophy and bhakti-yoga to His mother Devahūti.",
|
||
"totalChapters": 18,
|
||
"chapters": chapter_listing,
|
||
}
|
||
intro_path = os.path.join(output_base, "intro.json")
|
||
write_json(intro_meta, intro_path)
|
||
print(f"\n Written: {intro_path}")
|
||
|
||
# --- Write intro segments ---
|
||
print(f"\n--- Intro / Back-Matter Segments ---")
|
||
for seg in intro_segments:
|
||
filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
|
||
write_json(seg, filepath)
|
||
print(
|
||
f" {seg['order']:>2}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}"
|
||
)
|
||
|
||
# --- Write chapters ---
|
||
print(f"\n--- Chapters ---")
|
||
for ch in chapters:
|
||
ch_num_str = str(ch["chapter"]).zfill(2)
|
||
filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
|
||
write_json(ch, filepath)
|
||
print(
|
||
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
|
||
f"{ch['title']}"
|
||
)
|
||
|
||
# --- Summary ---
|
||
total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments)
|
||
total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
|
||
total_paras = total_intro_paras + total_chapter_paras
|
||
|
||
print(f"\n{'=' * 60}")
|
||
print(f" EXTRACTION SUMMARY")
|
||
print(f"{'=' * 60}")
|
||
print(f" Intro/back-matter segments: {len(intro_segments)}")
|
||
for seg in intro_segments:
|
||
print(
|
||
f" {seg['order']:>2}. {seg['title']}: {len(seg['paragraphs'])} paragraphs"
|
||
)
|
||
print(f" Chapters: {len(chapters)}")
|
||
print(f" Total paragraphs: {total_paras}")
|
||
print(f" Intro/back: {total_intro_paras}")
|
||
print(f" Chapters: {total_chapter_paras}")
|
||
if chapters:
|
||
print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
|
||
print(f" Output directory: {output_base}/")
|
||
|
||
# --- Samples ---
|
||
print(f"\n--- Sample: Chapter 1 ---")
|
||
if chapters:
|
||
ch1 = chapters[0]
|
||
for i, p in enumerate(ch1["paragraphs"][:5]):
|
||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
||
print(f" [{i}] {preview}...")
|
||
|
||
if len(chapters) >= 9:
|
||
print(f"\n--- Sample: Chapter 9 ---")
|
||
ch9 = [c for c in chapters if c["chapter"] == 9]
|
||
if ch9:
|
||
for i, p in enumerate(ch9[0]["paragraphs"][:3]):
|
||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
||
print(f" [{i}] {preview}...")
|
||
|
||
if len(chapters) >= 18:
|
||
print(f"\n--- Sample: Chapter 18 (last) ---")
|
||
last = chapters[-1]
|
||
for i, p in enumerate(last["paragraphs"][:3]):
|
||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
||
print(f" [{i}] {preview}...")
|
||
|
||
# --- Quality Checks ---
|
||
print(f"\n--- Quality Checks ---")
|
||
|
||
# Check all 18 chapters present
|
||
expected_chapters = set(range(1, 19))
|
||
actual_chapters = set(ch["chapter"] for ch in chapters)
|
||
missing = expected_chapters - actual_chapters
|
||
extra = actual_chapters - expected_chapters
|
||
if missing:
|
||
print(f" ⚠ Missing chapters: {sorted(missing)}")
|
||
if extra:
|
||
print(f" ⚠ Extra chapters: {sorted(extra)}")
|
||
if not missing and not extra:
|
||
print(f" ✅ All 18 chapters present")
|
||
|
||
# Check intro segments
|
||
expected_intros = {
|
||
"front-pages",
|
||
"foreword",
|
||
"about-the-author",
|
||
"credits",
|
||
"back-flap",
|
||
}
|
||
actual_intros = {s["slug"] for s in intro_segments}
|
||
missing_intros = expected_intros - actual_intros
|
||
extra_intros = actual_intros - expected_intros
|
||
if missing_intros:
|
||
print(f" ⚠ Missing intro segments: {missing_intros}")
|
||
if extra_intros:
|
||
print(f" ⚠ Extra intro segments: {extra_intros}")
|
||
if not missing_intros and not extra_intros:
|
||
print(f" ✅ All 5 intro/back-matter segments present")
|
||
|
||
# Check empty chapters
|
||
empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
|
||
if empty_chapters:
|
||
print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
|
||
else:
|
||
print(f" ✅ No empty chapters")
|
||
|
||
# Check empty intro segments
|
||
empty_intros = [s["slug"] for s in intro_segments if len(s["paragraphs"]) == 0]
|
||
if empty_intros:
|
||
print(f" ⚠ Empty intro segments (0 paragraphs): {empty_intros}")
|
||
else:
|
||
print(f" ✅ No empty intro segments")
|
||
|
||
# Check ScaGoudy decoding — look for common encoded characters
|
||
encoded_chars = set("äåöëòçüéàùè")
|
||
suspect_count = 0
|
||
all_sections = chapters + intro_segments
|
||
for section in all_sections:
|
||
for p in section["paragraphs"]:
|
||
plain = re.sub(r"<[^>]+>", "", p)
|
||
for c in encoded_chars:
|
||
if c in plain:
|
||
suspect_count += 1
|
||
break
|
||
if suspect_count == 0:
|
||
print(f" ✅ No suspected undecoded ScaGoudy characters found")
|
||
else:
|
||
print(
|
||
f" ℹ {suspect_count} paragraphs contain characters that may be undecoded ScaGoudy"
|
||
)
|
||
|
||
# Check verse blocks (centered italic divs)
|
||
verse_block_count = 0
|
||
for ch in chapters:
|
||
for p in ch["paragraphs"]:
|
||
if '<div style="text-align:center"><i>' in p:
|
||
verse_block_count += 1
|
||
print(f" ℹ {verse_block_count} Sanskrit verse blocks in chapters")
|
||
|
||
# Count structured verse headers (TEXT N, TRANSLATION, PURPORT)
|
||
text_n_count = 0
|
||
translation_header_count = 0
|
||
purport_header_count = 0
|
||
for ch in chapters:
|
||
for p in ch["paragraphs"]:
|
||
plain = re.sub(r"<[^>]+>", "", p).strip()
|
||
if re.match(r"TEXT\s+\d+", plain):
|
||
text_n_count += 1
|
||
elif plain == "TRANSLATION":
|
||
translation_header_count += 1
|
||
elif plain == "PURPORT":
|
||
purport_header_count += 1
|
||
print(f" ℹ TEXT N headers: {text_n_count}")
|
||
print(f" ℹ TRANSLATION headers: {translation_header_count}")
|
||
print(f" ℹ PURPORT headers: {purport_header_count}")
|
||
|
||
# ScaGoudy decoding spot check
|
||
print(f"\n--- ScaGoudy Decoding Spot Check ---")
|
||
if chapters:
|
||
ch1_text = " ".join(chapters[0]["paragraphs"][:20])
|
||
if "Prabhupāda" in ch1_text:
|
||
print(f" ✅ 'Prabhupāda' correctly decoded")
|
||
elif "Prabhupäda" in ch1_text:
|
||
print(f" ⚠ ScaGoudy decoding incomplete for 'Prabhupäda'")
|
||
else:
|
||
print(f" ℹ 'Prabhupāda' not found in Chapter 1 first 20 paragraphs")
|
||
|
||
if "Kṛṣṇa" in ch1_text:
|
||
print(f" ✅ 'Kṛṣṇa' correctly decoded in Chapter 1")
|
||
elif "Kåñëa" in ch1_text:
|
||
print(f" ⚠ 'Kåñëa' still encoded — ScaGoudy decoding may have failed!")
|
||
else:
|
||
print(
|
||
f" ℹ 'Kṛṣṇa' not found in Chapter 1 first 20 paragraphs (may be normal)"
|
||
)
|
||
|
||
# Check for Kapila
|
||
if "Kapila" in ch1_text:
|
||
print(f" ✅ 'Kapila' found in Chapter 1")
|
||
|
||
# Check for Devahūti
|
||
if "Devahūti" in ch1_text:
|
||
print(f" ✅ 'Devahūti' correctly decoded")
|
||
elif "Devahüti" in ch1_text:
|
||
print(f" ⚠ 'Devahüti' still encoded")
|
||
|
||
# Check for Sāṅkhya
|
||
if "Sāṅkhya" in ch1_text or "sāṅkhya" in ch1_text or "Sāṇkhya" in ch1_text:
|
||
print(f" ✅ 'Sāṅkhya' correctly decoded")
|
||
|
||
# Sample verse blocks
|
||
print(f"\n--- Sample Sanskrit Verse Blocks ---")
|
||
shown = 0
|
||
for ch in chapters:
|
||
for p in ch["paragraphs"]:
|
||
if '<div style="text-align:center"><i>' in p:
|
||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
||
print(f" Ch {ch['chapter']}: {preview}...")
|
||
shown += 1
|
||
if shown >= 3:
|
||
break
|
||
if shown >= 3:
|
||
break
|
||
|
||
# Sample structured verse sections (TEXT N + TRANSLATION)
|
||
print(f"\n--- Sample Structured Verse Sections ---")
|
||
shown = 0
|
||
for ch in chapters[:3]:
|
||
for i, p in enumerate(ch["paragraphs"]):
|
||
plain = re.sub(r"<[^>]+>", "", p).strip()
|
||
if re.match(r"TEXT\s+\d+", plain) and shown < 3:
|
||
# Show TEXT header and next few paragraphs
|
||
print(f" Ch {ch['chapter']}:")
|
||
for j in range(i, min(i + 5, len(ch["paragraphs"]))):
|
||
pj_preview = re.sub(r"<[^>]+>", "", ch["paragraphs"][j])[:100]
|
||
print(f" [{j}] {pj_preview}...")
|
||
shown += 1
|
||
print()
|
||
if shown >= 3:
|
||
break
|
||
|
||
# Check intro segments content
|
||
print(f"--- Intro Segment Samples ---")
|
||
for seg in intro_segments:
|
||
print(f"\n {seg['title']} ({seg['slug']}, order {seg['order']}):")
|
||
for i, p in enumerate(seg["paragraphs"][:2]):
|
||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
||
print(f" [{i}] {preview}...")
|
||
|
||
print(f"\nDone! ✨")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|