869 lines
28 KiB
Python
869 lines
28 KiB
Python
#!/usr/bin/env python3
|
||
"""Extract Teachings of Lord Caitanya (1968 Edition) from other.html.
|
||
|
||
The book has 9 intro segments, 32 chapters in 2 parts.
|
||
|
||
IMPORTANT: This is the 1968 edition with plain English transliterations.
|
||
NO ScaGoudy font decoding is needed.
|
||
|
||
Output:
|
||
lib/jsons/books/tlc/intro.json — Book metadata
|
||
lib/jsons/books/tlc/intro/*.json — Intro segments
|
||
lib/jsons/books/tlc/chapters/*.json — Chapters
|
||
|
||
Usage:
|
||
python3 scripts/tlc_extract.py
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import re
|
||
import sys
|
||
|
||
from bs4 import BeautifulSoup, Tag
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# CSS class categorization for TLC
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# Bookmark class — used for section boundary detection
|
||
BOOKMARK_CLASS = "p13"
|
||
|
||
# Book title header class
|
||
BOOK_HEADER_CLASS = "p12"
|
||
|
||
# Body paragraph classes (primary content)
|
||
BODY_CLASSES = frozenset(["p6", "p44", "p18", "p30"])
|
||
|
||
# Section heading (bold) — "Chapter One", "Preface", etc.
|
||
SECTION_HEADING_CLASS = "p31"
|
||
|
||
# Chapter title (bold) — "Teachings to Rupa Goswami", etc.
|
||
CHAPTER_TITLE_CLASS = "p27"
|
||
|
||
# Part header class — "PART I", "PART II"
|
||
PART_HEADER_CLASS = "p34"
|
||
|
||
# Title page / copyright text classes (in intro area)
|
||
TITLE_PAGE_CLASS = "p3"
|
||
COPYRIGHT_CLASS = "p32"
|
||
|
||
# Italic verse quote and attribution
|
||
VERSE_QUOTE_CLASS = "p39"
|
||
VERSE_ATTRIBUTION_CLASS = "p40"
|
||
|
||
# Photo/painting credits
|
||
CREDITS_CLASS = "p41"
|
||
|
||
# Attribution text ("The Publishers")
|
||
ATTRIBUTION_CLASS = "p42"
|
||
|
||
# Section title bold text
|
||
SECTION_TITLE_CLASS = "p43"
|
||
|
||
# Numbered items (in Mission section)
|
||
NUMBERED_ITEM_CLASS = "p24"
|
||
|
||
# Italic text (e.g., "Hari! Hari!")
|
||
ITALIC_TEXT_CLASS = "p19"
|
||
|
||
# Classes to skip entirely (empty/decorative)
|
||
SKIP_CLASSES = frozenset(["p9", "p15"])
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Intro segment definitions
|
||
# ---------------------------------------------------------------------------
|
||
|
||
INTRO_SEGMENTS = [
|
||
{
|
||
"slug": "front-flap",
|
||
"title": "Front Flap & Pages",
|
||
"order": 1,
|
||
"bookmark_text": "TLC-1968 Front flap & pages",
|
||
},
|
||
{
|
||
"slug": "the-publishers",
|
||
"title": "The Publishers",
|
||
"order": 2,
|
||
"bookmark_text": "TLC-1968 The Publishers",
|
||
},
|
||
{
|
||
"slug": "dedication",
|
||
"title": "Dedication",
|
||
"order": 3,
|
||
"bookmark_text": "TLC-1968 DEDICATION",
|
||
},
|
||
{
|
||
"slug": "acknowledgement",
|
||
"title": "Acknowledgement",
|
||
"order": 4,
|
||
"bookmark_text": "TLC-1968 ACKNOWLEDGEMENT",
|
||
},
|
||
{
|
||
"slug": "the-author",
|
||
"title": "The Author",
|
||
"order": 5,
|
||
"bookmark_text": "TLC-1968 The Author",
|
||
},
|
||
{
|
||
"slug": "preface",
|
||
"title": "Preface",
|
||
"order": 6,
|
||
"bookmark_text": "TLC-1968 Preface",
|
||
},
|
||
{
|
||
"slug": "prologue",
|
||
"title": "Prologue",
|
||
"order": 7,
|
||
"bookmark_text": "TLC-1968 Prologue",
|
||
},
|
||
{
|
||
"slug": "mission",
|
||
"title": "Lord Chaitanya's Mission",
|
||
"order": 8,
|
||
"bookmark_text": "TLC-1968 Mission",
|
||
},
|
||
{
|
||
"slug": "introduction",
|
||
"title": "Introduction",
|
||
"order": 9,
|
||
"bookmark_text": "TLC-1968 Intoduction", # Note: typo in original
|
||
},
|
||
]
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# HTML content helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def clean_html_content(tag: Tag) -> str:
|
||
"""Extract inner HTML from a tag, strip span artifacts, preserve formatting tags.
|
||
|
||
IMPORTANT: No ScaGoudy decoding — this is the 1968 plain English edition.
|
||
"""
|
||
inner = "".join(str(child) for child in tag.children)
|
||
# Remove span tags with CSS classes (RTF conversion artifacts)
|
||
inner = re.sub(
|
||
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
|
||
)
|
||
# Clean up Apple-tab-span leftovers (tab characters)
|
||
inner = inner.replace("\t", " ")
|
||
# Normalize whitespace (but preserve <br> tags)
|
||
inner = re.sub(r"\s+", " ", inner).strip()
|
||
return inner
|
||
|
||
|
||
def get_plain_text(tag: Tag) -> str:
|
||
"""Get plain text from a tag (no decoding needed for 1968 edition)."""
|
||
return tag.get_text().strip()
|
||
|
||
|
||
def get_css_class(tag: Tag) -> str:
|
||
"""Get the first CSS class of a tag, or empty string."""
|
||
classes = tag.get("class", [])
|
||
return classes[0] if classes else ""
|
||
|
||
|
||
def is_empty(tag: Tag) -> bool:
|
||
"""Check if a tag is empty or whitespace-only."""
|
||
text = tag.get_text().strip()
|
||
return not text or text == "\xa0"
|
||
|
||
|
||
def is_dash_line(text: str) -> bool:
|
||
"""Return True if text is only dashes, spaces, and/or whitespace."""
|
||
cleaned = text.strip()
|
||
if not cleaned:
|
||
return False
|
||
return all(c in "-—–\u2014\u2013 \t" for c in cleaned)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Extraction helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def find_tlc_boundaries(all_paragraphs: list) -> tuple:
|
||
"""Find the start and end paragraph indices of the TLC content.
|
||
|
||
Start: <p class="p12"> containing "Teachings of Lord Caitanya"
|
||
End: next <p class="p12"> after start, or a dashed line followed by p3,
|
||
or end of file
|
||
"""
|
||
start_idx = None
|
||
end_idx = len(all_paragraphs)
|
||
|
||
for i, p in enumerate(all_paragraphs):
|
||
cls = get_css_class(p)
|
||
if cls != BOOK_HEADER_CLASS:
|
||
continue
|
||
text = get_plain_text(p)
|
||
if "Teachings of Lord Caitanya" in text:
|
||
start_idx = i
|
||
elif start_idx is not None:
|
||
# Next p12 after TLC start = end of TLC
|
||
end_idx = i
|
||
break
|
||
|
||
if start_idx is None:
|
||
print(
|
||
"ERROR: Could not find 'Teachings of Lord Caitanya' header in other.html!"
|
||
)
|
||
sys.exit(1)
|
||
|
||
# Also check for the dashed-line boundary before end_idx
|
||
# The last content is followed by a dashed line, then the next book starts
|
||
for i in range(start_idx + 1, end_idx):
|
||
p = all_paragraphs[i]
|
||
cls = get_css_class(p)
|
||
text = get_plain_text(p)
|
||
# A dashed line in a p6 paragraph followed by a p3 paragraph = end boundary
|
||
if cls == "p6" and is_dash_line(text):
|
||
# Check if next non-empty paragraph is p3 (start of next book)
|
||
for j in range(i + 1, min(i + 5, end_idx)):
|
||
next_cls = get_css_class(all_paragraphs[j])
|
||
if next_cls == "p3":
|
||
end_idx = i # End before the dashed line
|
||
break
|
||
if end_idx == i:
|
||
break
|
||
|
||
return start_idx, end_idx
|
||
|
||
|
||
def find_all_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
|
||
"""Find all p13 bookmark tags in the TLC range.
|
||
|
||
Returns list of (paragraph_index, bookmark_text).
|
||
"""
|
||
bookmarks = []
|
||
for i in range(start_idx, end_idx):
|
||
p = all_paragraphs[i]
|
||
cls = get_css_class(p)
|
||
if cls == BOOKMARK_CLASS:
|
||
text = get_plain_text(p)
|
||
if "TLC" in text:
|
||
bookmarks.append((i, text))
|
||
return bookmarks
|
||
|
||
|
||
def parse_chapter_bookmark(text: str) -> tuple:
|
||
"""Parse a chapter bookmark like 'TLC 1-1968: Teachings to Rupa Goswami'.
|
||
|
||
Returns (chapter_number, title) or None if not a chapter bookmark.
|
||
"""
|
||
m = re.match(r"TLC\s+(\d+)-1968:\s*(.+)", text.strip())
|
||
if m:
|
||
return int(m.group(1)), m.group(2).strip()
|
||
return None
|
||
|
||
|
||
def parse_intro_bookmark(text: str) -> str:
|
||
"""Parse an intro bookmark like 'TLC-1968 Preface'.
|
||
|
||
Returns the segment name or None if not an intro bookmark.
|
||
"""
|
||
m = re.match(r"TLC-1968\s+(.+)", text.strip())
|
||
if m:
|
||
return m.group(1).strip()
|
||
return None
|
||
|
||
|
||
def extract_paragraphs(
|
||
all_paragraphs: list,
|
||
start_idx: int,
|
||
end_idx: int,
|
||
is_intro: bool = False,
|
||
) -> list:
|
||
"""Extract paragraphs for a section (chapter or intro segment).
|
||
|
||
Handles all CSS classes within the TLC content.
|
||
No ScaGoudy decoding is applied.
|
||
"""
|
||
result = []
|
||
|
||
for i in range(start_idx, end_idx):
|
||
p = all_paragraphs[i]
|
||
cls = get_css_class(p)
|
||
|
||
# Skip empty paragraphs
|
||
if is_empty(p):
|
||
continue
|
||
|
||
# Skip noise classes
|
||
if cls in SKIP_CLASSES:
|
||
continue
|
||
|
||
# Skip bookmark tags
|
||
if cls == BOOKMARK_CLASS:
|
||
continue
|
||
|
||
# Skip book title header
|
||
if cls == BOOK_HEADER_CLASS:
|
||
continue
|
||
|
||
# Check for dash lines — skip them
|
||
text = get_plain_text(p)
|
||
if is_dash_line(text):
|
||
continue
|
||
|
||
# Part headers (p34) — include as bold paragraph
|
||
if cls == PART_HEADER_CLASS:
|
||
html = clean_html_content(p)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Section heading (p31) — include as paragraph
|
||
if cls == SECTION_HEADING_CLASS:
|
||
html = clean_html_content(p)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Chapter title (p27) — include as bold paragraph
|
||
if cls == CHAPTER_TITLE_CLASS:
|
||
html = clean_html_content(p)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Verse quote (p39) — format as centered italic div
|
||
if cls == VERSE_QUOTE_CLASS:
|
||
html = clean_html_content(p)
|
||
if html:
|
||
# Strip inner <i></i> tags before re-wrapping
|
||
stripped = re.sub(r"</?i>", "", html)
|
||
result.append(f'<div style="text-align:center"><i>{stripped}</i></div>')
|
||
continue
|
||
|
||
# Verse attribution (p40) — format as centered italic div
|
||
if cls == VERSE_ATTRIBUTION_CLASS:
|
||
html = clean_html_content(p)
|
||
if html:
|
||
stripped = re.sub(r"</?i>", "", html)
|
||
result.append(f'<div style="text-align:center"><i>{stripped}</i></div>')
|
||
continue
|
||
|
||
# Title page / copyright text (in intro area)
|
||
if cls == TITLE_PAGE_CLASS:
|
||
html = clean_html_content(p)
|
||
if html and not is_dash_line(get_plain_text(p)):
|
||
# Detect Sanskrit verses disguised as title-page paragraphs
|
||
stripped_html = html.strip()
|
||
if stripped_html.startswith("<i>") and "<br" in stripped_html:
|
||
inner = re.sub(r"</?i>", "", stripped_html)
|
||
result.append(
|
||
f'<div style="text-align:center"><i>{inner}</i></div>'
|
||
)
|
||
else:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Copyright text (p32)
|
||
if cls == COPYRIGHT_CLASS:
|
||
html = clean_html_content(p)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Photo/painting credits (p41)
|
||
if cls == CREDITS_CLASS:
|
||
html = clean_html_content(p)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Attribution text (p42) — "The Publishers"
|
||
if cls == ATTRIBUTION_CLASS:
|
||
html = clean_html_content(p)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Section title bold text (p43)
|
||
if cls == SECTION_TITLE_CLASS:
|
||
html = clean_html_content(p)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Numbered items (p24) — include as regular paragraphs
|
||
if cls == NUMBERED_ITEM_CLASS:
|
||
html = clean_html_content(p)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Italic text like "Hari! Hari!" (p19)
|
||
if cls == ITALIC_TEXT_CLASS:
|
||
html = clean_html_content(p)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Body paragraphs (p6, p44, p18, p30)
|
||
if cls in BODY_CLASSES:
|
||
html = clean_html_content(p)
|
||
if html:
|
||
# Detect Sanskrit verses disguised as body paragraphs:
|
||
# they start with <i> and contain <br> (multi-line italic = verse)
|
||
stripped_html = html.strip()
|
||
if stripped_html.startswith("<i>") and "<br" in stripped_html:
|
||
# This is a verse — wrap in centered italic div
|
||
inner = re.sub(r"</?i>", "", stripped_html)
|
||
result.append(
|
||
f'<div style="text-align:center"><i>{inner}</i></div>'
|
||
)
|
||
else:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Any other class — try to extract content (be permissive)
|
||
html = clean_html_content(p)
|
||
if html and not is_dash_line(text):
|
||
result.append(html)
|
||
|
||
return result
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Main extraction logic
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def extract_tlc(soup: BeautifulSoup) -> dict:
|
||
"""Extract the entire Teachings of Lord Caitanya from the parsed HTML.
|
||
|
||
Returns a dict with:
|
||
"intro_segments": list of intro segment dicts
|
||
"chapters": list of chapter dicts
|
||
"parts": list of part assignment dicts
|
||
"chapter_listing": list of {number, title} for intro.json
|
||
"""
|
||
all_paragraphs = list(soup.find_all("p"))
|
||
|
||
# Step 1: Find TLC boundaries
|
||
tlc_start, tlc_end = find_tlc_boundaries(all_paragraphs)
|
||
print(f" TLC content: paragraphs {tlc_start} to {tlc_end}")
|
||
print(f" Total paragraphs in range: {tlc_end - tlc_start}")
|
||
|
||
# Step 2: Find all bookmarks
|
||
bookmarks = find_all_bookmarks(all_paragraphs, tlc_start, tlc_end)
|
||
print(f" Found {len(bookmarks)} TLC bookmarks")
|
||
|
||
# Step 3: Categorize bookmarks into intro vs chapter
|
||
intro_bookmarks = [] # (para_idx, bookmark_text)
|
||
chapter_bookmarks = [] # (para_idx, chapter_num, title)
|
||
|
||
for pos, text in bookmarks:
|
||
ch = parse_chapter_bookmark(text)
|
||
if ch:
|
||
chapter_bookmarks.append((pos, ch[0], ch[1]))
|
||
else:
|
||
intro_name = parse_intro_bookmark(text)
|
||
if intro_name:
|
||
intro_bookmarks.append((pos, text))
|
||
|
||
print(f" Intro bookmarks: {len(intro_bookmarks)}")
|
||
print(f" Chapter bookmarks: {len(chapter_bookmarks)}")
|
||
|
||
# Step 4: Track PART assignments by scanning for p34 tags
|
||
part_assignments = {} # chapter_num -> part_num
|
||
parts_data = []
|
||
|
||
part_positions = [] # (para_idx, part_title)
|
||
for i in range(tlc_start, tlc_end):
|
||
p = all_paragraphs[i]
|
||
cls = get_css_class(p)
|
||
if cls == PART_HEADER_CLASS:
|
||
text = get_plain_text(p)
|
||
if text:
|
||
part_positions.append((i, text))
|
||
|
||
print(f" Part headers found: {len(part_positions)}")
|
||
for pos, title in part_positions:
|
||
print(f" Paragraph ~{pos}: {title}")
|
||
|
||
# Map part positions to part numbers
|
||
part_number_map = {}
|
||
for pos, title in part_positions:
|
||
title_upper = title.upper().strip()
|
||
if "II" in title_upper and "III" not in title_upper:
|
||
part_number_map[pos] = 2
|
||
elif "I" in title_upper:
|
||
part_number_map[pos] = 1
|
||
|
||
# Assign parts to chapters based on position
|
||
sorted_part_positions = sorted(part_number_map.keys())
|
||
for ch_pos, ch_num, ch_title in chapter_bookmarks:
|
||
assigned_part = None
|
||
for pp in sorted_part_positions:
|
||
if pp < ch_pos:
|
||
assigned_part = part_number_map[pp]
|
||
else:
|
||
break
|
||
if assigned_part:
|
||
part_assignments[ch_num] = assigned_part
|
||
|
||
# Build parts_data
|
||
for part_num in sorted(set(part_assignments.values())):
|
||
chs = sorted([ch for ch, pn in part_assignments.items() if pn == part_num])
|
||
part_title = f"Part {'I' if part_num == 1 else 'II'}"
|
||
parts_data.append(
|
||
{
|
||
"number": part_num,
|
||
"title": part_title,
|
||
"chapters": chs,
|
||
}
|
||
)
|
||
|
||
# Step 5: Collect all section start positions for boundary detection
|
||
all_section_starts = sorted(
|
||
[pos for pos, _ in intro_bookmarks] + [pos for pos, _, _ in chapter_bookmarks]
|
||
)
|
||
|
||
# Step 6: Extract intro segments
|
||
intro_segments = []
|
||
for seg_def in INTRO_SEGMENTS:
|
||
# Find matching bookmark
|
||
bookmark_pos = None
|
||
for pos, btext in intro_bookmarks:
|
||
if btext.strip() == seg_def["bookmark_text"]:
|
||
bookmark_pos = pos
|
||
break
|
||
|
||
if bookmark_pos is None:
|
||
print(f" WARNING: Could not find bookmark for '{seg_def['slug']}'")
|
||
continue
|
||
|
||
# Content starts after the bookmark
|
||
content_start = bookmark_pos + 1
|
||
|
||
# Content ends at the next bookmark or part header or tlc_end
|
||
content_end = tlc_end
|
||
for s in all_section_starts:
|
||
if s > bookmark_pos:
|
||
content_end = s
|
||
break
|
||
|
||
# Also check if a PART header comes before the next bookmark
|
||
for pp in sorted_part_positions:
|
||
if pp > bookmark_pos and pp < content_end:
|
||
content_end = pp
|
||
break
|
||
|
||
paragraphs = extract_paragraphs(
|
||
all_paragraphs, content_start, content_end, is_intro=True
|
||
)
|
||
|
||
# Use the corrected title (e.g., "Introduction" not "Intoduction")
|
||
seg = {
|
||
"slug": seg_def["slug"],
|
||
"title": seg_def["title"],
|
||
"type": "intro-segment",
|
||
"book": "tlc",
|
||
"order": seg_def["order"],
|
||
"paragraphs": paragraphs,
|
||
}
|
||
intro_segments.append(seg)
|
||
|
||
# Step 7: Extract chapters
|
||
chapters = []
|
||
for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_bookmarks):
|
||
content_start = ch_pos + 1
|
||
|
||
# Content ends at next chapter bookmark or tlc_end
|
||
if idx + 1 < len(chapter_bookmarks):
|
||
content_end = chapter_bookmarks[idx + 1][0]
|
||
else:
|
||
content_end = tlc_end
|
||
|
||
# For the PART header: if a PART header appears right before this
|
||
# chapter's bookmark, include it in this chapter's content range.
|
||
actual_start = content_start
|
||
for pp in sorted_part_positions:
|
||
if pp < ch_pos and pp >= content_start - 5:
|
||
# Check that there's no other bookmark between pp and ch_pos
|
||
has_other_bookmark = False
|
||
for bpos, _ in bookmarks:
|
||
if bpos > pp and bpos < ch_pos:
|
||
has_other_bookmark = True
|
||
break
|
||
if not has_other_bookmark:
|
||
actual_start = pp
|
||
|
||
paragraphs = extract_paragraphs(all_paragraphs, actual_start, content_end)
|
||
|
||
ch = {
|
||
"book": "tlc",
|
||
"chapter": ch_num,
|
||
"title": ch_title,
|
||
"type": "chapter",
|
||
"paragraphs": paragraphs,
|
||
}
|
||
chapters.append(ch)
|
||
|
||
# Step 8: Build chapter listing for intro.json
|
||
chapter_listing = [
|
||
{"number": ch["chapter"], "title": ch["title"]} for ch in chapters
|
||
]
|
||
|
||
return {
|
||
"intro_segments": intro_segments,
|
||
"chapters": chapters,
|
||
"parts": parts_data,
|
||
"chapter_listing": chapter_listing,
|
||
}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Output
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def write_json(data, filepath):
|
||
"""Write data as formatted JSON to filepath, creating directories as needed."""
|
||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||
with open(filepath, "w", encoding="utf-8") as f:
|
||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||
|
||
|
||
def main():
|
||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
html_path = os.path.join(base_dir, "public", "html", "other.html")
|
||
output_base = os.path.join(base_dir, "lib", "jsons", "books", "tlc")
|
||
|
||
if not os.path.exists(html_path):
|
||
print(f"ERROR: {html_path} not found!")
|
||
sys.exit(1)
|
||
|
||
print(f"Reading {html_path}...")
|
||
with open(html_path, "r", encoding="utf-8") as f:
|
||
soup = BeautifulSoup(f, "html.parser")
|
||
|
||
print("Extracting Teachings of Lord Caitanya (1968 Edition)...\n")
|
||
result = extract_tlc(soup)
|
||
|
||
intro_segments = result["intro_segments"]
|
||
chapters = result["chapters"]
|
||
parts = result["parts"]
|
||
chapter_listing = result["chapter_listing"]
|
||
|
||
# --- Write intro.json ---
|
||
intro_meta = {
|
||
"book": "tlc",
|
||
"title": "Teachings of Lord Caitanya",
|
||
"subtitle": "A Treatise on Factual Spiritual Life",
|
||
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
||
"year": 1968,
|
||
"description": "The sublime life and teachings of Lord Caitanya Mahāprabhu, the most munificent incarnation of the Supreme Lord.",
|
||
"totalChapters": 32,
|
||
"parts": parts,
|
||
"chapters": chapter_listing,
|
||
}
|
||
intro_path = os.path.join(output_base, "intro.json")
|
||
write_json(intro_meta, intro_path)
|
||
print(f"\n Written: {intro_path}")
|
||
|
||
# --- Write intro segments ---
|
||
print(f"\n--- Intro Segments ---")
|
||
for seg in intro_segments:
|
||
filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
|
||
write_json(seg, filepath)
|
||
print(
|
||
f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}"
|
||
)
|
||
|
||
# --- Write chapters ---
|
||
print(f"\n--- Chapters ---")
|
||
for ch in chapters:
|
||
ch_num_str = str(ch["chapter"]).zfill(2)
|
||
filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
|
||
write_json(ch, filepath)
|
||
part_info = ""
|
||
for p in parts:
|
||
if ch["chapter"] in p["chapters"]:
|
||
part_info = f" [Part {p['number']}]"
|
||
break
|
||
print(
|
||
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
|
||
f"{ch['title']}{part_info}"
|
||
)
|
||
|
||
# --- Summary ---
|
||
total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments)
|
||
total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
|
||
total_paras = total_intro_paras + total_chapter_paras
|
||
|
||
print(f"\n{'=' * 60}")
|
||
print(f" EXTRACTION SUMMARY")
|
||
print(f"{'=' * 60}")
|
||
print(f" Intro segments: {len(intro_segments)}")
|
||
print(f" Chapters: {len(chapters)}")
|
||
print(f" Parts: {len(parts)}")
|
||
for p in parts:
|
||
print(
|
||
f" Part {p['number']} ({p['title']}): chapters {p['chapters'][0]}-{p['chapters'][-1]} ({len(p['chapters'])} chapters)"
|
||
)
|
||
print(f" Total paragraphs: {total_paras}")
|
||
print(f" Intro: {total_intro_paras}")
|
||
print(f" Chapters: {total_chapter_paras}")
|
||
if chapters:
|
||
print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
|
||
print(f" Output directory: {output_base}/")
|
||
|
||
# --- Samples ---
|
||
print(f"\n--- Sample: Chapter 1 ---")
|
||
if chapters:
|
||
ch1 = chapters[0]
|
||
for i, p in enumerate(ch1["paragraphs"][:4]):
|
||
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
||
print(f" [{i}] {preview}...")
|
||
|
||
if chapters and len(chapters) > 1:
|
||
print(f"\n--- Sample: Chapter {chapters[-1]['chapter']} (last) ---")
|
||
last = chapters[-1]
|
||
for i, p in enumerate(last["paragraphs"][:3]):
|
||
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
||
print(f" [{i}] {preview}...")
|
||
|
||
if intro_segments:
|
||
print(f"\n--- Sample: First intro segment ({intro_segments[0]['title']}) ---")
|
||
first_intro = intro_segments[0]
|
||
for i, p in enumerate(first_intro["paragraphs"][:3]):
|
||
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
||
print(f" [{i}] {preview}...")
|
||
|
||
# --- Quality Checks ---
|
||
print(f"\n--- Quality Checks ---")
|
||
|
||
# Check all 32 chapters present
|
||
expected_chapters = set(range(1, 33))
|
||
actual_chapters = set(ch["chapter"] for ch in chapters)
|
||
missing = expected_chapters - actual_chapters
|
||
extra = actual_chapters - expected_chapters
|
||
if missing:
|
||
print(f" ⚠ Missing chapters: {sorted(missing)}")
|
||
if extra:
|
||
print(f" ⚠ Extra chapters: {sorted(extra)}")
|
||
if not missing and not extra:
|
||
print(f" ✅ All 32 chapters present")
|
||
|
||
# Check all 9 intro segments present
|
||
expected_intros = {
|
||
"front-flap",
|
||
"the-publishers",
|
||
"dedication",
|
||
"acknowledgement",
|
||
"the-author",
|
||
"preface",
|
||
"prologue",
|
||
"mission",
|
||
"introduction",
|
||
}
|
||
actual_intros = {s["slug"] for s in intro_segments}
|
||
missing_intros = expected_intros - actual_intros
|
||
extra_intros = actual_intros - expected_intros
|
||
if missing_intros:
|
||
print(f" ⚠ Missing intro segments: {missing_intros}")
|
||
if extra_intros:
|
||
print(f" ⚠ Extra intro segments: {extra_intros}")
|
||
if not missing_intros and not extra_intros:
|
||
print(f" ✅ All 9 intro segments present")
|
||
|
||
# Check empty chapters
|
||
empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
|
||
if empty_chapters:
|
||
print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
|
||
else:
|
||
print(f" ✅ No empty chapters")
|
||
|
||
# Check empty intro segments
|
||
empty_intros = [s["slug"] for s in intro_segments if len(s["paragraphs"]) == 0]
|
||
if empty_intros:
|
||
print(f" ⚠ Empty intro segments: {empty_intros}")
|
||
else:
|
||
print(f" ✅ No empty intro segments")
|
||
|
||
# Check part assignments
|
||
all_part_chapters = set()
|
||
for p in parts:
|
||
all_part_chapters.update(p["chapters"])
|
||
unassigned = [
|
||
ch["chapter"] for ch in chapters if ch["chapter"] not in all_part_chapters
|
||
]
|
||
if unassigned:
|
||
print(f" ⚠ Chapters not assigned to any part: {unassigned}")
|
||
else:
|
||
print(f" ✅ All chapters assigned to parts")
|
||
|
||
# Verify Part I has chapters 1-16 and Part II has chapters 17-32
|
||
for p in parts:
|
||
if p["number"] == 1:
|
||
expected = list(range(1, 17))
|
||
if p["chapters"] == expected:
|
||
print(f" ✅ Part I correctly has chapters 1-16")
|
||
else:
|
||
print(f" ⚠ Part I has chapters {p['chapters']}, expected {expected}")
|
||
elif p["number"] == 2:
|
||
expected = list(range(17, 33))
|
||
if p["chapters"] == expected:
|
||
print(f" ✅ Part II correctly has chapters 17-32")
|
||
else:
|
||
print(f" ⚠ Part II has chapters {p['chapters']}, expected {expected}")
|
||
|
||
# Verify no ScaGoudy artifacts (there shouldn't be any for 1968 edition)
|
||
# Look for common ScaGoudy encoded words that would indicate wrongly applied decoding
|
||
print(f"\n--- 1968 Edition Check ---")
|
||
if chapters:
|
||
ch1_text = " ".join(chapters[0]["paragraphs"][:10])
|
||
if "Krishna" in ch1_text:
|
||
print(f" ✅ 'Krishna' found in Chapter 1 (plain English as expected)")
|
||
if "Kṛṣṇa" in ch1_text:
|
||
print(
|
||
f" ℹ 'Kṛṣṇa' (IAST) found in Chapter 1 — unexpected for 1968 edition"
|
||
)
|
||
if "Goswami" in ch1_text or "Rupa Goswami" in ch1_text:
|
||
print(f" ✅ 'Goswami' found (plain English as expected)")
|
||
|
||
# Check for residual span tags
|
||
span_count = 0
|
||
for ch in chapters:
|
||
for p in ch["paragraphs"]:
|
||
if 'class="' in p and "<span" in p:
|
||
span_count += 1
|
||
for seg in intro_segments:
|
||
for p in seg["paragraphs"]:
|
||
if 'class="' in p and "<span" in p:
|
||
span_count += 1
|
||
if span_count == 0:
|
||
print(f" ✅ No residual span class artifacts found")
|
||
else:
|
||
print(f" ⚠ {span_count} paragraphs still contain span class artifacts")
|
||
|
||
# Check for dash-line paragraphs that shouldn't be there
|
||
dash_count = 0
|
||
for ch in chapters:
|
||
for p in ch["paragraphs"]:
|
||
plain = re.sub(r"<[^>]+>", "", p)
|
||
if is_dash_line(plain):
|
||
dash_count += 1
|
||
for seg in intro_segments:
|
||
for p in seg["paragraphs"]:
|
||
plain = re.sub(r"<[^>]+>", "", p)
|
||
if is_dash_line(plain):
|
||
dash_count += 1
|
||
if dash_count == 0:
|
||
print(f" ✅ No dashed-line paragraphs in output")
|
||
else:
|
||
print(f" ⚠ {dash_count} dashed-line paragraphs found in output")
|
||
|
||
print(f"\nDone! ✨")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|