368 lines
11 KiB
Python
368 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Extract Kṛṣṇa Book Volume 2 chapters from other.html.
|
|
|
|
Volume 2 has 35 chapters (KB 1970-2-1 through KB 1970-2-35).
|
|
Each chapter is a prose chapter (no verses) — just paragraphs of purport text.
|
|
|
|
The chapters are delimited by p13 bookmark tags: "KB 1970-2-N / Title"
|
|
Each chapter starts with a p27 heading: "N /Title"
|
|
Body paragraphs are in p6 (primary) and p28 classes.
|
|
Chapter end markers: "Thus ends the Bhaktivedanta purport of the Nth Chapter..."
|
|
|
|
Output schema (one JSON per chapter):
|
|
{
|
|
"book": "kb",
|
|
"volume": 2,
|
|
"chapter": 1,
|
|
"title": "The Story of the Syamantaka Jewel",
|
|
"type": "chapter",
|
|
"paragraphs": ["<p>...</p>", ...]
|
|
}
|
|
|
|
Output: lib/jsons/books/kb/vol-2/chapters/01.json through 35.json
|
|
|
|
Usage:
|
|
python3 scripts/kb_extract_vol2_chapters.py
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
from bs4 import BeautifulSoup, Tag
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ScaGoudy → IAST Unicode mapping
|
|
# ---------------------------------------------------------------------------
|
|
SCAGOUDY_TO_IAST = {
|
|
"ä": "ā",
|
|
"Ä": "Ā",
|
|
"å": "ṛ",
|
|
"Å": "Ṛ",
|
|
"ñ": "ṣ",
|
|
"Ñ": "Ṣ",
|
|
"ö": "ṭ",
|
|
"Ö": "Ṭ",
|
|
"ë": "ṇ",
|
|
"Ë": "Ṇ",
|
|
"ò": "ḍ",
|
|
"Ò": "Ḍ",
|
|
"ç": "ś",
|
|
"Ç": "Ś",
|
|
"ì": "ṅ",
|
|
"ï": "ñ",
|
|
"ü": "ū",
|
|
"Ü": "Ū",
|
|
"é": "ī",
|
|
"É": "Ī",
|
|
"à": "ṁ",
|
|
"ù": "ḥ",
|
|
"è": "ṝ",
|
|
"û": "ṁl",
|
|
"ÿ": "ḷ",
|
|
}
|
|
|
|
|
|
def decode_scagoudy(text: str) -> str:
|
|
for src, dst in SCAGOUDY_TO_IAST.items():
|
|
text = text.replace(src, dst)
|
|
return text
|
|
|
|
|
|
def decode_html_content(tag: Tag) -> str:
|
|
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
|
|
inner = "".join(str(child) for child in tag.children)
|
|
inner = decode_scagoudy(inner)
|
|
# Remove span tags with CSS classes (RTF conversion artifacts)
|
|
inner = re.sub(r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner)
|
|
inner = re.sub(r"\s+", " ", inner).strip()
|
|
return inner
|
|
|
|
|
|
def decode_text(tag: Tag) -> str:
|
|
return decode_scagoudy(tag.get_text().strip())
|
|
|
|
|
|
def get_css_class(tag: Tag) -> str:
|
|
classes = tag.get("class", [])
|
|
return classes[0] if classes else ""
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# CSS classes for Sanskrit verse pādas (center-aligned italic blocks
|
|
# that sometimes appear within chapter prose — e.g., quoted verses)
|
|
# ---------------------------------------------------------------------------
|
|
VERSE_PADA_CLASSES = frozenset(["p4", "p5", "p10", "p19", "p24"])
|
|
|
|
# Primary body paragraph classes
|
|
BODY_CLASSES = frozenset(["p6", "p28", "p23"])
|
|
|
|
# Chapter heading class (bold title like "1 /The Story of the Syamantaka Jewel")
|
|
HEADING_CLASS = "p27"
|
|
|
|
# Bookmark class
|
|
BOOKMARK_CLASS = "p13"
|
|
|
|
# Volume header class
|
|
VOLUME_HEADER_CLASS = "p12"
|
|
|
|
# Classes to skip entirely
|
|
SKIP_CLASSES = frozenset(
|
|
[
|
|
"p9", # empty / decorative breaks
|
|
"p11", # empty breaks
|
|
"p15", # empty breaks
|
|
"p17", # empty breaks
|
|
"p20", # empty breaks
|
|
"p14", # section labels
|
|
"p16", # "Not for sale in India"
|
|
"p25", # handwritten notes marker
|
|
"p26", # Apple Corps header
|
|
"p29", # image descriptions (color plate references)
|
|
"p30", # image descriptions
|
|
]
|
|
)
|
|
|
|
|
|
def is_empty(tag: Tag) -> bool:
|
|
text = tag.get_text().strip()
|
|
return not text or text == "\xa0"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Extraction
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def extract_vol2_chapters(soup: BeautifulSoup):
|
|
"""Extract all 35 chapters of Volume 2.
|
|
|
|
Returns a list of chapter dicts.
|
|
"""
|
|
all_paragraphs = list(soup.find_all("p"))
|
|
|
|
# Step 1: Find all Vol 2 chapter bookmark positions
|
|
# Format: "KB 1970-2-N / Title"
|
|
chapter_bookmarks = [] # (paragraph_index, chapter_num, title)
|
|
|
|
for i, p in enumerate(all_paragraphs):
|
|
cls = get_css_class(p)
|
|
if cls != BOOKMARK_CLASS:
|
|
continue
|
|
text = decode_text(p)
|
|
m = re.match(r"KB\s+1970-2-(\d+)\s*/\s*(.+)", text)
|
|
if m:
|
|
ch_num = int(m.group(1))
|
|
title = m.group(2).strip()
|
|
chapter_bookmarks.append((i, ch_num, title))
|
|
|
|
if not chapter_bookmarks:
|
|
print("ERROR: No Vol 2 chapter bookmarks found!")
|
|
return []
|
|
|
|
print(f" Found {len(chapter_bookmarks)} Vol 2 chapter bookmarks")
|
|
|
|
# Step 2: Find the end boundary — the next book section or end of file
|
|
# Look for the next p12 volume header that is NOT KB Vol 2
|
|
vol2_end = len(all_paragraphs)
|
|
# Start searching after the last Vol 2 chapter bookmark
|
|
last_bookmark_pos = chapter_bookmarks[-1][0]
|
|
for i in range(last_bookmark_pos + 1, len(all_paragraphs)):
|
|
p = all_paragraphs[i]
|
|
cls = get_css_class(p)
|
|
if cls == VOLUME_HEADER_CLASS:
|
|
text = decode_text(p)
|
|
# If it's a different book header (not KB), that's our end
|
|
if "Volume 2" not in text:
|
|
vol2_end = i
|
|
break
|
|
|
|
# Step 3: Extract each chapter
|
|
chapters = []
|
|
|
|
for idx, (bookmark_pos, ch_num, title) in enumerate(chapter_bookmarks):
|
|
# Chapter content starts after the bookmark
|
|
content_start = bookmark_pos + 1
|
|
|
|
# Chapter content ends at the next chapter's bookmark (or end boundary)
|
|
if idx + 1 < len(chapter_bookmarks):
|
|
content_end = chapter_bookmarks[idx + 1][0]
|
|
else:
|
|
content_end = vol2_end
|
|
|
|
# Extract paragraphs for this chapter
|
|
paragraphs = extract_chapter_paragraphs(
|
|
all_paragraphs, content_start, content_end
|
|
)
|
|
|
|
chapters.append(
|
|
{
|
|
"book": "kb",
|
|
"volume": 2,
|
|
"chapter": ch_num,
|
|
"title": title,
|
|
"type": "chapter",
|
|
"paragraphs": paragraphs,
|
|
}
|
|
)
|
|
|
|
return chapters
|
|
|
|
|
|
def extract_chapter_paragraphs(
|
|
all_paragraphs: list,
|
|
start_idx: int,
|
|
end_idx: int,
|
|
) -> list[str]:
|
|
"""Extract the prose paragraphs for a single chapter.
|
|
|
|
Handles:
|
|
- Regular body paragraphs (p6, p28)
|
|
- Quoted Sanskrit verse blocks (p4, p5, p19, p24) → grouped into centered italic
|
|
- Chapter heading (p27) → included as first paragraph
|
|
- Chapter end marker ("Thus ends...") → included as last paragraph (italic)
|
|
- Skips empty/decorative paragraphs
|
|
"""
|
|
result: list[str] = []
|
|
verse_accumulator: list[str] = []
|
|
|
|
def flush_verse():
|
|
if verse_accumulator:
|
|
joined = "<br>".join(verse_accumulator)
|
|
result.append(f'<div style="text-align:center"><i>{joined}</i></div>')
|
|
verse_accumulator.clear()
|
|
|
|
for i in range(start_idx, end_idx):
|
|
p = all_paragraphs[i]
|
|
cls = get_css_class(p)
|
|
|
|
# Skip empty paragraphs
|
|
if is_empty(p):
|
|
continue
|
|
|
|
# Skip noise classes
|
|
if cls in SKIP_CLASSES:
|
|
continue
|
|
|
|
# Skip bookmark tags that might be inside the range
|
|
if cls == BOOKMARK_CLASS:
|
|
continue
|
|
|
|
# Skip volume headers
|
|
if cls == VOLUME_HEADER_CLASS:
|
|
continue
|
|
|
|
# Sanskrit verse pādas — accumulate into centered block
|
|
# Use decode_html_content to preserve <br> tags within multi-line verse elements
|
|
if cls in VERSE_PADA_CLASSES:
|
|
html = decode_html_content(p)
|
|
if html:
|
|
verse_accumulator.append(html)
|
|
continue
|
|
|
|
# Non-verse paragraph — flush any accumulated verse first
|
|
flush_verse()
|
|
|
|
# Chapter heading (p27) — keep it, it's the chapter title
|
|
if cls == HEADING_CLASS:
|
|
html = decode_html_content(p)
|
|
if html:
|
|
result.append(html)
|
|
continue
|
|
|
|
# Body paragraphs and everything else
|
|
html = decode_html_content(p)
|
|
if html:
|
|
result.append(html)
|
|
|
|
# Flush any trailing verse
|
|
flush_verse()
|
|
|
|
return result
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Output
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def write_json(data, filepath):
|
|
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
|
with open(filepath, "w", encoding="utf-8") as f:
|
|
json.dump(data, f, ensure_ascii=False, indent=2)
|
|
|
|
|
|
def main():
|
|
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
html_path = os.path.join(base_dir, "public", "html", "other.html")
|
|
output_dir = os.path.join(
|
|
base_dir, "lib", "jsons", "books", "kb", "vol-2", "chapters"
|
|
)
|
|
|
|
if not os.path.exists(html_path):
|
|
print(f"ERROR: {html_path} not found!")
|
|
sys.exit(1)
|
|
|
|
print(f"Reading {html_path}...")
|
|
with open(html_path, "r", encoding="utf-8") as f:
|
|
soup = BeautifulSoup(f, "html.parser")
|
|
|
|
print("Extracting Volume 2 chapters...")
|
|
chapters = extract_vol2_chapters(soup)
|
|
|
|
if not chapters:
|
|
print("ERROR: No chapters extracted!")
|
|
sys.exit(1)
|
|
|
|
print(f"\nExtracted {len(chapters)} chapters\n")
|
|
|
|
# Write each chapter
|
|
for ch in chapters:
|
|
ch_num = str(ch["chapter"]).zfill(2)
|
|
filepath = os.path.join(output_dir, f"{ch_num}.json")
|
|
write_json(ch, filepath)
|
|
print(
|
|
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — {ch['title']}"
|
|
)
|
|
|
|
# Summary
|
|
total_paragraphs = sum(len(ch["paragraphs"]) for ch in chapters)
|
|
print(f"\n--- Summary ---")
|
|
print(f"Total chapters: {len(chapters)}")
|
|
print(f"Total paragraphs: {total_paragraphs}")
|
|
print(f"Average paragraphs per chapter: {total_paragraphs / len(chapters):.1f}")
|
|
print(f"Output: {output_dir}/")
|
|
|
|
# Show samples from first and last chapter
|
|
print(f"\n--- Sample: Chapter 1 ---")
|
|
ch1 = chapters[0]
|
|
for i, p in enumerate(ch1["paragraphs"][:3]):
|
|
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
|
print(f" [{i}] {preview}...")
|
|
|
|
print(f"\n--- Sample: Chapter {chapters[-1]['chapter']} ---")
|
|
last = chapters[-1]
|
|
for i, p in enumerate(last["paragraphs"][:3]):
|
|
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
|
print(f" [{i}] {preview}...")
|
|
|
|
# Quality check: verify all chapters have the end marker
|
|
print(f"\n--- Quality Check ---")
|
|
missing_end = 0
|
|
for ch in chapters:
|
|
has_end = any("Thus end" in p for p in ch["paragraphs"])
|
|
if not has_end:
|
|
missing_end += 1
|
|
print(
|
|
f" ⚠ Chapter {ch['chapter']} ({ch['title']}) — no 'Thus ends' marker"
|
|
)
|
|
if missing_end == 0:
|
|
print(f" ✅ All {len(chapters)} chapters have 'Thus ends' markers")
|
|
else:
|
|
print(f" ⚠ {missing_end} chapters missing 'Thus ends' markers")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|