#!/usr/bin/env python3
"""Extract Kṛṣṇa Book Volume 1 chapters from other.html.
Volume 1 has 54 chapters (KB 1970-1-1 through KB 1970-1-54).
Each chapter is a prose chapter (no verses) — just paragraphs of purport text.
The chapters are delimited by p13 bookmark tags: "KB 1970-1-N / Title"
Each chapter starts with a p27 heading: "N /Title"
Body paragraphs are in p6 (primary) and p28 classes.
Chapter end markers: "Thus ends the Bhaktivedanta purport of the Nth Chapter..."
Output schema (one JSON per chapter):
{
"book": "kb",
"volume": 1,
"chapter": 1,
"title": "Advent of Lord Kṛṣṇa",
"type": "chapter",
"paragraphs": ["...
", ...]
}
Output: lib/jsons/books/kb/vol-1/chapters/01.json through 54.json
Usage:
python3 scripts/kb_extract_vol1_chapters.py
"""
import json
import os
import re
import sys
from bs4 import BeautifulSoup, Tag
# ---------------------------------------------------------------------------
# ScaGoudy → IAST Unicode mapping
# ---------------------------------------------------------------------------
SCAGOUDY_TO_IAST = {
"ä": "ā",
"Ä": "Ā",
"å": "ṛ",
"Å": "Ṛ",
"ñ": "ṣ",
"Ñ": "Ṣ",
"ö": "ṭ",
"Ö": "Ṭ",
"ë": "ṇ",
"Ë": "Ṇ",
"ò": "ḍ",
"Ò": "Ḍ",
"ç": "ś",
"Ç": "Ś",
"ì": "ṅ",
"ï": "ñ",
"ü": "ū",
"Ü": "Ū",
"é": "ī",
"É": "Ī",
"à": "ṁ",
"ù": "ḥ",
"è": "ṝ",
"û": "ṁl",
"ÿ": "ḷ",
}
def decode_scagoudy(text: str) -> str:
for src, dst in SCAGOUDY_TO_IAST.items():
text = text.replace(src, dst)
return text
def decode_html_content(tag: Tag) -> str:
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
inner = "".join(str(child) for child in tag.children)
inner = decode_scagoudy(inner)
# Remove span tags with CSS classes (RTF conversion artifacts)
inner = re.sub(r']*class="[^"]*"[^>]*>(.*?)', r"\1", inner)
inner = re.sub(r"\s+", " ", inner).strip()
return inner
def decode_text(tag: Tag) -> str:
return decode_scagoudy(tag.get_text().strip())
def get_css_class(tag: Tag) -> str:
classes = tag.get("class", [])
return classes[0] if classes else ""
# ---------------------------------------------------------------------------
# CSS classes for Sanskrit verse pādas (center-aligned italic blocks
# that sometimes appear within chapter prose — e.g., quoted verses)
# ---------------------------------------------------------------------------
VERSE_PADA_CLASSES = frozenset(["p4", "p5", "p10", "p19", "p24"])
# Primary body paragraph classes
BODY_CLASSES = frozenset(["p6", "p28", "p23"])
# Chapter heading class (bold title like "1 /Advent of Lord Kṛṣṇa")
HEADING_CLASS = "p27"
# Bookmark class
BOOKMARK_CLASS = "p13"
# Volume header class
VOLUME_HEADER_CLASS = "p12"
# Classes to skip entirely
SKIP_CLASSES = frozenset(
[
"p9", # empty / decorative breaks
"p11", # empty breaks
"p15", # empty breaks
"p17", # empty breaks
"p20", # empty breaks
"p14", # section labels like "Flap:"
"p16", # "Not for sale in India"
"p25", # handwritten notes marker
"p26", # Apple Corps header
"p29", # image descriptions (color plate references)
"p30", # image descriptions
]
)
def is_empty(tag: Tag) -> bool:
text = tag.get_text().strip()
return not text or text == "\xa0"
# ---------------------------------------------------------------------------
# Extraction
# ---------------------------------------------------------------------------
def extract_vol1_chapters(soup: BeautifulSoup):
"""Extract all 54 chapters of Volume 1.
Returns a list of chapter dicts.
"""
all_paragraphs = list(soup.find_all("p"))
# Step 1: Find all Vol 1 chapter bookmark positions
# Format: "KB 1970-1-N / Title"
chapter_bookmarks = [] # (paragraph_index, chapter_num, title)
for i, p in enumerate(all_paragraphs):
cls = get_css_class(p)
if cls != BOOKMARK_CLASS:
continue
text = decode_text(p)
m = re.match(r"KB\s+1970-1-(\d+)\s*/\s*(.+)", text)
if m:
ch_num = int(m.group(1))
title = m.group(2).strip()
chapter_bookmarks.append((i, ch_num, title))
if not chapter_bookmarks:
print("ERROR: No Vol 1 chapter bookmarks found!")
return []
print(f" Found {len(chapter_bookmarks)} Vol 1 chapter bookmarks")
# Step 2: Find the end boundary (Volume 2 header or end of file)
vol2_start = len(all_paragraphs)
for i, p in enumerate(all_paragraphs):
cls = get_css_class(p)
if cls == VOLUME_HEADER_CLASS:
text = decode_text(p)
if "Volume 2" in text:
vol2_start = i
break
# Step 3: Extract each chapter
chapters = []
for idx, (bookmark_pos, ch_num, title) in enumerate(chapter_bookmarks):
# Chapter content starts after the bookmark
content_start = bookmark_pos + 1
# Chapter content ends at the next chapter's bookmark (or Vol 2 boundary)
if idx + 1 < len(chapter_bookmarks):
content_end = chapter_bookmarks[idx + 1][0]
else:
content_end = vol2_start
# Extract paragraphs for this chapter
paragraphs = extract_chapter_paragraphs(
all_paragraphs, content_start, content_end
)
chapters.append(
{
"book": "kb",
"volume": 1,
"chapter": ch_num,
"title": title,
"type": "chapter",
"paragraphs": paragraphs,
}
)
return chapters
def extract_chapter_paragraphs(
all_paragraphs: list,
start_idx: int,
end_idx: int,
) -> list[str]:
"""Extract the prose paragraphs for a single chapter.
Handles:
- Regular body paragraphs (p6, p28)
- Quoted Sanskrit verse blocks (p4, p5, p19, p24) → grouped into centered italic
- Chapter heading (p27) → included as first paragraph
- Chapter end marker ("Thus ends...") → included as last paragraph (italic)
- Skips empty/decorative paragraphs
"""
result: list[str] = []
verse_accumulator: list[str] = []
def flush_verse():
if verse_accumulator:
joined = "
".join(verse_accumulator)
result.append(f'{joined}
')
verse_accumulator.clear()
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
# Skip empty paragraphs
if is_empty(p):
continue
# Skip noise classes
if cls in SKIP_CLASSES:
continue
# Skip bookmark tags that might be inside the range
if cls == BOOKMARK_CLASS:
continue
# Skip volume headers
if cls == VOLUME_HEADER_CLASS:
continue
# Sanskrit verse pādas — accumulate into centered block
# Use decode_html_content to preserve
tags within multi-line verse elements
if cls in VERSE_PADA_CLASSES:
html = decode_html_content(p)
if html:
verse_accumulator.append(html)
continue
# Non-verse paragraph — flush any accumulated verse first
flush_verse()
# Chapter heading (p27) — keep it, it's the chapter title
if cls == HEADING_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Body paragraphs and everything else
html = decode_html_content(p)
if html:
result.append(html)
# Flush any trailing verse
flush_verse()
return result
# ---------------------------------------------------------------------------
# Output
# ---------------------------------------------------------------------------
def write_json(data, filepath):
os.makedirs(os.path.dirname(filepath), exist_ok=True)
with open(filepath, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False, indent=2)
def main():
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
html_path = os.path.join(base_dir, "public", "html", "other.html")
output_dir = os.path.join(
base_dir, "lib", "jsons", "books", "kb", "vol-1", "chapters"
)
if not os.path.exists(html_path):
print(f"ERROR: {html_path} not found!")
sys.exit(1)
print(f"Reading {html_path}...")
with open(html_path, "r", encoding="utf-8") as f:
soup = BeautifulSoup(f, "html.parser")
print("Extracting Volume 1 chapters...")
chapters = extract_vol1_chapters(soup)
if not chapters:
print("ERROR: No chapters extracted!")
sys.exit(1)
print(f"\nExtracted {len(chapters)} chapters\n")
# Write each chapter
for ch in chapters:
ch_num = str(ch["chapter"]).zfill(2)
filepath = os.path.join(output_dir, f"{ch_num}.json")
write_json(ch, filepath)
print(
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — {ch['title']}"
)
# Summary
total_paragraphs = sum(len(ch["paragraphs"]) for ch in chapters)
print(f"\n--- Summary ---")
print(f"Total chapters: {len(chapters)}")
print(f"Total paragraphs: {total_paragraphs}")
print(f"Average paragraphs per chapter: {total_paragraphs / len(chapters):.1f}")
print(f"Output: {output_dir}/")
# Show samples from first and last chapter
print(f"\n--- Sample: Chapter 1 ---")
ch1 = chapters[0]
for i, p in enumerate(ch1["paragraphs"][:3]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
print(f"\n--- Sample: Chapter {chapters[-1]['chapter']} ---")
last = chapters[-1]
for i, p in enumerate(last["paragraphs"][:3]):
preview = re.sub(r"<[^>]+>", "", p)[:100]
print(f" [{i}] {preview}...")
# Quality check: verify all chapters have the end marker
print(f"\n--- Quality Check ---")
missing_end = 0
for ch in chapters:
has_end = any("Thus end" in p for p in ch["paragraphs"])
if not has_end:
missing_end += 1
print(
f" ⚠ Chapter {ch['chapter']} ({ch['title']}) — no 'Thus ends' marker"
)
if missing_end == 0:
print(f" ✅ All {len(chapters)} chapters have 'Thus ends' markers")
else:
print(f" ⚠ {missing_end} chapters missing 'Thus ends' markers")
if __name__ == "__main__":
main()