containing "Bhagaväner kathä" End:
containing "Indian Bhagavatams" (stop before) Chapter boundaries:
tags containing "Bk:Chapter"
IMPORTANT: Unlike MOG/CBHC, this book's body text uses ScaGoudy font encoding.
Characters like ä, å, ñ, ö, ë, ò, ç, ì, ï, ü, é, à, ù must be decoded to
proper IAST Unicode (ā, ṛ, ṣ, ṭ, ṇ, ḍ, ś, ṅ, ñ, ū, ī, ṁ, ḥ).
CSS classes:
p56 — Book title header (skip)
p17 — Chapter bookmark headers ("Bk:Chapter N") — boundary detection, skip
p5 — Chapter title headings (skip — we derive titles ourselves)
p6 — Body paragraphs (MAIN content) — HAS ScaGoudy encoding
p12 — Body paragraph variant (longer text) — HAS ScaGoudy encoding
p38 — Body paragraph variant — HAS ScaGoudy encoding
p9 — Sanskrit verse lines (italic, with ) — centered italic
p10 — Reference markers like [RTW 1.1] — include as paragraphs
p61 — Reference marker variant [RTW 1.2] — include
p57 — Footnote header ("Footnote") — include
p58 — Footnote text — include
p19 — Sanskrit verse block (multi-line) — centered italic
p59 — Sanskrit verse block variant — centered italic
p23 — Sanskrit verse line variant — centered italic
p36 — Sanskrit verse line variant — centered italic
p60 — Sanskrit verse line variant — centered italic
Output:
lib/jsons/books/bk/intro.json — Book metadata
lib/jsons/books/bk/chapters/01.json — Chapter 1: This World of Misery
lib/jsons/books/bk/chapters/02.json — Chapter 2: The Cause of Misery
Usage:
python3 scripts/bk_extract.py
"""
import json
import os
import re
import sys
from bs4 import BeautifulSoup, Tag
# ---------------------------------------------------------------------------
# ScaGoudy → IAST Unicode mapping
# ---------------------------------------------------------------------------
SCAGOUDY_TO_IAST = {
"ä": "ā",
"Ä": "Ā",
"å": "ṛ",
"Å": "Ṛ",
"ñ": "ṣ",
"Ñ": "Ṣ",
"ö": "ṭ",
"Ö": "Ṭ",
"ë": "ṇ",
"Ë": "Ṇ",
"ò": "ḍ",
"Ò": "Ḍ",
"ç": "ś",
"Ç": "Ś",
"ì": "ṅ",
"Ì": "Ṅ",
"ï": "ñ",
"Ï": "Ñ",
"ü": "ū",
"Ü": "Ū",
"é": "ī",
"É": "Ī",
"à": "ṁ",
"À": "Ṁ",
"ù": "ḥ",
"Ù": "Ḥ",
"è": "ṝ",
"û": "ṁl",
"ÿ": "ḷ",
}
# Build a translation table for fast character-by-character replacement
_SCAGOUDY_TRANS = str.maketrans(
{k: v for k, v in SCAGOUDY_TO_IAST.items() if len(k) == 1}
)
def scagoudy_decode(text: str) -> str:
"""Decode ScaGoudy-encoded text to proper IAST Unicode."""
return text.translate(_SCAGOUDY_TRANS)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def decode_html_content(tag: Tag, apply_scagoudy: bool = True) -> str:
"""Extract inner HTML from a tag, preserve formatting tags, strip spans,
and optionally apply ScaGoudy decoding.
Steps:
1. Get inner HTML
2. Strip wrappers (keep content)
3. Apply ScaGoudy decode to the entire result (if requested)
4. Normalize whitespace
"""
inner = "".join(str(child) for child in tag.children)
# Remove span tags with CSS classes (RTF conversion artifacts) — keep content
inner = re.sub(
r' containing "Bhagaväner kathä"
End: containing "Indian Bhagavatams" (stop before)
"""
start_idx = None
end_idx = len(all_paragraphs)
for i, p in enumerate(all_paragraphs):
cls = get_css_class(p)
text = decode_text(p)
# Find start: p56 containing "Bhagaväner" or p2 TOC entry
if start_idx is None:
if cls == BOOK_TITLE_CLASS and "Bhag" in text and "kath" in text:
start_idx = i
continue
continue
# Find end: p35 containing "Indian Bhagavatams"
if cls == "p35" and "Indian" in text and "Bhagavatam" in text:
end_idx = i
break
if start_idx is None:
print("ERROR: Could not find 'Bhagaväner kathä' in others-before-1977.html!")
sys.exit(1)
return start_idx, end_idx
def find_chapter_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
"""Find all BK chapter bookmark tags (p17) in the BK range.
Returns list of (paragraph_index, raw_title_text).
"""
bookmarks = []
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
if cls == CHAPTER_BOOKMARK_CLASS:
text = decode_text(p).strip()
if text and "Bk:" in text:
bookmarks.append((i, text))
return bookmarks
# ---------------------------------------------------------------------------
# Paragraph extraction
# ---------------------------------------------------------------------------
def extract_paragraphs(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
"""Extract body paragraphs for a chapter section.
Collects:
- p6, p12, p38 (body paragraphs — ScaGoudy decoded)
- p9, p19, p59, p23, p36, p60 (verse lines — centered italic, ScaGoudy decoded)
- p10, p61 (reference markers — ScaGoudy decoded)
- p57, p58 (footnotes — ScaGoudy decoded)
Skips:
- p56 (book title), p17 (bookmarks), p5 (chapter titles), p2 (TOC), empties
"""
result = []
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
# Skip empty paragraphs
if is_empty(p):
continue
# Skip known skip classes
if cls in SKIP_CLASSES:
continue
# Skip book title header
if cls == BOOK_TITLE_CLASS:
continue
# Skip chapter bookmark headers
if cls == CHAPTER_BOOKMARK_CLASS:
continue
# Skip chapter title headings (p5)
if cls == CHAPTER_TITLE_CLASS:
continue
# Skip TOC entries (p2)
if cls == TOC_CLASS:
continue
# Body paragraphs (ScaGoudy decoded)
if cls in BODY_CLASSES:
html = decode_html_content(p, apply_scagoudy=True)
if html:
result.append(html)
continue
# Sanskrit verse lines (centered italic, ScaGoudy decoded)
if cls in VERSE_CLASSES:
html = decode_html_content_as_verse(p)
if html:
result.append(html)
continue
# Reference markers (ScaGoudy decoded)
if cls in REFERENCE_CLASSES:
html = decode_html_content(p, apply_scagoudy=True)
if html:
result.append(html)
continue
# Footnote classes (ScaGoudy decoded)
if cls in FOOTNOTE_CLASSES:
html = decode_html_content(p, apply_scagoudy=True)
if html:
result.append(html)
continue
# Fallback: try to extract content from unknown classes
html = decode_html_content(p, apply_scagoudy=True)
if html:
print(f" ⚠ Unknown class '{cls}' at paragraph {i}: {html[:80]}...")
result.append(html)
return result
# ---------------------------------------------------------------------------
# Main extraction logic
# ---------------------------------------------------------------------------
def extract_bk(soup: BeautifulSoup) -> dict:
"""Extract the entire Bhagavāner Kathā from the parsed HTML.
Returns a dict with:
"chapters": list of chapter dicts
"chapter_listing": list of {number, title} for intro.json
"""
all_paragraphs = list(soup.find_all("p"))
# Step 1: Find BK boundaries
bk_start, bk_end = find_bk_boundaries(all_paragraphs)
print(f" BK content: paragraphs {bk_start} to {bk_end}")
print(f" Total paragraphs in range: {bk_end - bk_start}")
# Step 2: Find all chapter bookmarks (p17 with "Bk:")
bookmarks = find_chapter_bookmarks(all_paragraphs, bk_start, bk_end)
print(f" Found {len(bookmarks)} BK chapter bookmarks:")
for pos, title in bookmarks:
print(f" Paragraph {pos}: {title}")
# Step 3: Map bookmarks to chapters with proper titles
# Expected: 2 chapters
chapter_titles_map = {
1: "This World of Misery",
2: "The Cause of Misery",
}
chapter_bookmarks = [] # (para_idx, chapter_num, normalized_title)
for idx, (pos, _raw_title) in enumerate(bookmarks):
ch_num = idx + 1
ch_title = chapter_titles_map.get(ch_num, f"Chapter {ch_num}")
chapter_bookmarks.append((pos, ch_num, ch_title))
print(f"\n Mapped {len(chapter_bookmarks)} chapters:")
for pos, num, title in chapter_bookmarks:
print(f" Chapter {num}: {title} (paragraph {pos})")
# Step 4: Extract chapters
chapters = []
for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_bookmarks):
# Content starts after the bookmark line (skip p17 and p5 heading)
content_start = ch_pos + 1
# Content ends at next chapter bookmark or bk_end
if idx + 1 < len(chapter_bookmarks):
content_end = chapter_bookmarks[idx + 1][0]
else:
content_end = bk_end
paragraphs = extract_paragraphs(all_paragraphs, content_start, content_end)
ch = {
"book": "bk",
"chapter": ch_num,
"title": ch_title,
"type": "chapter",
"paragraphs": paragraphs,
}
chapters.append(ch)
# Step 5: Build chapter listing for intro.json
chapter_listing = [
{"number": ch["chapter"], "title": ch["title"]} for ch in chapters
]
return {
"chapters": chapters,
"chapter_listing": chapter_listing,
}
# ---------------------------------------------------------------------------
# Output
# ---------------------------------------------------------------------------
def write_json(data, filepath):
"""Write data as formatted JSON to filepath, creating directories as needed."""
os.makedirs(os.path.dirname(filepath), exist_ok=True)
with open(filepath, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False, indent=2)
def main():
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html")
output_base = os.path.join(base_dir, "lib", "jsons", "books", "bk")
if not os.path.exists(html_path):
print(f"ERROR: {html_path} not found!")
sys.exit(1)
print(f"Reading {html_path}...")
with open(html_path, "r", encoding="utf-8") as f:
soup = BeautifulSoup(f, "html.parser")
print("Extracting Bhagavāner Kathā...\n")
result = extract_bk(soup)
chapters = result["chapters"]
chapter_listing = result["chapter_listing"]
# --- Write intro.json ---
intro_meta = {
"book": "bk",
"title": "Bhagavāner Kathā",
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
"year": None,
"description": (
"Early Bengali writings on the topics of Godhead "
"— part of 'Renunciation through Wisdom' (Vairāgya-vidyā)."
),
"totalChapters": 2,
"chapters": chapter_listing,
}
intro_path = os.path.join(output_base, "intro.json")
write_json(intro_meta, intro_path)
print(f"\n Written: {intro_path}")
# --- Write chapters ---
print(f"\n--- Chapters ---")
for ch in chapters:
ch_num_str = str(ch["chapter"]).zfill(2)
filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
write_json(ch, filepath)
print(
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
f"{ch['title']}"
)
# --- Summary ---
total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
print(f"\n{'=' * 60}")
print(f" EXTRACTION SUMMARY")
print(f"{'=' * 60}")
print(f" Chapters: {len(chapters)}")
for ch in chapters:
print(
f" Ch {ch['chapter']}: {len(ch['paragraphs'])} paragraphs — {ch['title']}"
)
print(f" Total paragraphs: {total_chapter_paras}")
if chapters:
print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
print(f" Output directory: {output_base}/")
# --- Samples ---
for ch in chapters:
ch_num = ch["chapter"]
print(f"\n--- Sample: Chapter {ch_num} ({ch['title']}) ---")
for i, p in enumerate(ch["paragraphs"][:3]):
preview = re.sub(r"<[^>]+>", "", p)[:120]
print(f" [{i}] {preview}...")
if len(ch["paragraphs"]) > 3:
print(f" ... ({len(ch['paragraphs']) - 3} more paragraphs)")
# --- Quality Checks ---
print(f"\n--- Quality Checks ---")
# Check all 2 chapters present
expected_chapters = set(range(1, 3))
actual_chapters = set(ch["chapter"] for ch in chapters)
missing = expected_chapters - actual_chapters
extra = actual_chapters - expected_chapters
if missing:
print(f" ⚠ Missing chapters: {sorted(missing)}")
if extra:
print(f" ⚠ Extra chapters: {sorted(extra)}")
if not missing and not extra:
print(f" ✅ All 2 chapters present")
# Check empty chapters
empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
if empty_chapters:
print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
else:
print(f" ✅ No empty chapters")
# Check ScaGoudy decoding worked — look for common ScaGoudy characters
scagoudy_chars = set("äåñöëòçìïüéàù")
all_text = ""
for ch in chapters:
all_text += " ".join(ch["paragraphs"])
remaining_scagoudy = [c for c in all_text if c in scagoudy_chars]
if remaining_scagoudy:
# Count occurrences
from collections import Counter
counts = Counter(remaining_scagoudy)
print(
f" ⚠ Remaining ScaGoudy characters found (may be false positives): {dict(counts)}"
)
else:
print(f" ✅ No ScaGoudy characters remaining — decoding looks clean")
# Check for IAST characters (positive signal)
iast_chars = set("āṛṣṭṇḍśṅñūīṁḥ")
iast_found = [c for c in all_text if c in iast_chars]
if iast_found:
from collections import Counter
counts = Counter(iast_found)
print(f" ✅ IAST Unicode characters present: {dict(counts)}")
else:
print(f" ⚠ No IAST Unicode characters found — decoding may have failed")
# Check chapter 1 has [RTW 1.1] reference
if chapters:
ch1_text = " ".join(chapters[0]["paragraphs"])
if "RTW" in ch1_text and "1.1" in ch1_text:
print(f" ✅ '[RTW 1.1]' reference found in Chapter 1")
else:
print(f" ⚠ '[RTW 1.1]' reference NOT found in Chapter 1")
# Check chapter 2 has [RTW 1.2] reference
if len(chapters) >= 2:
ch2_text = " ".join(chapters[1]["paragraphs"])
if "RTW" in ch2_text and "1.2" in ch2_text:
print(f" ✅ '[RTW 1.2]' reference found in Chapter 2")
else:
print(f" ⚠ '[RTW 1.2]' reference NOT found in Chapter 2")
# Check for content leak from next work (Indian Bhagavatams)
if "Indian Bhagavatam" in all_text:
print(f" ⚠ Content from next work (Indian Bhagavatams) may have leaked!")
else:
print(f" ✅ No content leak from next work detected")
# Check Kṛṣṇa appears (decoded from Kåñëa)
if "Kṛṣṇa" in all_text:
print(f" ✅ 'Kṛṣṇa' properly decoded (from 'Kåñëa')")
else:
print(f" ⚠ 'Kṛṣṇa' not found — ScaGoudy decoding may need review")
# Check last chapter ends cleanly
if chapters:
last_ch = chapters[-1]
last_para = last_ch["paragraphs"][-1] if last_ch["paragraphs"] else ""
last_preview = re.sub(r"<[^>]+>", "", last_para)[:100]
print(f' ℹ Last paragraph of Ch {last_ch["chapter"]}: "{last_preview}..."')
print(f"\nDone! ✨")
if __name__ == "__main__":
main()
and reference tags like [Bg. 7.14].
"""
inner = "".join(str(child) for child in tag.children)
# Remove span tags with CSS classes — keep content
inner = re.sub(
r'
tags)
# Replace multiple spaces with single space
inner = re.sub(r"[ ]+", " ", inner).strip()
# Remove trailing
tags
inner = re.sub(r"(