feat: kb both volumes
This commit is contained in:
gosangam
@@ -0,0 +1,362 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract Kṛṣṇa Book Volume 1 chapters from other.html.
|
||||
|
||||
Volume 1 has 54 chapters (KB 1970-1-1 through KB 1970-1-54).
|
||||
Each chapter is a prose chapter (no verses) — just paragraphs of purport text.
|
||||
|
||||
The chapters are delimited by p13 bookmark tags: "KB 1970-1-N / Title"
|
||||
Each chapter starts with a p27 heading: "N /Title"
|
||||
Body paragraphs are in p6 (primary) and p28 classes.
|
||||
Chapter end markers: "Thus ends the Bhaktivedanta purport of the Nth Chapter..."
|
||||
|
||||
Output schema (one JSON per chapter):
|
||||
{
|
||||
"book": "kb",
|
||||
"volume": 1,
|
||||
"chapter": 1,
|
||||
"title": "Advent of Lord Kṛṣṇa",
|
||||
"type": "chapter",
|
||||
"paragraphs": ["<p>...</p>", ...]
|
||||
}
|
||||
|
||||
Output: lib/jsons/books/kb/vol-1/chapters/01.json through 54.json
|
||||
|
||||
Usage:
|
||||
python3 scripts/kb_extract_vol1_chapters.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ScaGoudy → IAST Unicode mapping
|
||||
# ---------------------------------------------------------------------------
|
||||
SCAGOUDY_TO_IAST = {
|
||||
"ä": "ā",
|
||||
"Ä": "Ā",
|
||||
"å": "ṛ",
|
||||
"Å": "Ṛ",
|
||||
"ñ": "ṣ",
|
||||
"Ñ": "Ṣ",
|
||||
"ö": "ṭ",
|
||||
"Ö": "Ṭ",
|
||||
"ë": "ṇ",
|
||||
"Ë": "Ṇ",
|
||||
"ò": "ḍ",
|
||||
"Ò": "Ḍ",
|
||||
"ç": "ś",
|
||||
"Ç": "Ś",
|
||||
"ì": "ṅ",
|
||||
"ï": "ñ",
|
||||
"ü": "ū",
|
||||
"Ü": "Ū",
|
||||
"é": "ī",
|
||||
"É": "Ī",
|
||||
"à": "ṁ",
|
||||
"ù": "ḥ",
|
||||
"è": "ṝ",
|
||||
"û": "ṁl",
|
||||
"ÿ": "ḷ",
|
||||
}
|
||||
|
||||
|
||||
def decode_scagoudy(text: str) -> str:
|
||||
for src, dst in SCAGOUDY_TO_IAST.items():
|
||||
text = text.replace(src, dst)
|
||||
return text
|
||||
|
||||
|
||||
def decode_html_content(tag: Tag) -> str:
|
||||
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
|
||||
inner = "".join(str(child) for child in tag.children)
|
||||
inner = decode_scagoudy(inner)
|
||||
# Remove span tags with CSS classes (RTF conversion artifacts)
|
||||
inner = re.sub(r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner)
|
||||
inner = re.sub(r"\s+", " ", inner).strip()
|
||||
return inner
|
||||
|
||||
|
||||
def decode_text(tag: Tag) -> str:
|
||||
return decode_scagoudy(tag.get_text().strip())
|
||||
|
||||
|
||||
def get_css_class(tag: Tag) -> str:
|
||||
classes = tag.get("class", [])
|
||||
return classes[0] if classes else ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CSS classes for Sanskrit verse pādas (center-aligned italic blocks
|
||||
# that sometimes appear within chapter prose — e.g., quoted verses)
|
||||
# ---------------------------------------------------------------------------
|
||||
VERSE_PADA_CLASSES = frozenset(["p4", "p5", "p10", "p19", "p24"])
|
||||
|
||||
# Primary body paragraph classes
|
||||
BODY_CLASSES = frozenset(["p6", "p28", "p23"])
|
||||
|
||||
# Chapter heading class (bold title like "1 /Advent of Lord Kṛṣṇa")
|
||||
HEADING_CLASS = "p27"
|
||||
|
||||
# Bookmark class
|
||||
BOOKMARK_CLASS = "p13"
|
||||
|
||||
# Volume header class
|
||||
VOLUME_HEADER_CLASS = "p12"
|
||||
|
||||
# Classes to skip entirely
|
||||
SKIP_CLASSES = frozenset(
|
||||
[
|
||||
"p9", # empty / decorative breaks
|
||||
"p11", # empty breaks
|
||||
"p15", # empty breaks
|
||||
"p17", # empty breaks
|
||||
"p20", # empty breaks
|
||||
"p14", # section labels like "Flap:"
|
||||
"p16", # "Not for sale in India"
|
||||
"p25", # handwritten notes marker
|
||||
"p26", # Apple Corps header
|
||||
"p29", # image descriptions (color plate references)
|
||||
"p30", # image descriptions
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def is_empty(tag: Tag) -> bool:
|
||||
text = tag.get_text().strip()
|
||||
return not text or text == "\xa0"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def extract_vol1_chapters(soup: BeautifulSoup):
|
||||
"""Extract all 54 chapters of Volume 1.
|
||||
|
||||
Returns a list of chapter dicts.
|
||||
"""
|
||||
all_paragraphs = list(soup.find_all("p"))
|
||||
|
||||
# Step 1: Find all Vol 1 chapter bookmark positions
|
||||
# Format: "KB 1970-1-N / Title"
|
||||
chapter_bookmarks = [] # (paragraph_index, chapter_num, title)
|
||||
|
||||
for i, p in enumerate(all_paragraphs):
|
||||
cls = get_css_class(p)
|
||||
if cls != BOOKMARK_CLASS:
|
||||
continue
|
||||
text = decode_text(p)
|
||||
m = re.match(r"KB\s+1970-1-(\d+)\s*/\s*(.+)", text)
|
||||
if m:
|
||||
ch_num = int(m.group(1))
|
||||
title = m.group(2).strip()
|
||||
chapter_bookmarks.append((i, ch_num, title))
|
||||
|
||||
if not chapter_bookmarks:
|
||||
print("ERROR: No Vol 1 chapter bookmarks found!")
|
||||
return []
|
||||
|
||||
print(f" Found {len(chapter_bookmarks)} Vol 1 chapter bookmarks")
|
||||
|
||||
# Step 2: Find the end boundary (Volume 2 header or end of file)
|
||||
vol2_start = len(all_paragraphs)
|
||||
for i, p in enumerate(all_paragraphs):
|
||||
cls = get_css_class(p)
|
||||
if cls == VOLUME_HEADER_CLASS:
|
||||
text = decode_text(p)
|
||||
if "Volume 2" in text:
|
||||
vol2_start = i
|
||||
break
|
||||
|
||||
# Step 3: Extract each chapter
|
||||
chapters = []
|
||||
|
||||
for idx, (bookmark_pos, ch_num, title) in enumerate(chapter_bookmarks):
|
||||
# Chapter content starts after the bookmark
|
||||
content_start = bookmark_pos + 1
|
||||
|
||||
# Chapter content ends at the next chapter's bookmark (or Vol 2 boundary)
|
||||
if idx + 1 < len(chapter_bookmarks):
|
||||
content_end = chapter_bookmarks[idx + 1][0]
|
||||
else:
|
||||
content_end = vol2_start
|
||||
|
||||
# Extract paragraphs for this chapter
|
||||
paragraphs = extract_chapter_paragraphs(
|
||||
all_paragraphs, content_start, content_end
|
||||
)
|
||||
|
||||
chapters.append(
|
||||
{
|
||||
"book": "kb",
|
||||
"volume": 1,
|
||||
"chapter": ch_num,
|
||||
"title": title,
|
||||
"type": "chapter",
|
||||
"paragraphs": paragraphs,
|
||||
}
|
||||
)
|
||||
|
||||
return chapters
|
||||
|
||||
|
||||
def extract_chapter_paragraphs(
|
||||
all_paragraphs: list,
|
||||
start_idx: int,
|
||||
end_idx: int,
|
||||
) -> list[str]:
|
||||
"""Extract the prose paragraphs for a single chapter.
|
||||
|
||||
Handles:
|
||||
- Regular body paragraphs (p6, p28)
|
||||
- Quoted Sanskrit verse blocks (p4, p5, p19, p24) → grouped into centered italic
|
||||
- Chapter heading (p27) → included as first paragraph
|
||||
- Chapter end marker ("Thus ends...") → included as last paragraph (italic)
|
||||
- Skips empty/decorative paragraphs
|
||||
"""
|
||||
result: list[str] = []
|
||||
verse_accumulator: list[str] = []
|
||||
|
||||
def flush_verse():
|
||||
if verse_accumulator:
|
||||
joined = "<br>".join(verse_accumulator)
|
||||
result.append(f'<div style="text-align:center"><i>{joined}</i></div>')
|
||||
verse_accumulator.clear()
|
||||
|
||||
for i in range(start_idx, end_idx):
|
||||
p = all_paragraphs[i]
|
||||
cls = get_css_class(p)
|
||||
|
||||
# Skip empty paragraphs
|
||||
if is_empty(p):
|
||||
continue
|
||||
|
||||
# Skip noise classes
|
||||
if cls in SKIP_CLASSES:
|
||||
continue
|
||||
|
||||
# Skip bookmark tags that might be inside the range
|
||||
if cls == BOOKMARK_CLASS:
|
||||
continue
|
||||
|
||||
# Skip volume headers
|
||||
if cls == VOLUME_HEADER_CLASS:
|
||||
continue
|
||||
|
||||
# Sanskrit verse pādas — accumulate into centered block
|
||||
# Use decode_html_content to preserve <br> tags within multi-line verse elements
|
||||
if cls in VERSE_PADA_CLASSES:
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
verse_accumulator.append(html)
|
||||
continue
|
||||
|
||||
# Non-verse paragraph — flush any accumulated verse first
|
||||
flush_verse()
|
||||
|
||||
# Chapter heading (p27) — keep it, it's the chapter title
|
||||
if cls == HEADING_CLASS:
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
result.append(html)
|
||||
continue
|
||||
|
||||
# Body paragraphs and everything else
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
result.append(html)
|
||||
|
||||
# Flush any trailing verse
|
||||
flush_verse()
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Output
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def write_json(data, filepath):
|
||||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||||
with open(filepath, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "other.html")
|
||||
output_dir = os.path.join(
|
||||
base_dir, "lib", "jsons", "books", "kb", "vol-1", "chapters"
|
||||
)
|
||||
|
||||
if not os.path.exists(html_path):
|
||||
print(f"ERROR: {html_path} not found!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Reading {html_path}...")
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
soup = BeautifulSoup(f, "html.parser")
|
||||
|
||||
print("Extracting Volume 1 chapters...")
|
||||
chapters = extract_vol1_chapters(soup)
|
||||
|
||||
if not chapters:
|
||||
print("ERROR: No chapters extracted!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"\nExtracted {len(chapters)} chapters\n")
|
||||
|
||||
# Write each chapter
|
||||
for ch in chapters:
|
||||
ch_num = str(ch["chapter"]).zfill(2)
|
||||
filepath = os.path.join(output_dir, f"{ch_num}.json")
|
||||
write_json(ch, filepath)
|
||||
print(
|
||||
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — {ch['title']}"
|
||||
)
|
||||
|
||||
# Summary
|
||||
total_paragraphs = sum(len(ch["paragraphs"]) for ch in chapters)
|
||||
print(f"\n--- Summary ---")
|
||||
print(f"Total chapters: {len(chapters)}")
|
||||
print(f"Total paragraphs: {total_paragraphs}")
|
||||
print(f"Average paragraphs per chapter: {total_paragraphs / len(chapters):.1f}")
|
||||
print(f"Output: {output_dir}/")
|
||||
|
||||
# Show samples from first and last chapter
|
||||
print(f"\n--- Sample: Chapter 1 ---")
|
||||
ch1 = chapters[0]
|
||||
for i, p in enumerate(ch1["paragraphs"][:3]):
|
||||
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
||||
print(f" [{i}] {preview}...")
|
||||
|
||||
print(f"\n--- Sample: Chapter {chapters[-1]['chapter']} ---")
|
||||
last = chapters[-1]
|
||||
for i, p in enumerate(last["paragraphs"][:3]):
|
||||
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
||||
print(f" [{i}] {preview}...")
|
||||
|
||||
# Quality check: verify all chapters have the end marker
|
||||
print(f"\n--- Quality Check ---")
|
||||
missing_end = 0
|
||||
for ch in chapters:
|
||||
has_end = any("Thus end" in p for p in ch["paragraphs"])
|
||||
if not has_end:
|
||||
missing_end += 1
|
||||
print(
|
||||
f" ⚠ Chapter {ch['chapter']} ({ch['title']}) — no 'Thus ends' marker"
|
||||
)
|
||||
if missing_end == 0:
|
||||
print(f" ✅ All {len(chapters)} chapters have 'Thus ends' markers")
|
||||
else:
|
||||
print(f" ⚠ {missing_end} chapters missing 'Thus ends' markers")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user