933 lines
30 KiB
Python
933 lines
30 KiB
Python
#!/usr/bin/env python3
|
||
"""Extract The Kṛṣṇa Consciousness Handbook (1971) from other.html.
|
||
|
||
The book has 2 intro segments (front-pages, introduction) and 7 chapters.
|
||
|
||
Main chapter bookmarks (p13 class):
|
||
- HKH: Introduction
|
||
- HKH 1: A Brief History of ISKCON
|
||
- HKH 2: Teachings of ISKCON
|
||
- HKH 3: Brief Biography of the Spiritual Master
|
||
- HKH 4: Paramparā
|
||
- HKH 5: Ministers for Kṛṣṇa Consciousness: ...
|
||
- HKH 6: Transcendental Aspects
|
||
- HKH 7: The Qualifications and Curriculum of the Divinity Student
|
||
|
||
Sub-section bookmarks (p49 class):
|
||
- HKH 6a–6f within chapter 6
|
||
- HKH 7a–7l within chapter 7
|
||
- HKH 7m is the END boundary (book catalog, not handbook content)
|
||
|
||
Output:
|
||
lib/jsons/books/hkh/intro.json — Book metadata
|
||
lib/jsons/books/hkh/intro/front-pages.json
|
||
lib/jsons/books/hkh/intro/introduction.json
|
||
lib/jsons/books/hkh/chapters/01.json through 07.json
|
||
|
||
Usage:
|
||
python3 scripts/hkh_extract.py
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import re
|
||
import sys
|
||
|
||
from bs4 import BeautifulSoup, Tag
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# ScaGoudy → IAST Unicode mapping
|
||
# ---------------------------------------------------------------------------
|
||
SCAGOUDY_TO_IAST = {
|
||
"ä": "ā",
|
||
"Ä": "Ā",
|
||
"å": "ṛ",
|
||
"Å": "Ṛ",
|
||
"ñ": "ṣ",
|
||
"Ñ": "Ṣ",
|
||
"ö": "ṭ",
|
||
"Ö": "Ṭ",
|
||
"ë": "ṇ",
|
||
"Ë": "Ṇ",
|
||
"ò": "ḍ",
|
||
"Ò": "Ḍ",
|
||
"ç": "ś",
|
||
"Ç": "Ś",
|
||
"ì": "ṅ",
|
||
"ï": "ñ",
|
||
"ü": "ū",
|
||
"Ü": "Ū",
|
||
"é": "ī",
|
||
"É": "Ī",
|
||
"à": "ṁ",
|
||
"ù": "ḥ",
|
||
"è": "ṝ",
|
||
"û": "ṁl",
|
||
"ÿ": "ḷ",
|
||
}
|
||
|
||
|
||
def decode_scagoudy(text: str) -> str:
|
||
"""Replace ScaGoudy-encoded characters with proper IAST Unicode."""
|
||
for src, dst in SCAGOUDY_TO_IAST.items():
|
||
text = text.replace(src, dst)
|
||
return text
|
||
|
||
|
||
def decode_html_content(tag: Tag) -> str:
|
||
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
|
||
inner = "".join(str(child) for child in tag.children)
|
||
inner = decode_scagoudy(inner)
|
||
# Remove span tags with CSS classes (RTF conversion artifacts)
|
||
inner = re.sub(
|
||
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
|
||
)
|
||
# Clean up Apple-tab-span leftovers (tab characters)
|
||
inner = inner.replace("\t", " ")
|
||
# Normalize whitespace (but preserve <br> tags)
|
||
inner = re.sub(r"\s+", " ", inner).strip()
|
||
return inner
|
||
|
||
|
||
def decode_text(tag: Tag) -> str:
|
||
"""Get plain text from a tag with ScaGoudy decoding."""
|
||
return decode_scagoudy(tag.get_text().strip())
|
||
|
||
|
||
def get_css_class(tag: Tag) -> str:
|
||
"""Get the first CSS class of a tag, or empty string."""
|
||
classes = tag.get("class", [])
|
||
return classes[0] if classes else ""
|
||
|
||
|
||
def is_empty(tag: Tag) -> bool:
|
||
"""Check if a tag is empty or whitespace-only."""
|
||
text = tag.get_text().strip()
|
||
return not text or text == "\xa0"
|
||
|
||
|
||
def is_dash_line(text: str) -> bool:
|
||
"""Return True if text is only dashes, spaces, and/or whitespace."""
|
||
cleaned = text.strip()
|
||
if not cleaned:
|
||
return False
|
||
return all(c in "-—–\u2014\u2013 \t" for c in cleaned)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# CSS class categorization for HKH
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# Book title header class
|
||
BOOK_HEADER_CLASS = "p12"
|
||
|
||
# Chapter bookmark class (main chapters)
|
||
CHAPTER_BOOKMARK_CLASS = "p13"
|
||
|
||
# Sub-section bookmark class
|
||
SUBSECTION_BOOKMARK_CLASS = "p49"
|
||
|
||
# Sanskrit verse pāda classes (italic, centered)
|
||
VERSE_PADA_CLASSES = frozenset(["p53"])
|
||
|
||
# Classes to skip entirely
|
||
SKIP_CLASSES = frozenset(["p9", "p70"])
|
||
|
||
# Body paragraph classes — be permissive, handle everything not otherwise special
|
||
BODY_CLASSES = frozenset(
|
||
[
|
||
"p6",
|
||
"p67",
|
||
"p104",
|
||
"p101",
|
||
"p46",
|
||
"p48",
|
||
"p103",
|
||
"p105",
|
||
"p36",
|
||
"p37",
|
||
"p24",
|
||
"p44",
|
||
"p28",
|
||
"p109",
|
||
"p110",
|
||
"p111",
|
||
"p112",
|
||
"p113",
|
||
"p114",
|
||
"p115",
|
||
"p95",
|
||
"p82",
|
||
"p25",
|
||
"p118",
|
||
]
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Extraction helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def find_hkh_boundaries(all_paragraphs: list) -> tuple:
|
||
"""Find the start and end paragraph indices of the HKH content.
|
||
|
||
Start: <p class="p12"> containing "Krsna Consciousness Handbook" and "1971"
|
||
End: <p class="p49"> containing "HKH 7m:" (book catalog section)
|
||
"""
|
||
start_idx = None
|
||
end_idx = len(all_paragraphs)
|
||
|
||
for i, p in enumerate(all_paragraphs):
|
||
cls = get_css_class(p)
|
||
text = p.get_text().strip()
|
||
|
||
# Find start
|
||
if (
|
||
cls == BOOK_HEADER_CLASS
|
||
and "Krsna Consciousness Handbook" in text
|
||
and "1971" in text
|
||
):
|
||
start_idx = i
|
||
|
||
# Find end: p49 containing "HKH 7m:"
|
||
if start_idx is not None and cls == SUBSECTION_BOOKMARK_CLASS:
|
||
if re.search(r"HKH\s+7m:", text):
|
||
end_idx = i
|
||
break
|
||
|
||
if start_idx is None:
|
||
print(
|
||
"ERROR: Could not find 'The Krsna Consciousness Handbook (1971)' header in other.html!"
|
||
)
|
||
sys.exit(1)
|
||
|
||
return start_idx, end_idx
|
||
|
||
|
||
def find_chapter_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
|
||
"""Find all HKH chapter/intro bookmark tags (p13) in the HKH range.
|
||
|
||
Returns list of (paragraph_index, chapter_number_or_None, chapter_title, bookmark_type).
|
||
bookmark_type is "intro" for "HKH: Introduction" and "chapter" for numbered chapters.
|
||
"""
|
||
bookmarks = []
|
||
for i in range(start_idx, end_idx):
|
||
p = all_paragraphs[i]
|
||
cls = get_css_class(p)
|
||
if cls != CHAPTER_BOOKMARK_CLASS:
|
||
continue
|
||
text = decode_text(p)
|
||
|
||
# Check for introduction bookmark
|
||
if re.match(r"HKH:\s+Introduction", text.strip()):
|
||
bookmarks.append((i, None, "Introduction", "intro"))
|
||
continue
|
||
|
||
# Check for numbered chapter bookmark (handle multi-line titles)
|
||
# Collapse newlines for matching
|
||
collapsed = re.sub(r"\s+", " ", text.strip())
|
||
m = re.match(r"HKH\s+(\d+):\s+(.+)", collapsed)
|
||
if m:
|
||
ch_num = int(m.group(1))
|
||
ch_title = m.group(2).strip()
|
||
bookmarks.append((i, ch_num, ch_title, "chapter"))
|
||
|
||
return bookmarks
|
||
|
||
|
||
def find_subsection_bookmarks(
|
||
all_paragraphs: list, start_idx: int, end_idx: int
|
||
) -> list:
|
||
"""Find all HKH sub-section bookmark tags (p49) in a range.
|
||
|
||
Returns list of (paragraph_index, subsection_code, subsection_title).
|
||
e.g. (12149, "6a", "Śikhā")
|
||
"""
|
||
bookmarks = []
|
||
for i in range(start_idx, end_idx):
|
||
p = all_paragraphs[i]
|
||
cls = get_css_class(p)
|
||
if cls != SUBSECTION_BOOKMARK_CLASS:
|
||
continue
|
||
text = decode_text(p)
|
||
|
||
# Match HKH Nx: Title patterns
|
||
collapsed = re.sub(r"\s+", " ", text.strip())
|
||
m = re.match(r"HKH\s+(\d+[a-z]):\s*(.*)", collapsed)
|
||
if m:
|
||
code = m.group(1)
|
||
title = m.group(2).strip()
|
||
# Clean up title — remove surrounding <i> artifacts and leading/trailing whitespace
|
||
title = re.sub(r"</?i>", "", title).strip()
|
||
bookmarks.append((i, code, title))
|
||
return bookmarks
|
||
|
||
|
||
def extract_paragraphs(
|
||
all_paragraphs: list,
|
||
start_idx: int,
|
||
end_idx: int,
|
||
is_front_pages: bool = False,
|
||
subsection_bookmarks: list = None,
|
||
) -> list:
|
||
"""Extract paragraphs for a section (chapter or intro segment).
|
||
|
||
Handles all CSS classes within the HKH content.
|
||
Groups consecutive p53 verse pādas into centered italic divs.
|
||
Sub-section bookmarks (p49) are converted to bold heading paragraphs.
|
||
"""
|
||
result = []
|
||
verse_accumulator = []
|
||
|
||
if subsection_bookmarks is None:
|
||
subsection_bookmarks = []
|
||
|
||
# Build a set of subsection bookmark indices for quick lookup
|
||
subsection_indices = {b[0] for b in subsection_bookmarks}
|
||
|
||
def flush_verse():
|
||
if verse_accumulator:
|
||
joined = "<br>".join(verse_accumulator)
|
||
result.append(f'<div style="text-align:center"><i>{joined}</i></div>')
|
||
verse_accumulator.clear()
|
||
|
||
for i in range(start_idx, end_idx):
|
||
p = all_paragraphs[i]
|
||
cls = get_css_class(p)
|
||
|
||
# Skip empty paragraphs
|
||
if is_empty(p):
|
||
continue
|
||
|
||
# Skip noise classes
|
||
if cls in SKIP_CLASSES:
|
||
continue
|
||
|
||
# Skip book title header (p12)
|
||
if cls == BOOK_HEADER_CLASS:
|
||
continue
|
||
|
||
# Skip chapter bookmark tags (p13) — boundaries only
|
||
if cls == CHAPTER_BOOKMARK_CLASS:
|
||
continue
|
||
|
||
# Handle sub-section bookmarks (p49) — convert to bold headings
|
||
if cls == SUBSECTION_BOOKMARK_CLASS:
|
||
flush_verse()
|
||
if i in subsection_indices:
|
||
# Find the matching subsection bookmark
|
||
for bm in subsection_bookmarks:
|
||
if bm[0] == i:
|
||
sub_title = bm[2]
|
||
if sub_title:
|
||
result.append(f"<b>{sub_title}</b>")
|
||
break
|
||
# Skip p49 tags that aren't in subsection_bookmarks (shouldn't happen in range)
|
||
continue
|
||
|
||
# Check for dash lines — skip them
|
||
text = decode_text(p)
|
||
if is_dash_line(text):
|
||
continue
|
||
|
||
# Sanskrit verse pādas (p53) — accumulate into centered block
|
||
if cls in VERSE_PADA_CLASSES:
|
||
html = decode_html_content(p)
|
||
if html:
|
||
# Remove inner <i> tags before re-wrapping
|
||
stripped = re.sub(r"</?i>", "", html)
|
||
verse_accumulator.append(stripped)
|
||
continue
|
||
|
||
# Non-verse paragraph — flush any accumulated verse first
|
||
flush_verse()
|
||
|
||
# Section headings (p31, p27) — include as paragraphs
|
||
if cls in ("p31", "p27"):
|
||
html = decode_html_content(p)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Misc text (p19, p21) — include
|
||
if cls in ("p19", "p21"):
|
||
html = decode_html_content(p)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Contents heading (p69) — include
|
||
if cls == "p69":
|
||
html = decode_html_content(p)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Contact info (p102) — include in front pages
|
||
if cls == "p102":
|
||
html = decode_html_content(p)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Body paragraphs and everything else — be permissive
|
||
html = decode_html_content(p)
|
||
if html and not is_dash_line(text):
|
||
# Detect Sanskrit verses disguised as body paragraphs:
|
||
# they start with <i> and contain <br> (multi-line italic = verse)
|
||
stripped_html = html.strip()
|
||
if stripped_html.startswith("<i>") and "<br" in stripped_html:
|
||
# This is a verse — wrap in centered italic div
|
||
inner = re.sub(r"</?i>", "", stripped_html)
|
||
result.append(f'<div style="text-align:center"><i>{inner}</i></div>')
|
||
else:
|
||
result.append(html)
|
||
|
||
# Flush any trailing verse
|
||
flush_verse()
|
||
|
||
return result
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Clean chapter titles
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# Map of chapter number to clean Unicode title
|
||
CLEAN_TITLES = {
|
||
1: "A Brief History of ISKCON",
|
||
2: "Teachings of ISKCON",
|
||
3: "Brief Biography of the Spiritual Master",
|
||
4: "Paramparā",
|
||
5: "Ministers for Kṛṣṇa Consciousness",
|
||
6: "Transcendental Aspects",
|
||
7: "The Qualifications and Curriculum of the Divinity Student",
|
||
}
|
||
|
||
|
||
def clean_chapter_title(ch_num: int, raw_title: str) -> str:
|
||
"""Return a clean Unicode title for the chapter."""
|
||
if ch_num in CLEAN_TITLES:
|
||
return CLEAN_TITLES[ch_num]
|
||
# Fallback: decode ScaGoudy from raw title
|
||
return decode_scagoudy(raw_title)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Main extraction logic
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def extract_hkh(soup: BeautifulSoup) -> dict:
|
||
"""Extract the entire Kṛṣṇa Consciousness Handbook (1971) from the parsed HTML.
|
||
|
||
Chapter 6 ("Transcendental Aspects") sub-sections (Śikhā, Beads, Ārātrika,
|
||
Garlands, Deities, Diet) are promoted to their own chapters (6-11), and the
|
||
old chapter 7 becomes chapter 12. This gives 12 total chapters.
|
||
|
||
Returns a dict with:
|
||
"intro_segments": list of intro segment dicts
|
||
"chapters": list of chapter dicts
|
||
"chapter_listing": list of {number, title} for intro.json
|
||
"""
|
||
all_paragraphs = list(soup.find_all("p"))
|
||
|
||
# Step 1: Find HKH boundaries
|
||
hkh_start, hkh_end = find_hkh_boundaries(all_paragraphs)
|
||
print(f" HKH content: paragraphs {hkh_start} to {hkh_end}")
|
||
print(f" Total paragraphs in range: {hkh_end - hkh_start}")
|
||
|
||
# Step 2: Find all chapter bookmarks
|
||
chapter_bookmarks = find_chapter_bookmarks(all_paragraphs, hkh_start, hkh_end)
|
||
print(f" Found {len(chapter_bookmarks)} HKH chapter bookmarks")
|
||
for pos, ch_num, ch_title, bm_type in chapter_bookmarks:
|
||
if bm_type == "intro":
|
||
print(f" [introduction] paragraph {pos}: {ch_title}")
|
||
else:
|
||
print(f" [chapter {ch_num}] paragraph {pos}: {ch_title}")
|
||
|
||
# Step 3: Find all sub-section bookmarks in the full range
|
||
all_subsections = find_subsection_bookmarks(all_paragraphs, hkh_start, hkh_end)
|
||
print(f" Found {len(all_subsections)} sub-section bookmarks")
|
||
|
||
# Step 4: Separate intro bookmark from chapter bookmarks
|
||
intro_bookmark = None
|
||
numbered_bookmarks = []
|
||
for bm in chapter_bookmarks:
|
||
if bm[3] == "intro":
|
||
intro_bookmark = bm
|
||
else:
|
||
numbered_bookmarks.append(bm)
|
||
|
||
# Step 5: Extract front pages
|
||
intro_segments = []
|
||
|
||
if intro_bookmark:
|
||
front_pages_end = intro_bookmark[0]
|
||
elif numbered_bookmarks:
|
||
front_pages_end = numbered_bookmarks[0][0]
|
||
else:
|
||
front_pages_end = hkh_end
|
||
|
||
front_pages_paragraphs = extract_paragraphs(
|
||
all_paragraphs, hkh_start, front_pages_end, is_front_pages=True
|
||
)
|
||
intro_segments.append(
|
||
{
|
||
"slug": "front-pages",
|
||
"title": "Front Pages",
|
||
"type": "intro-segment",
|
||
"book": "hkh",
|
||
"order": 1,
|
||
"paragraphs": front_pages_paragraphs,
|
||
}
|
||
)
|
||
|
||
# Step 6: Extract introduction segment
|
||
if intro_bookmark:
|
||
intro_start = intro_bookmark[0] + 1
|
||
if numbered_bookmarks:
|
||
intro_end = numbered_bookmarks[0][0]
|
||
else:
|
||
intro_end = hkh_end
|
||
|
||
intro_paragraphs = extract_paragraphs(all_paragraphs, intro_start, intro_end)
|
||
intro_segments.append(
|
||
{
|
||
"slug": "introduction",
|
||
"title": "Introduction",
|
||
"type": "intro-segment",
|
||
"book": "hkh",
|
||
"order": 2,
|
||
"paragraphs": intro_paragraphs,
|
||
}
|
||
)
|
||
|
||
# Step 7: Build the effective chapter list.
|
||
#
|
||
# Chapters 1-5 stay as-is from p13 bookmarks.
|
||
# Chapter 6 ("Transcendental Aspects") is SPLIT: its p49 sub-sections
|
||
# (6a=Śikhā, 6b=Beads, 6c=Ārātrika, 6d=Garlands, 6e=Deities, 6f=Diet)
|
||
# each become their own chapter (renumbered 6-11).
|
||
# Old chapter 7 is also SPLIT: 7a+7b stay together as "The Qualifications
|
||
# and Curriculum of the Divinity Student" (chapter 12), and 7c-7l each
|
||
# become their own chapters (13-21). Total: 21 chapters.
|
||
|
||
# Identify the chapter-6 and chapter-7 bookmarks
|
||
ch6_bm = None
|
||
ch7_bm = None
|
||
for bm in numbered_bookmarks:
|
||
if bm[1] == 6:
|
||
ch6_bm = bm
|
||
elif bm[1] == 7:
|
||
ch7_bm = bm
|
||
|
||
# Find chapter 6's sub-section bookmarks (6a through 6f)
|
||
ch6_subsections = []
|
||
if ch6_bm and ch7_bm:
|
||
ch6_range_start = ch6_bm[0]
|
||
ch6_range_end = ch7_bm[0]
|
||
ch6_subsections = [
|
||
s for s in all_subsections if ch6_range_start <= s[0] < ch6_range_end
|
||
]
|
||
|
||
# Find chapter 7's sub-section bookmarks (7a through 7l)
|
||
ch7_subsections = []
|
||
if ch7_bm:
|
||
ch7_range_start = ch7_bm[0]
|
||
ch7_range_end = hkh_end
|
||
ch7_subsections = [
|
||
s for s in all_subsections if ch7_range_start <= s[0] < ch7_range_end
|
||
]
|
||
|
||
# Sub-sections to skip entirely:
|
||
# 7a = "Entrance Qualifications", 7b = "Curriculum" (administrative)
|
||
# 7h = "ISKCON Press" (organizational, not spiritual content)
|
||
CH7_SKIP_CODES = {"7a", "7b", "7h"}
|
||
ch7_split = [
|
||
s
|
||
for s in ch7_subsections
|
||
if s[1].split(":")[0].strip().replace("HKH ", "") not in CH7_SKIP_CODES
|
||
]
|
||
|
||
chapters = []
|
||
output_ch_num = 0
|
||
|
||
for idx, (ch_pos, ch_num, ch_title, _) in enumerate(numbered_bookmarks):
|
||
content_start = ch_pos + 1
|
||
|
||
if idx + 1 < len(numbered_bookmarks):
|
||
content_end = numbered_bookmarks[idx + 1][0]
|
||
else:
|
||
content_end = hkh_end
|
||
|
||
# --- Special handling for chapter 6: split into sub-section chapters ---
|
||
if ch_num == 6 and ch6_subsections:
|
||
for sub_idx, (sub_pos, sub_code, sub_title) in enumerate(ch6_subsections):
|
||
output_ch_num += 1
|
||
sub_start = sub_pos + 1
|
||
if sub_idx + 1 < len(ch6_subsections):
|
||
sub_end = ch6_subsections[sub_idx + 1][0]
|
||
else:
|
||
sub_end = content_end
|
||
|
||
paragraphs = extract_paragraphs(all_paragraphs, sub_start, sub_end)
|
||
clean_title = decode_scagoudy(sub_title).strip()
|
||
|
||
chapters.append(
|
||
{
|
||
"book": "hkh",
|
||
"chapter": output_ch_num,
|
||
"title": clean_title,
|
||
"type": "chapter",
|
||
"paragraphs": paragraphs,
|
||
}
|
||
)
|
||
continue
|
||
|
||
# --- Special handling for chapter 7: skip 7a+7b (curriculum), split 7c-7l ---
|
||
if ch_num == 7 and ch7_split:
|
||
# Skip excluded sub-sections (7a, 7b, 7h).
|
||
# Only emit remaining sub-sections as separate chapters.
|
||
|
||
for sub_idx, (sub_pos, sub_code, sub_title) in enumerate(ch7_split):
|
||
output_ch_num += 1
|
||
sub_start = sub_pos + 1
|
||
if sub_idx + 1 < len(ch7_split):
|
||
sub_end = ch7_split[sub_idx + 1][0]
|
||
else:
|
||
sub_end = content_end
|
||
|
||
paragraphs = extract_paragraphs(all_paragraphs, sub_start, sub_end)
|
||
clean_title = decode_scagoudy(sub_title).strip()
|
||
|
||
chapters.append(
|
||
{
|
||
"book": "hkh",
|
||
"chapter": output_ch_num,
|
||
"title": clean_title,
|
||
"type": "chapter",
|
||
"paragraphs": paragraphs,
|
||
}
|
||
)
|
||
continue
|
||
|
||
# --- Normal chapters (1-5) ---
|
||
output_ch_num += 1
|
||
|
||
paragraphs = extract_paragraphs(
|
||
all_paragraphs,
|
||
content_start,
|
||
content_end,
|
||
)
|
||
|
||
clean_title = clean_chapter_title(ch_num, ch_title)
|
||
|
||
chapters.append(
|
||
{
|
||
"book": "hkh",
|
||
"chapter": output_ch_num,
|
||
"title": clean_title,
|
||
"type": "chapter",
|
||
"paragraphs": paragraphs,
|
||
}
|
||
)
|
||
|
||
# Step 8: Build chapter listing for intro.json
|
||
chapter_listing = [
|
||
{"number": ch["chapter"], "title": ch["title"]} for ch in chapters
|
||
]
|
||
|
||
return {
|
||
"intro_segments": intro_segments,
|
||
"chapters": chapters,
|
||
"chapter_listing": chapter_listing,
|
||
}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Output
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def write_json(data, filepath):
|
||
"""Write data as formatted JSON to filepath, creating directories as needed."""
|
||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||
with open(filepath, "w", encoding="utf-8") as f:
|
||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||
|
||
|
||
def main():
|
||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
html_path = os.path.join(base_dir, "public", "html", "other.html")
|
||
output_base = os.path.join(base_dir, "lib", "jsons", "books", "hkh")
|
||
|
||
if not os.path.exists(html_path):
|
||
print(f"ERROR: {html_path} not found!")
|
||
sys.exit(1)
|
||
|
||
print(f"Reading {html_path}...")
|
||
with open(html_path, "r", encoding="utf-8") as f:
|
||
soup = BeautifulSoup(f, "html.parser")
|
||
|
||
print("Extracting The Kṛṣṇa Consciousness Handbook (1971)...\n")
|
||
result = extract_hkh(soup)
|
||
|
||
intro_segments = result["intro_segments"]
|
||
chapters = result["chapters"]
|
||
chapter_listing = result["chapter_listing"]
|
||
|
||
# --- Write intro.json ---
|
||
intro_meta = {
|
||
"book": "hkh",
|
||
"title": "The Kṛṣṇa Consciousness Handbook",
|
||
"author": "International Society for Krishna Consciousness",
|
||
"year": 1971,
|
||
"description": "The first annual handbook of ISKCON, covering the history, teachings, curriculum, and practices of Kṛṣṇa consciousness.",
|
||
"totalChapters": 7,
|
||
"chapters": chapter_listing,
|
||
}
|
||
intro_path = os.path.join(output_base, "intro.json")
|
||
write_json(intro_meta, intro_path)
|
||
print(f"\n Written: {intro_path}")
|
||
|
||
# --- Write intro segments ---
|
||
print(f"\n--- Intro Segments ---")
|
||
for seg in intro_segments:
|
||
filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
|
||
write_json(seg, filepath)
|
||
print(
|
||
f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}"
|
||
)
|
||
|
||
# --- Write chapters ---
|
||
print(f"\n--- Chapters ---")
|
||
for ch in chapters:
|
||
ch_num_str = str(ch["chapter"]).zfill(2)
|
||
filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
|
||
write_json(ch, filepath)
|
||
print(
|
||
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
|
||
f"{ch['title']}"
|
||
)
|
||
|
||
# --- Summary ---
|
||
total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments)
|
||
total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
|
||
total_paras = total_intro_paras + total_chapter_paras
|
||
|
||
print(f"\n{'=' * 60}")
|
||
print(f" EXTRACTION SUMMARY")
|
||
print(f"{'=' * 60}")
|
||
print(f" Intro segments: {len(intro_segments)}")
|
||
print(f" Chapters: {len(chapters)}")
|
||
print(f" Total paragraphs: {total_paras}")
|
||
print(f" Intro: {total_intro_paras}")
|
||
print(f" Chapters: {total_chapter_paras}")
|
||
if chapters:
|
||
print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
|
||
print(f" Output directory: {output_base}/")
|
||
|
||
# --- Samples ---
|
||
print(f"\n--- Sample: Front Pages ---")
|
||
if intro_segments:
|
||
front = intro_segments[0]
|
||
for i, p in enumerate(front["paragraphs"][:5]):
|
||
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
||
print(f" [{i}] {preview}...")
|
||
|
||
if len(intro_segments) > 1:
|
||
print(f"\n--- Sample: Introduction ---")
|
||
intro_seg = intro_segments[1]
|
||
for i, p in enumerate(intro_seg["paragraphs"][:5]):
|
||
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
||
print(f" [{i}] {preview}...")
|
||
|
||
if chapters:
|
||
print(f"\n--- Sample: Chapter 1 ({chapters[0]['title']}) ---")
|
||
ch1 = chapters[0]
|
||
for i, p in enumerate(ch1["paragraphs"][:5]):
|
||
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
||
print(f" [{i}] {preview}...")
|
||
|
||
if chapters and len(chapters) > 1:
|
||
print(
|
||
f"\n--- Sample: Chapter {chapters[-1]['chapter']} ({chapters[-1]['title']}) ---"
|
||
)
|
||
last = chapters[-1]
|
||
for i, p in enumerate(last["paragraphs"][:5]):
|
||
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
||
print(f" [{i}] {preview}...")
|
||
|
||
# --- Quality Checks ---
|
||
print(f"\n--- Quality Checks ---")
|
||
|
||
# Check all 7 chapters present
|
||
expected_chapters = set(range(1, 8))
|
||
actual_chapters = set(ch["chapter"] for ch in chapters)
|
||
missing = expected_chapters - actual_chapters
|
||
extra = actual_chapters - expected_chapters
|
||
if missing:
|
||
print(f" ⚠ Missing chapters: {sorted(missing)}")
|
||
if extra:
|
||
print(f" ⚠ Extra chapters: {sorted(extra)}")
|
||
if not missing and not extra:
|
||
print(f" ✅ All 7 chapters present")
|
||
|
||
# Check all 2 intro segments present
|
||
expected_intros = {"front-pages", "introduction"}
|
||
actual_intros = {s["slug"] for s in intro_segments}
|
||
missing_intros = expected_intros - actual_intros
|
||
extra_intros = actual_intros - expected_intros
|
||
if missing_intros:
|
||
print(f" ⚠ Missing intro segments: {missing_intros}")
|
||
if extra_intros:
|
||
print(f" ⚠ Extra intro segments: {extra_intros}")
|
||
if not missing_intros and not extra_intros:
|
||
print(f" ✅ All 2 intro segments present")
|
||
|
||
# Check empty chapters
|
||
empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
|
||
if empty_chapters:
|
||
print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
|
||
else:
|
||
print(f" ✅ No empty chapters")
|
||
|
||
# Check empty intro segments
|
||
empty_intros = [s["slug"] for s in intro_segments if len(s["paragraphs"]) == 0]
|
||
if empty_intros:
|
||
print(f" ⚠ Empty intro segments: {empty_intros}")
|
||
else:
|
||
print(f" ✅ No empty intro segments")
|
||
|
||
# Check ScaGoudy decoding
|
||
print(f"\n--- ScaGoudy Decoding Check ---")
|
||
all_text = ""
|
||
for ch in chapters:
|
||
all_text += " ".join(ch["paragraphs"][:20])
|
||
for seg in intro_segments:
|
||
all_text += " ".join(seg["paragraphs"][:10])
|
||
|
||
# Check for properly decoded words
|
||
if "Kṛṣṇa" in all_text:
|
||
print(f" ✅ 'Kṛṣṇa' found (ScaGoudy properly decoded)")
|
||
else:
|
||
print(f" ⚠ 'Kṛṣṇa' not found — ScaGoudy decoding may have failed")
|
||
|
||
if "Bhagavad-gītā" in all_text:
|
||
print(f" ✅ 'Bhagavad-gītā' found (ScaGoudy properly decoded)")
|
||
elif "gītā" in all_text or "Gītā" in all_text:
|
||
print(f" ✅ 'gītā' found (ScaGoudy properly decoded)")
|
||
else:
|
||
print(f" ⚠ 'Gītā' not found — ScaGoudy decoding may have failed")
|
||
|
||
if "Prabhupāda" in all_text:
|
||
print(f" ✅ 'Prabhupāda' found (ScaGoudy properly decoded)")
|
||
|
||
if "ISKCON" in all_text:
|
||
print(f" ✅ 'ISKCON' found")
|
||
|
||
# Check for un-decoded ScaGoudy remnants (common encoded chars)
|
||
scagoudy_remnants = 0
|
||
for ch in chapters:
|
||
for p in ch["paragraphs"]:
|
||
if "Kåñëa" in p or "Prabhupäda" in p:
|
||
scagoudy_remnants += 1
|
||
for seg in intro_segments:
|
||
for p in seg["paragraphs"]:
|
||
if "Kåñëa" in p or "Prabhupäda" in p:
|
||
scagoudy_remnants += 1
|
||
if scagoudy_remnants == 0:
|
||
print(f" ✅ No ScaGoudy remnants detected")
|
||
else:
|
||
print(f" ⚠ {scagoudy_remnants} paragraphs with ScaGoudy remnants")
|
||
|
||
# Check for centered verse blocks
|
||
verse_block_count = 0
|
||
for ch in chapters:
|
||
for p in ch["paragraphs"]:
|
||
if '<div style="text-align:center"><i>' in p:
|
||
verse_block_count += 1
|
||
if verse_block_count > 0:
|
||
print(f" ✅ {verse_block_count} centered Sanskrit verse blocks found")
|
||
else:
|
||
print(f" ⚠ No centered verse blocks found — verse grouping may have failed")
|
||
|
||
# Check for residual span tags
|
||
span_count = 0
|
||
for ch in chapters:
|
||
for p in ch["paragraphs"]:
|
||
if 'class="' in p and "<span" in p:
|
||
span_count += 1
|
||
for seg in intro_segments:
|
||
for p in seg["paragraphs"]:
|
||
if 'class="' in p and "<span" in p:
|
||
span_count += 1
|
||
if span_count == 0:
|
||
print(f" ✅ No residual span class artifacts found")
|
||
else:
|
||
print(f" ⚠ {span_count} paragraphs still contain span class artifacts")
|
||
|
||
# Check for dash-line paragraphs that shouldn't be there
|
||
dash_count = 0
|
||
for ch in chapters:
|
||
for p in ch["paragraphs"]:
|
||
plain = re.sub(r"<[^>]+>", "", p)
|
||
if is_dash_line(plain):
|
||
dash_count += 1
|
||
for seg in intro_segments:
|
||
for p in seg["paragraphs"]:
|
||
plain = re.sub(r"<[^>]+>", "", p)
|
||
if is_dash_line(plain):
|
||
dash_count += 1
|
||
if dash_count == 0:
|
||
print(f" ✅ No dashed-line paragraphs in output")
|
||
else:
|
||
print(f" ⚠ {dash_count} dashed-line paragraphs found in output")
|
||
|
||
# Show a verse block sample
|
||
print(f"\n--- Sample: Verse Block ---")
|
||
for ch in chapters:
|
||
for p in ch["paragraphs"]:
|
||
if '<div style="text-align:center"><i>' in p:
|
||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
||
print(f" {preview}...")
|
||
break
|
||
else:
|
||
continue
|
||
break
|
||
|
||
# Check sub-section headings in chapters 6 and 7
|
||
print(f"\n--- Sub-section Heading Check ---")
|
||
for ch in chapters:
|
||
if ch["chapter"] in (6, 7):
|
||
bold_headings = [
|
||
p
|
||
for p in ch["paragraphs"]
|
||
if p.startswith("<b>")
|
||
and p.endswith("</b>")
|
||
and len(re.sub(r"<[^>]+>", "", p)) < 80
|
||
]
|
||
print(
|
||
f" Chapter {ch['chapter']}: {len(bold_headings)} sub-section headings found"
|
||
)
|
||
for h in bold_headings[:5]:
|
||
preview = re.sub(r"<[^>]+>", "", h)
|
||
print(f" → {preview}")
|
||
|
||
# Chapter title decoding check
|
||
print(f"\n--- Chapter Title Decoding Check ---")
|
||
for ch in chapters:
|
||
title = ch["title"]
|
||
if "å" in title or "ë" in title or "ñ" in title or "ä" in title:
|
||
print(f" ⚠ Chapter {ch['chapter']} title has un-decoded ScaGoudy: {title}")
|
||
else:
|
||
print(f" ✅ Ch {ch['chapter']}: {title}")
|
||
|
||
print(f"\nDone! ✨")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|