314 lines
9.1 KiB
Python
314 lines
9.1 KiB
Python
#!/usr/bin/env python3
|
||
"""Extract Kṛṣṇa Book Volume 1 introductory segments from other.html.
|
||
|
||
The Volume 1 front matter (lines ~772–901) contains 5 segments before Chapter 1:
|
||
1. Flap intro — book jacket text
|
||
2. Dedication — "To My Father, Gour Mohon De (1849-1930)"
|
||
3. Words from Apple — George Harrison's introduction
|
||
4. Preface — Prabhupāda's preface
|
||
5. Introduction — Prabhupāda's introduction
|
||
|
||
Each segment is extracted into a JSON file with a common schema:
|
||
{
|
||
"slug": "flap-intro",
|
||
"title": "Flap Introduction",
|
||
"type": "intro-segment",
|
||
"book": "kb",
|
||
"volume": 1,
|
||
"order": 1,
|
||
"paragraphs": ["<p>...</p>", ...]
|
||
}
|
||
|
||
Output: lib/jsons/books/kb/vol-1/intro/*.json
|
||
|
||
Usage:
|
||
python3 scripts/kb_extract_vol1_intro.py
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import re
|
||
import sys
|
||
|
||
from bs4 import BeautifulSoup, Tag
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# ScaGoudy → IAST Unicode mapping
|
||
# ---------------------------------------------------------------------------
|
||
SCAGOUDY_TO_IAST = {
|
||
"ä": "ā",
|
||
"Ä": "Ā",
|
||
"å": "ṛ",
|
||
"Å": "Ṛ",
|
||
"ñ": "ṣ",
|
||
"Ñ": "Ṣ",
|
||
"ö": "ṭ",
|
||
"Ö": "Ṭ",
|
||
"ë": "ṇ",
|
||
"Ë": "Ṇ",
|
||
"ò": "ḍ",
|
||
"Ò": "Ḍ",
|
||
"ç": "ś",
|
||
"Ç": "Ś",
|
||
"ì": "ṅ",
|
||
"ï": "ñ",
|
||
"ü": "ū",
|
||
"Ü": "Ū",
|
||
"é": "ī",
|
||
"É": "Ī",
|
||
"à": "ṁ",
|
||
"ù": "ḥ",
|
||
"è": "ṝ",
|
||
"û": "ṁl",
|
||
"ÿ": "ḷ",
|
||
}
|
||
|
||
|
||
def decode_scagoudy(text: str) -> str:
|
||
for src, dst in SCAGOUDY_TO_IAST.items():
|
||
text = text.replace(src, dst)
|
||
return text
|
||
|
||
|
||
def decode_html_content(tag: Tag) -> str:
|
||
inner = "".join(str(child) for child in tag.children)
|
||
inner = decode_scagoudy(inner)
|
||
inner = re.sub(r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner)
|
||
inner = re.sub(r"\s+", " ", inner).strip()
|
||
return inner
|
||
|
||
|
||
def decode_text(tag: Tag) -> str:
|
||
return decode_scagoudy(tag.get_text().strip())
|
||
|
||
|
||
def get_css_class(tag: Tag) -> str:
|
||
classes = tag.get("class", [])
|
||
return classes[0] if classes else ""
|
||
|
||
|
||
# CSS classes for Sanskrit verse pādas (center-aligned italic)
|
||
VERSE_PADA_CLASSES = frozenset(["p4", "p5", "p10", "p19", "p24"])
|
||
|
||
|
||
def is_empty(tag: Tag) -> bool:
|
||
"""Check if a tag is empty / whitespace only."""
|
||
text = tag.get_text().strip()
|
||
return not text or text == "\xa0"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Segment definitions
|
||
# ---------------------------------------------------------------------------
|
||
|
||
SEGMENTS = [
|
||
{
|
||
"slug": "flap-intro",
|
||
"title": "Flap Introduction",
|
||
"order": 1,
|
||
"bookmark": "Kb 1970 /Flap intro",
|
||
},
|
||
{
|
||
"slug": "dedication",
|
||
"title": "Dedication — To My Father, Gour Mohon De (1849–1930)",
|
||
"order": 2,
|
||
"bookmark": "Kb 1970 / To My Father, Gour Mohon De (1849-1930)",
|
||
},
|
||
{
|
||
"slug": "words-from-apple",
|
||
"title": "Words from Apple — by George Harrison",
|
||
"order": 3,
|
||
"bookmark": "Kb 1970 / Words from Apple",
|
||
},
|
||
{
|
||
"slug": "preface",
|
||
"title": "Preface",
|
||
"order": 4,
|
||
"bookmark": "Kb 1970 / Preface",
|
||
},
|
||
{
|
||
"slug": "introduction",
|
||
"title": "Introduction",
|
||
"order": 5,
|
||
"bookmark": "Kb 1970 / Introduction",
|
||
},
|
||
]
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Extraction
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def find_segment_ranges(soup: BeautifulSoup):
|
||
"""Find the start/end paragraph indices for each segment.
|
||
|
||
Segments are delimited by p13 bookmark tags. The first chapter marker
|
||
(KB 1970-1-1) ends the last intro segment.
|
||
"""
|
||
all_paragraphs = list(soup.find_all("p"))
|
||
|
||
# Find all p13 bookmark indices within the KB volume 1 area
|
||
bookmark_positions = [] # (index, text)
|
||
for i, p in enumerate(all_paragraphs):
|
||
cls = get_css_class(p)
|
||
if cls == "p13":
|
||
text = decode_text(p)
|
||
bookmark_positions.append((i, text))
|
||
|
||
# Map each segment to its start/end paragraph indices
|
||
ranges = {}
|
||
for seg in SEGMENTS:
|
||
# Find the bookmark that starts this segment
|
||
start_idx = None
|
||
for i, (pos, text) in enumerate(bookmark_positions):
|
||
if seg["bookmark"] in text:
|
||
# Content starts AFTER the bookmark tag
|
||
start_idx = pos + 1
|
||
# End is the next bookmark
|
||
if i + 1 < len(bookmark_positions):
|
||
end_idx = bookmark_positions[i + 1][0]
|
||
else:
|
||
end_idx = len(all_paragraphs)
|
||
ranges[seg["slug"]] = (start_idx, end_idx, all_paragraphs)
|
||
break
|
||
|
||
if start_idx is None:
|
||
print(
|
||
f" WARNING: Could not find bookmark for '{seg['slug']}': '{seg['bookmark']}'"
|
||
)
|
||
|
||
return ranges
|
||
|
||
|
||
def extract_segment_paragraphs(
|
||
all_paragraphs: list,
|
||
start_idx: int,
|
||
end_idx: int,
|
||
) -> list[str]:
|
||
"""Extract paragraphs for a single segment, handling verse grouping."""
|
||
result: list[str] = []
|
||
verse_accumulator: list[str] = []
|
||
|
||
def flush_verse():
|
||
if verse_accumulator:
|
||
joined = "<br>".join(verse_accumulator)
|
||
result.append(f'<div style="text-align:center"><i>{joined}</i></div>')
|
||
verse_accumulator.clear()
|
||
|
||
for i in range(start_idx, end_idx):
|
||
p = all_paragraphs[i]
|
||
cls = get_css_class(p)
|
||
text = p.get_text().strip()
|
||
|
||
# Skip the segment's own heading (p12, p14, p22, p27 bold headers)
|
||
# These are the display titles like "Flap:", "Words From Apple", "Preface", etc.
|
||
# We keep them as the first paragraph if they have meaningful content
|
||
if cls in ("p12", "p14"):
|
||
# Volume header or section label — skip
|
||
continue
|
||
|
||
# Skip empty paragraphs
|
||
if is_empty(p):
|
||
continue
|
||
|
||
# Skip bookmark tags (p13) that might be nested
|
||
if cls == "p13":
|
||
continue
|
||
|
||
# Sanskrit verse pādas — accumulate
|
||
# Use decode_html_content to preserve <br> tags within multi-line verse elements
|
||
if cls in VERSE_PADA_CLASSES:
|
||
html = decode_html_content(p)
|
||
if html:
|
||
verse_accumulator.append(html)
|
||
continue
|
||
|
||
# Non-verse paragraph — flush any accumulated verse first
|
||
flush_verse()
|
||
|
||
# Extract the HTML content
|
||
html = decode_html_content(p)
|
||
if html:
|
||
result.append(html)
|
||
|
||
# Flush any trailing verse
|
||
flush_verse()
|
||
|
||
return result
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Output
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def write_json(data, filepath):
|
||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||
with open(filepath, "w", encoding="utf-8") as f:
|
||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||
print(f" Written: {filepath}")
|
||
|
||
|
||
def main():
|
||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
html_path = os.path.join(base_dir, "public", "html", "other.html")
|
||
output_dir = os.path.join(base_dir, "lib", "jsons", "books", "kb", "vol-1", "intro")
|
||
|
||
if not os.path.exists(html_path):
|
||
print(f"ERROR: {html_path} not found!")
|
||
sys.exit(1)
|
||
|
||
print(f"Reading {html_path}...")
|
||
with open(html_path, "r", encoding="utf-8") as f:
|
||
soup = BeautifulSoup(f, "html.parser")
|
||
|
||
print("Finding segment boundaries...")
|
||
ranges = find_segment_ranges(soup)
|
||
|
||
print(f"Extracting {len(SEGMENTS)} introductory segments...\n")
|
||
|
||
for seg in SEGMENTS:
|
||
slug = seg["slug"]
|
||
if slug not in ranges:
|
||
print(f" ⏭ Skipping '{slug}' — bookmark not found")
|
||
continue
|
||
|
||
start_idx, end_idx, all_paragraphs = ranges[slug]
|
||
paragraphs = extract_segment_paragraphs(all_paragraphs, start_idx, end_idx)
|
||
|
||
doc = {
|
||
"slug": slug,
|
||
"title": seg["title"],
|
||
"type": "intro-segment",
|
||
"book": "kb",
|
||
"volume": 1,
|
||
"order": seg["order"],
|
||
"paragraphs": paragraphs,
|
||
}
|
||
|
||
filepath = os.path.join(output_dir, f"{slug}.json")
|
||
write_json(doc, filepath)
|
||
print(f" {seg['title']}: {len(paragraphs)} paragraphs")
|
||
|
||
# Summary
|
||
print(f"\n--- Summary ---")
|
||
for seg in SEGMENTS:
|
||
slug = seg["slug"]
|
||
filepath = os.path.join(output_dir, f"{slug}.json")
|
||
if os.path.exists(filepath):
|
||
with open(filepath, "r", encoding="utf-8") as f:
|
||
doc = json.load(f)
|
||
# Show first paragraph preview
|
||
first = doc["paragraphs"][0] if doc["paragraphs"] else "(empty)"
|
||
# Strip HTML tags for preview
|
||
preview = re.sub(r"<[^>]+>", "", first)[:80]
|
||
print(f" {seg['order']}. {seg['title']}")
|
||
print(f' {len(doc["paragraphs"])} paragraphs — "{preview}..."')
|
||
else:
|
||
print(f" {seg['order']}. {seg['title']} — NOT EXTRACTED")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|