feat: kb both volumes
This commit is contained in:
gosangam
@@ -0,0 +1,313 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract Kṛṣṇa Book Volume 1 introductory segments from other.html.
|
||||
|
||||
The Volume 1 front matter (lines ~772–901) contains 5 segments before Chapter 1:
|
||||
1. Flap intro — book jacket text
|
||||
2. Dedication — "To My Father, Gour Mohon De (1849-1930)"
|
||||
3. Words from Apple — George Harrison's introduction
|
||||
4. Preface — Prabhupāda's preface
|
||||
5. Introduction — Prabhupāda's introduction
|
||||
|
||||
Each segment is extracted into a JSON file with a common schema:
|
||||
{
|
||||
"slug": "flap-intro",
|
||||
"title": "Flap Introduction",
|
||||
"type": "intro-segment",
|
||||
"book": "kb",
|
||||
"volume": 1,
|
||||
"order": 1,
|
||||
"paragraphs": ["<p>...</p>", ...]
|
||||
}
|
||||
|
||||
Output: lib/jsons/books/kb/vol-1/intro/*.json
|
||||
|
||||
Usage:
|
||||
python3 scripts/kb_extract_vol1_intro.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ScaGoudy → IAST Unicode mapping
|
||||
# ---------------------------------------------------------------------------
|
||||
SCAGOUDY_TO_IAST = {
|
||||
"ä": "ā",
|
||||
"Ä": "Ā",
|
||||
"å": "ṛ",
|
||||
"Å": "Ṛ",
|
||||
"ñ": "ṣ",
|
||||
"Ñ": "Ṣ",
|
||||
"ö": "ṭ",
|
||||
"Ö": "Ṭ",
|
||||
"ë": "ṇ",
|
||||
"Ë": "Ṇ",
|
||||
"ò": "ḍ",
|
||||
"Ò": "Ḍ",
|
||||
"ç": "ś",
|
||||
"Ç": "Ś",
|
||||
"ì": "ṅ",
|
||||
"ï": "ñ",
|
||||
"ü": "ū",
|
||||
"Ü": "Ū",
|
||||
"é": "ī",
|
||||
"É": "Ī",
|
||||
"à": "ṁ",
|
||||
"ù": "ḥ",
|
||||
"è": "ṝ",
|
||||
"û": "ṁl",
|
||||
"ÿ": "ḷ",
|
||||
}
|
||||
|
||||
|
||||
def decode_scagoudy(text: str) -> str:
|
||||
for src, dst in SCAGOUDY_TO_IAST.items():
|
||||
text = text.replace(src, dst)
|
||||
return text
|
||||
|
||||
|
||||
def decode_html_content(tag: Tag) -> str:
|
||||
inner = "".join(str(child) for child in tag.children)
|
||||
inner = decode_scagoudy(inner)
|
||||
inner = re.sub(r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner)
|
||||
inner = re.sub(r"\s+", " ", inner).strip()
|
||||
return inner
|
||||
|
||||
|
||||
def decode_text(tag: Tag) -> str:
|
||||
return decode_scagoudy(tag.get_text().strip())
|
||||
|
||||
|
||||
def get_css_class(tag: Tag) -> str:
|
||||
classes = tag.get("class", [])
|
||||
return classes[0] if classes else ""
|
||||
|
||||
|
||||
# CSS classes for Sanskrit verse pādas (center-aligned italic)
|
||||
VERSE_PADA_CLASSES = frozenset(["p4", "p5", "p10", "p19", "p24"])
|
||||
|
||||
|
||||
def is_empty(tag: Tag) -> bool:
|
||||
"""Check if a tag is empty / whitespace only."""
|
||||
text = tag.get_text().strip()
|
||||
return not text or text == "\xa0"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Segment definitions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
SEGMENTS = [
|
||||
{
|
||||
"slug": "flap-intro",
|
||||
"title": "Flap Introduction",
|
||||
"order": 1,
|
||||
"bookmark": "Kb 1970 /Flap intro",
|
||||
},
|
||||
{
|
||||
"slug": "dedication",
|
||||
"title": "Dedication — To My Father, Gour Mohon De (1849–1930)",
|
||||
"order": 2,
|
||||
"bookmark": "Kb 1970 / To My Father, Gour Mohon De (1849-1930)",
|
||||
},
|
||||
{
|
||||
"slug": "words-from-apple",
|
||||
"title": "Words from Apple — by George Harrison",
|
||||
"order": 3,
|
||||
"bookmark": "Kb 1970 / Words from Apple",
|
||||
},
|
||||
{
|
||||
"slug": "preface",
|
||||
"title": "Preface",
|
||||
"order": 4,
|
||||
"bookmark": "Kb 1970 / Preface",
|
||||
},
|
||||
{
|
||||
"slug": "introduction",
|
||||
"title": "Introduction",
|
||||
"order": 5,
|
||||
"bookmark": "Kb 1970 / Introduction",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def find_segment_ranges(soup: BeautifulSoup):
|
||||
"""Find the start/end paragraph indices for each segment.
|
||||
|
||||
Segments are delimited by p13 bookmark tags. The first chapter marker
|
||||
(KB 1970-1-1) ends the last intro segment.
|
||||
"""
|
||||
all_paragraphs = list(soup.find_all("p"))
|
||||
|
||||
# Find all p13 bookmark indices within the KB volume 1 area
|
||||
bookmark_positions = [] # (index, text)
|
||||
for i, p in enumerate(all_paragraphs):
|
||||
cls = get_css_class(p)
|
||||
if cls == "p13":
|
||||
text = decode_text(p)
|
||||
bookmark_positions.append((i, text))
|
||||
|
||||
# Map each segment to its start/end paragraph indices
|
||||
ranges = {}
|
||||
for seg in SEGMENTS:
|
||||
# Find the bookmark that starts this segment
|
||||
start_idx = None
|
||||
for i, (pos, text) in enumerate(bookmark_positions):
|
||||
if seg["bookmark"] in text:
|
||||
# Content starts AFTER the bookmark tag
|
||||
start_idx = pos + 1
|
||||
# End is the next bookmark
|
||||
if i + 1 < len(bookmark_positions):
|
||||
end_idx = bookmark_positions[i + 1][0]
|
||||
else:
|
||||
end_idx = len(all_paragraphs)
|
||||
ranges[seg["slug"]] = (start_idx, end_idx, all_paragraphs)
|
||||
break
|
||||
|
||||
if start_idx is None:
|
||||
print(
|
||||
f" WARNING: Could not find bookmark for '{seg['slug']}': '{seg['bookmark']}'"
|
||||
)
|
||||
|
||||
return ranges
|
||||
|
||||
|
||||
def extract_segment_paragraphs(
|
||||
all_paragraphs: list,
|
||||
start_idx: int,
|
||||
end_idx: int,
|
||||
) -> list[str]:
|
||||
"""Extract paragraphs for a single segment, handling verse grouping."""
|
||||
result: list[str] = []
|
||||
verse_accumulator: list[str] = []
|
||||
|
||||
def flush_verse():
|
||||
if verse_accumulator:
|
||||
joined = "<br>".join(verse_accumulator)
|
||||
result.append(f'<div style="text-align:center"><i>{joined}</i></div>')
|
||||
verse_accumulator.clear()
|
||||
|
||||
for i in range(start_idx, end_idx):
|
||||
p = all_paragraphs[i]
|
||||
cls = get_css_class(p)
|
||||
text = p.get_text().strip()
|
||||
|
||||
# Skip the segment's own heading (p12, p14, p22, p27 bold headers)
|
||||
# These are the display titles like "Flap:", "Words From Apple", "Preface", etc.
|
||||
# We keep them as the first paragraph if they have meaningful content
|
||||
if cls in ("p12", "p14"):
|
||||
# Volume header or section label — skip
|
||||
continue
|
||||
|
||||
# Skip empty paragraphs
|
||||
if is_empty(p):
|
||||
continue
|
||||
|
||||
# Skip bookmark tags (p13) that might be nested
|
||||
if cls == "p13":
|
||||
continue
|
||||
|
||||
# Sanskrit verse pādas — accumulate
|
||||
# Use decode_html_content to preserve <br> tags within multi-line verse elements
|
||||
if cls in VERSE_PADA_CLASSES:
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
verse_accumulator.append(html)
|
||||
continue
|
||||
|
||||
# Non-verse paragraph — flush any accumulated verse first
|
||||
flush_verse()
|
||||
|
||||
# Extract the HTML content
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
result.append(html)
|
||||
|
||||
# Flush any trailing verse
|
||||
flush_verse()
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Output
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def write_json(data, filepath):
|
||||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||||
with open(filepath, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
print(f" Written: {filepath}")
|
||||
|
||||
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "other.html")
|
||||
output_dir = os.path.join(base_dir, "lib", "jsons", "books", "kb", "vol-1", "intro")
|
||||
|
||||
if not os.path.exists(html_path):
|
||||
print(f"ERROR: {html_path} not found!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Reading {html_path}...")
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
soup = BeautifulSoup(f, "html.parser")
|
||||
|
||||
print("Finding segment boundaries...")
|
||||
ranges = find_segment_ranges(soup)
|
||||
|
||||
print(f"Extracting {len(SEGMENTS)} introductory segments...\n")
|
||||
|
||||
for seg in SEGMENTS:
|
||||
slug = seg["slug"]
|
||||
if slug not in ranges:
|
||||
print(f" ⏭ Skipping '{slug}' — bookmark not found")
|
||||
continue
|
||||
|
||||
start_idx, end_idx, all_paragraphs = ranges[slug]
|
||||
paragraphs = extract_segment_paragraphs(all_paragraphs, start_idx, end_idx)
|
||||
|
||||
doc = {
|
||||
"slug": slug,
|
||||
"title": seg["title"],
|
||||
"type": "intro-segment",
|
||||
"book": "kb",
|
||||
"volume": 1,
|
||||
"order": seg["order"],
|
||||
"paragraphs": paragraphs,
|
||||
}
|
||||
|
||||
filepath = os.path.join(output_dir, f"{slug}.json")
|
||||
write_json(doc, filepath)
|
||||
print(f" {seg['title']}: {len(paragraphs)} paragraphs")
|
||||
|
||||
# Summary
|
||||
print(f"\n--- Summary ---")
|
||||
for seg in SEGMENTS:
|
||||
slug = seg["slug"]
|
||||
filepath = os.path.join(output_dir, f"{slug}.json")
|
||||
if os.path.exists(filepath):
|
||||
with open(filepath, "r", encoding="utf-8") as f:
|
||||
doc = json.load(f)
|
||||
# Show first paragraph preview
|
||||
first = doc["paragraphs"][0] if doc["paragraphs"] else "(empty)"
|
||||
# Strip HTML tags for preview
|
||||
preview = re.sub(r"<[^>]+>", "", first)[:80]
|
||||
print(f" {seg['order']}. {seg['title']}")
|
||||
print(f' {len(doc["paragraphs"])} paragraphs — "{preview}..."')
|
||||
else:
|
||||
print(f" {seg['order']}. {seg['title']} — NOT EXTRACTED")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user