534 lines
17 KiB
Python
534 lines
17 KiB
Python
#!/usr/bin/env python3
|
||
"""Extract Prayers of King Kulaśekhara (1958) from others-before-1977.html.
|
||
|
||
The book has 6 chapters (one per verse/prayer):
|
||
1. Prayer 1 — "Śrī Vallabha" (prayer to Mukunda)
|
||
2. Prayer 2 — "Jayatu Jayatu" (glories to Devakī-nandana)
|
||
3. Prayer 3 — "Mukunda Mūrdhnā" (prayer for remembrance)
|
||
4. Prayer 4 — "Nāhaṁ Vande" (neither heaven nor hell)
|
||
5. Prayer 5 — "Nāsthā Dharme" (unflinching devotion)
|
||
6. Prayer 6 — "Divi Vā Bhuvi" (remembrance at death)
|
||
|
||
Output:
|
||
lib/jsons/books/pk/intro.json — Book metadata
|
||
lib/jsons/books/pk/chapters/*.json — Chapters (01–06)
|
||
|
||
Usage:
|
||
python3 scripts/pk_extract.py
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import re
|
||
import sys
|
||
|
||
from bs4 import BeautifulSoup, Tag
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def decode_html_content(tag: Tag) -> str:
|
||
"""Extract inner HTML from a tag, preserve formatting tags, strip spans."""
|
||
inner = "".join(str(child) for child in tag.children)
|
||
# Remove span tags with CSS classes (RTF conversion artifacts)
|
||
inner = re.sub(
|
||
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
|
||
)
|
||
# Clean up Apple-tab-span leftovers (tab characters)
|
||
inner = inner.replace("\t", " ")
|
||
# Normalize whitespace
|
||
inner = re.sub(r"\s+", " ", inner).strip()
|
||
return inner
|
||
|
||
|
||
def decode_text(tag: Tag) -> str:
|
||
"""Get plain text from a tag."""
|
||
return tag.get_text().strip()
|
||
|
||
|
||
def get_css_class(tag: Tag) -> str:
|
||
"""Get the first CSS class of a tag, or empty string."""
|
||
classes = tag.get("class", [])
|
||
return classes[0] if classes else ""
|
||
|
||
|
||
def is_empty(tag: Tag) -> bool:
|
||
"""Check if a tag is empty or whitespace-only."""
|
||
text = tag.get_text().strip()
|
||
return not text or text == "\xa0"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# CSS class categorization for PK
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# Document header
|
||
DOC_HEADER_CLASS = "p4" # "Prayers of King Kulaçekhara 1958 [Handwritten]"
|
||
TITLE_CLASS = "p115" # Bold title
|
||
|
||
# Chapter bookmark class
|
||
CHAPTER_BOOKMARK_CLASS = "p17" # "Prayers of King Kulaçekhara 1950: N"
|
||
|
||
# Verse number markers — skip
|
||
VERSE_MARKER_CLASSES = frozenset(["p108", "p9"])
|
||
|
||
# Devanagari marker — skip
|
||
DEVANAGARI_MARKER_CLASS = "p36"
|
||
|
||
# Section header class (Wordings., Synonyms., Translation., Purport.)
|
||
SECTION_HEADER_CLASSES = frozenset(["p20", "p121"])
|
||
|
||
# Body paragraph classes
|
||
BODY_CLASSES = frozenset(["p6", "p34", "p38", "p71"])
|
||
|
||
# Page number class — skip
|
||
PAGE_NUMBER_CLASSES = frozenset(["p15", "p41", "p124"])
|
||
|
||
# Empty/break classes — skip
|
||
SKIP_CLASSES = frozenset(["p24", "p94", "p119", "p122", "p123", "p120"])
|
||
|
||
# Next work boundary
|
||
NEXT_WORK_BOUNDARY_CLASS = "p125"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Boundary detection
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def find_pk_boundaries(all_elements: list) -> tuple:
|
||
"""Find the start and end element indices of the PK content.
|
||
|
||
Start: <p class="p4"> containing "Prayers of King Kula"
|
||
End: <p class="p125"> that starts the next work ("Shaktipujah")
|
||
"""
|
||
start_idx = None
|
||
end_idx = len(all_elements)
|
||
|
||
for i, el in enumerate(all_elements):
|
||
if el.name != "p":
|
||
continue
|
||
cls = get_css_class(el)
|
||
text = decode_text(el)
|
||
|
||
# Find start: p4 containing "Prayers of King"
|
||
if start_idx is None:
|
||
if cls == DOC_HEADER_CLASS and "Prayers of King" in text:
|
||
start_idx = i
|
||
continue
|
||
continue
|
||
|
||
# Find end: p125 that starts a different work
|
||
if cls == NEXT_WORK_BOUNDARY_CLASS:
|
||
end_idx = i
|
||
break
|
||
|
||
if start_idx is None:
|
||
print(
|
||
"ERROR: Could not find 'Prayers of King Kulaçekhara' in others-before-1977.html!"
|
||
)
|
||
sys.exit(1)
|
||
|
||
return start_idx, end_idx
|
||
|
||
|
||
def find_chapter_bookmarks(all_elements: list, start_idx: int, end_idx: int) -> list:
|
||
"""Find all PK chapter bookmark tags (p17) in the PK range.
|
||
|
||
Returns list of (element_index, verse_number).
|
||
"""
|
||
bookmarks = []
|
||
for i in range(start_idx, end_idx):
|
||
el = all_elements[i]
|
||
if el.name != "p":
|
||
continue
|
||
cls = get_css_class(el)
|
||
if cls == CHAPTER_BOOKMARK_CLASS:
|
||
text = decode_text(el).strip()
|
||
if "Prayers of King" in text or "Kulaçekhara" in text.replace("ç", "ç"):
|
||
# Extract the verse number from the end
|
||
match = re.search(r":\s*(\d+)\s*$", text)
|
||
if match:
|
||
verse_num = int(match.group(1))
|
||
bookmarks.append((i, verse_num))
|
||
return bookmarks
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Table synonym extraction
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def extract_synonyms_from_table(table: Tag) -> str:
|
||
"""Extract synonyms from a <table class="t1"> and convert to text.
|
||
|
||
Each cell contains a <p> with "word=meaning" format.
|
||
Convert to "word—meaning; word—meaning; ..." format.
|
||
"""
|
||
entries = []
|
||
for row in table.find_all("tr"):
|
||
for cell in row.find_all("td"):
|
||
# Get all <p> tags inside the cell
|
||
for p in cell.find_all("p"):
|
||
text = decode_text(p).strip()
|
||
if not text or text == "\xa0":
|
||
continue
|
||
# Must contain '=' to be a synonym entry
|
||
if "=" not in text:
|
||
continue
|
||
# Replace = with em-dash
|
||
text = text.replace("=", "—", 1)
|
||
# Clean trailing whitespace/period if needed
|
||
text = text.strip()
|
||
entries.append(text)
|
||
|
||
if not entries:
|
||
return ""
|
||
return "; ".join(entries)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Paragraph extraction for a chapter
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def extract_chapter_paragraphs(
|
||
all_elements: list,
|
||
start_idx: int,
|
||
end_idx: int,
|
||
) -> list:
|
||
"""Extract paragraphs for a chapter (verse).
|
||
|
||
Handles section headers (Wordings., Synonyms., Translation., Purport.),
|
||
body paragraphs, and synonym tables.
|
||
|
||
Returns a list of HTML paragraph strings.
|
||
"""
|
||
result = []
|
||
|
||
# Build a set of <p> tags that are inside tables, so we don't double-process
|
||
table_p_tags = set()
|
||
for i in range(start_idx, end_idx):
|
||
el = all_elements[i]
|
||
if el.name == "table":
|
||
for p in el.find_all("p"):
|
||
table_p_tags.add(id(p))
|
||
|
||
for i in range(start_idx, end_idx):
|
||
el = all_elements[i]
|
||
|
||
# Handle tables (synonyms)
|
||
if el.name == "table":
|
||
table_cls = get_css_class(el)
|
||
if table_cls == "t1":
|
||
synonyms_text = extract_synonyms_from_table(el)
|
||
if synonyms_text:
|
||
result.append(synonyms_text)
|
||
continue
|
||
|
||
# Only process <p> tags from here
|
||
if el.name != "p":
|
||
continue
|
||
|
||
# Skip <p> tags that are inside tables (already processed)
|
||
if id(el) in table_p_tags:
|
||
continue
|
||
|
||
cls = get_css_class(el)
|
||
|
||
# Skip empty paragraphs
|
||
if is_empty(el):
|
||
continue
|
||
|
||
# Skip page numbers
|
||
if cls in PAGE_NUMBER_CLASSES:
|
||
continue
|
||
|
||
# Skip verse markers
|
||
if cls in VERSE_MARKER_CLASSES:
|
||
text = decode_text(el)
|
||
# p9 can also be "(Devanägaré)" or prayer number markers — skip all
|
||
continue
|
||
|
||
# Skip devanagari markers
|
||
if cls == DEVANAGARI_MARKER_CLASS:
|
||
continue
|
||
|
||
# Skip chapter bookmarks
|
||
if cls == CHAPTER_BOOKMARK_CLASS:
|
||
continue
|
||
|
||
# Skip document header and title (only at very start)
|
||
if cls == DOC_HEADER_CLASS:
|
||
continue
|
||
if cls == TITLE_CLASS:
|
||
continue
|
||
|
||
# Skip empty/break classes
|
||
if cls in SKIP_CLASSES:
|
||
continue
|
||
|
||
# Section headers: Wordings., Synonyms., Translation., Purport.
|
||
if cls in SECTION_HEADER_CLASSES:
|
||
text = decode_text(el).strip()
|
||
# Normalize section header names
|
||
text_lower = text.lower().rstrip(".")
|
||
if "wording" in text_lower:
|
||
result.append("<b>Wordings</b>")
|
||
elif (
|
||
"synonym" in text_lower
|
||
or "synomym" in text_lower
|
||
or "synonysm" in text_lower
|
||
):
|
||
result.append("<b>Synonyms</b>")
|
||
elif "translation" in text_lower:
|
||
result.append("<b>Translation</b>")
|
||
elif "purport" in text_lower:
|
||
result.append("<b>Purport</b>")
|
||
else:
|
||
# Unknown header — include as bold
|
||
html = decode_html_content(el)
|
||
if html:
|
||
result.append(f"<b>{html}</b>")
|
||
continue
|
||
|
||
# Body paragraphs
|
||
if cls in BODY_CLASSES:
|
||
html = decode_html_content(el)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Fallback: try to extract content from unknown classes
|
||
html = decode_html_content(el)
|
||
if html:
|
||
result.append(html)
|
||
|
||
return result
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Main extraction logic
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def extract_pk(soup: BeautifulSoup) -> dict:
|
||
"""Extract the entire Prayers of King Kulaśekhara from the parsed HTML.
|
||
|
||
Returns a dict with:
|
||
"chapters": list of chapter dicts
|
||
"chapter_listing": list of {number, title} for intro.json
|
||
"""
|
||
# We need both <p> and <table> elements in document order
|
||
all_elements = list(soup.find_all(["p", "table"]))
|
||
|
||
# Step 1: Find PK boundaries
|
||
pk_start, pk_end = find_pk_boundaries(all_elements)
|
||
print(f" PK content: elements {pk_start} to {pk_end}")
|
||
print(f" Total elements in range: {pk_end - pk_start}")
|
||
|
||
# Step 2: Find all chapter bookmarks (p17)
|
||
bookmarks = find_chapter_bookmarks(all_elements, pk_start, pk_end)
|
||
print(f" Found {len(bookmarks)} PK chapter bookmarks:")
|
||
for pos, verse_num in bookmarks:
|
||
print(f" Element {pos}: Verse {verse_num}")
|
||
|
||
# If we didn't find 6 bookmarks, also look for the first verse that starts
|
||
# before any p17 bookmark (verse 1 may start with p108 [1])
|
||
# Actually, verse 1's bookmark IS in the HTML. Let's verify.
|
||
if len(bookmarks) < 6:
|
||
print(f" WARNING: Expected 6 bookmarks, found {len(bookmarks)}")
|
||
|
||
# Step 3: Build chapter ranges
|
||
# Each bookmark marks the START of a verse section.
|
||
# Content for verse N runs from bookmark N to bookmark N+1 (or pk_end).
|
||
chapters = []
|
||
for idx, (bk_pos, verse_num) in enumerate(bookmarks):
|
||
# Content starts after the bookmark
|
||
content_start = bk_pos + 1
|
||
|
||
# Content ends at next bookmark or pk_end
|
||
if idx + 1 < len(bookmarks):
|
||
content_end = bookmarks[idx + 1][0]
|
||
else:
|
||
content_end = pk_end
|
||
|
||
paragraphs = extract_chapter_paragraphs(
|
||
all_elements, content_start, content_end
|
||
)
|
||
|
||
ch = {
|
||
"book": "pk",
|
||
"chapter": verse_num,
|
||
"title": f"Prayer {verse_num}",
|
||
"type": "chapter",
|
||
"paragraphs": paragraphs,
|
||
}
|
||
chapters.append(ch)
|
||
|
||
# Step 4: Build chapter listing for intro.json
|
||
chapter_listing = [
|
||
{"number": ch["chapter"], "title": ch["title"]} for ch in chapters
|
||
]
|
||
|
||
return {
|
||
"chapters": chapters,
|
||
"chapter_listing": chapter_listing,
|
||
}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Output
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def write_json(data, filepath):
|
||
"""Write data as formatted JSON to filepath, creating directories as needed."""
|
||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||
with open(filepath, "w", encoding="utf-8") as f:
|
||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||
|
||
|
||
def main():
|
||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html")
|
||
output_base = os.path.join(base_dir, "lib", "jsons", "books", "pk")
|
||
|
||
if not os.path.exists(html_path):
|
||
print(f"ERROR: {html_path} not found!")
|
||
sys.exit(1)
|
||
|
||
print(f"Reading {html_path}...")
|
||
with open(html_path, "r", encoding="utf-8") as f:
|
||
soup = BeautifulSoup(f, "html.parser")
|
||
|
||
print("Extracting Prayers of King Kulaśekhara (1958)...\n")
|
||
result = extract_pk(soup)
|
||
|
||
chapters = result["chapters"]
|
||
chapter_listing = result["chapter_listing"]
|
||
|
||
# --- Write intro.json ---
|
||
intro_meta = {
|
||
"book": "pk",
|
||
"title": "Prayers of King Kulaśekhara (1958)",
|
||
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
||
"year": 1958,
|
||
"description": "Early handwritten translations and purports of the Mukunda-mālā-stotra prayers (verses 1–6).",
|
||
"totalChapters": 6,
|
||
"chapters": chapter_listing,
|
||
}
|
||
intro_path = os.path.join(output_base, "intro.json")
|
||
write_json(intro_meta, intro_path)
|
||
print(f"\n Written: {intro_path}")
|
||
|
||
# --- Write chapters ---
|
||
print(f"\n--- Chapters ---")
|
||
for ch in chapters:
|
||
ch_num_str = str(ch["chapter"]).zfill(2)
|
||
filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
|
||
write_json(ch, filepath)
|
||
print(
|
||
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
|
||
f"{ch['title']}"
|
||
)
|
||
|
||
# --- Summary ---
|
||
total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
|
||
|
||
print(f"\n{'=' * 60}")
|
||
print(f" EXTRACTION SUMMARY")
|
||
print(f"{'=' * 60}")
|
||
print(f" Chapters: {len(chapters)}")
|
||
for ch in chapters:
|
||
print(
|
||
f" Ch {ch['chapter']}: {len(ch['paragraphs'])} paragraphs — {ch['title']}"
|
||
)
|
||
print(f" Total paragraphs: {total_chapter_paras}")
|
||
if chapters:
|
||
print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
|
||
print(f" Output directory: {output_base}/")
|
||
|
||
# --- Samples ---
|
||
for ch_idx, ch in enumerate(chapters):
|
||
print(f"\n--- Sample: Chapter {ch['chapter']} ({ch['title']}) ---")
|
||
for i, p in enumerate(ch["paragraphs"][:5]):
|
||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
||
print(f" [{i}] {preview}")
|
||
if len(ch["paragraphs"]) > 5:
|
||
print(f" ... ({len(ch['paragraphs'])} total paragraphs)")
|
||
|
||
# --- Quality Checks ---
|
||
print(f"\n--- Quality Checks ---")
|
||
|
||
# Check all 6 chapters present
|
||
expected_chapters = set(range(1, 7))
|
||
actual_chapters = set(ch["chapter"] for ch in chapters)
|
||
missing = expected_chapters - actual_chapters
|
||
extra = actual_chapters - expected_chapters
|
||
if missing:
|
||
print(f" ⚠ Missing chapters: {sorted(missing)}")
|
||
if extra:
|
||
print(f" ⚠ Extra chapters: {sorted(extra)}")
|
||
if not missing and not extra:
|
||
print(f" ✅ All 6 chapters present")
|
||
|
||
# Check empty chapters
|
||
empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
|
||
if empty_chapters:
|
||
print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
|
||
else:
|
||
print(f" ✅ No empty chapters")
|
||
|
||
# Check each chapter has expected sections
|
||
for ch in chapters:
|
||
ch_text = " ".join(ch["paragraphs"])
|
||
has_wordings = "<b>Wordings</b>" in ch_text
|
||
has_synonyms = "<b>Synonyms</b>" in ch_text
|
||
has_translation = "<b>Translation</b>" in ch_text
|
||
has_purport = "<b>Purport</b>" in ch_text
|
||
sections = []
|
||
if has_wordings:
|
||
sections.append("W")
|
||
if has_synonyms:
|
||
sections.append("S")
|
||
if has_translation:
|
||
sections.append("T")
|
||
if has_purport:
|
||
sections.append("P")
|
||
status = "✅" if len(sections) == 4 else "⚠"
|
||
print(f" {status} Ch {ch['chapter']}: sections [{', '.join(sections)}]")
|
||
|
||
# Check synonym entries use em-dash format
|
||
synonym_count = 0
|
||
for ch in chapters:
|
||
for p in ch["paragraphs"]:
|
||
if "—" in p and "<b>" not in p:
|
||
synonym_count += 1
|
||
break
|
||
print(f" ℹ Chapters with em-dash synonyms: {synonym_count}/6")
|
||
|
||
# Verify no Shaktipujah content leaked in
|
||
all_text = ""
|
||
for ch in chapters:
|
||
all_text += " ".join(ch["paragraphs"])
|
||
if "Shaktipujah" in all_text or "Baidya Nath" in all_text:
|
||
print(f" ⚠ Content from Shaktipujah may have leaked into PK!")
|
||
else:
|
||
print(f" ✅ No content leak from next work detected")
|
||
|
||
# Check last chapter ends cleanly
|
||
if chapters:
|
||
last_ch = chapters[-1]
|
||
last_para = last_ch["paragraphs"][-1] if last_ch["paragraphs"] else ""
|
||
last_preview = re.sub(r"<[^>]+>", "", last_para)[:80]
|
||
print(f' ℹ Last paragraph of Ch {last_ch["chapter"]}: "{last_preview}..."')
|
||
|
||
print(f"\nDone! ✨")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|