krispire/spsays-training
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add journals

Browse Source
This commit is contained in:
gosangam
2026-05-03 21:02:41 +05:30
parent 89106efc98
commit 35d1d14f37
47 changed files with 11365 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/pk_extract.py
+533
View File
@@ -0,0 +1,533 @@
#!/usr/bin/env python3
"""Extract Prayers of King Kulaśekhara (1958) from others-before-1977.html.
The book has 6 chapters (one per verse/prayer):
1. Prayer 1 — "Śrī Vallabha" (prayer to Mukunda)
2. Prayer 2 — "Jayatu Jayatu" (glories to Devakī-nandana)
3. Prayer 3 — "Mukunda Mūrdhnā" (prayer for remembrance)
4. Prayer 4 — "Nāhaṁ Vande" (neither heaven nor hell)
5. Prayer 5 — "Nāsthā Dharme" (unflinching devotion)
6. Prayer 6 — "Divi Vā Bhuvi" (remembrance at death)
Output:
lib/jsons/books/pk/intro.json — Book metadata
lib/jsons/books/pk/chapters/*.json — Chapters (01–06)
Usage:
python3 scripts/pk_extract.py
"""
import json
import os
import re
import sys
from bs4 import BeautifulSoup, Tag
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def decode_html_content(tag: Tag) -> str:
"""Extract inner HTML from a tag, preserve formatting tags, strip spans."""
inner = "".join(str(child) for child in tag.children)
# Remove span tags with CSS classes (RTF conversion artifacts)
inner = re.sub(
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
)
# Clean up Apple-tab-span leftovers (tab characters)
inner = inner.replace("\t", " ")
# Normalize whitespace
inner = re.sub(r"\s+", " ", inner).strip()
return inner
def decode_text(tag: Tag) -> str:
"""Get plain text from a tag."""
return tag.get_text().strip()
def get_css_class(tag: Tag) -> str:
"""Get the first CSS class of a tag, or empty string."""
classes = tag.get("class", [])
return classes[0] if classes else ""
def is_empty(tag: Tag) -> bool:
"""Check if a tag is empty or whitespace-only."""
text = tag.get_text().strip()
return not text or text == "\xa0"
# ---------------------------------------------------------------------------
# CSS class categorization for PK
# ---------------------------------------------------------------------------
# Document header
DOC_HEADER_CLASS = "p4" # "Prayers of King Kulaçekhara 1958 [Handwritten]"
TITLE_CLASS = "p115" # Bold title
# Chapter bookmark class
CHAPTER_BOOKMARK_CLASS = "p17" # "Prayers of King Kulaçekhara 1950: N"
# Verse number markers — skip
VERSE_MARKER_CLASSES = frozenset(["p108", "p9"])
# Devanagari marker — skip
DEVANAGARI_MARKER_CLASS = "p36"
# Section header class (Wordings., Synonyms., Translation., Purport.)
SECTION_HEADER_CLASSES = frozenset(["p20", "p121"])
# Body paragraph classes
BODY_CLASSES = frozenset(["p6", "p34", "p38", "p71"])
# Page number class — skip
PAGE_NUMBER_CLASSES = frozenset(["p15", "p41", "p124"])
# Empty/break classes — skip
SKIP_CLASSES = frozenset(["p24", "p94", "p119", "p122", "p123", "p120"])
# Next work boundary
NEXT_WORK_BOUNDARY_CLASS = "p125"
# ---------------------------------------------------------------------------
# Boundary detection
# ---------------------------------------------------------------------------
def find_pk_boundaries(all_elements: list) -> tuple:
"""Find the start and end element indices of the PK content.
Start: <p class="p4"> containing "Prayers of King Kula"
End: <p class="p125"> that starts the next work ("Shaktipujah")
"""
start_idx = None
end_idx = len(all_elements)
for i, el in enumerate(all_elements):
if el.name != "p":
continue
cls = get_css_class(el)
text = decode_text(el)
# Find start: p4 containing "Prayers of King"
if start_idx is None:
if cls == DOC_HEADER_CLASS and "Prayers of King" in text:
start_idx = i
continue
continue
# Find end: p125 that starts a different work
if cls == NEXT_WORK_BOUNDARY_CLASS:
end_idx = i
break
if start_idx is None:
print(
"ERROR: Could not find 'Prayers of King Kulaçekhara' in others-before-1977.html!"
)
sys.exit(1)
return start_idx, end_idx
def find_chapter_bookmarks(all_elements: list, start_idx: int, end_idx: int) -> list:
"""Find all PK chapter bookmark tags (p17) in the PK range.
Returns list of (element_index, verse_number).
"""
bookmarks = []
for i in range(start_idx, end_idx):
el = all_elements[i]
if el.name != "p":
continue
cls = get_css_class(el)
if cls == CHAPTER_BOOKMARK_CLASS:
text = decode_text(el).strip()
if "Prayers of King" in text or "Kulaçekhara" in text.replace("ç", "ç"):
# Extract the verse number from the end
match = re.search(r":\s*(\d+)\s*$", text)
if match:
verse_num = int(match.group(1))
bookmarks.append((i, verse_num))
return bookmarks
# ---------------------------------------------------------------------------
# Table synonym extraction
# ---------------------------------------------------------------------------
def extract_synonyms_from_table(table: Tag) -> str:
"""Extract synonyms from a <table class="t1"> and convert to text.
Each cell contains a <p> with "word=meaning" format.
Convert to "word—meaning; word—meaning; ..." format.
"""
entries = []
for row in table.find_all("tr"):
for cell in row.find_all("td"):
# Get all <p> tags inside the cell
for p in cell.find_all("p"):
text = decode_text(p).strip()
if not text or text == "\xa0":
continue
# Must contain '=' to be a synonym entry
if "=" not in text:
continue
# Replace = with em-dash
text = text.replace("=", "—", 1)
# Clean trailing whitespace/period if needed
text = text.strip()
entries.append(text)
if not entries:
return ""
return "; ".join(entries)
# ---------------------------------------------------------------------------
# Paragraph extraction for a chapter
# ---------------------------------------------------------------------------
def extract_chapter_paragraphs(
all_elements: list,
start_idx: int,
end_idx: int,
) -> list:
"""Extract paragraphs for a chapter (verse).
Handles section headers (Wordings., Synonyms., Translation., Purport.),
body paragraphs, and synonym tables.
Returns a list of HTML paragraph strings.
"""
result = []
# Build a set of <p> tags that are inside tables, so we don't double-process
table_p_tags = set()
for i in range(start_idx, end_idx):
el = all_elements[i]
if el.name == "table":
for p in el.find_all("p"):
table_p_tags.add(id(p))
for i in range(start_idx, end_idx):
el = all_elements[i]
# Handle tables (synonyms)
if el.name == "table":
table_cls = get_css_class(el)
if table_cls == "t1":
synonyms_text = extract_synonyms_from_table(el)
if synonyms_text:
result.append(synonyms_text)
continue
# Only process <p> tags from here
if el.name != "p":
continue
# Skip <p> tags that are inside tables (already processed)
if id(el) in table_p_tags:
continue
cls = get_css_class(el)
# Skip empty paragraphs
if is_empty(el):
continue
# Skip page numbers
if cls in PAGE_NUMBER_CLASSES:
continue
# Skip verse markers
if cls in VERSE_MARKER_CLASSES:
text = decode_text(el)
# p9 can also be "(Devanägaré)" or prayer number markers — skip all
continue
# Skip devanagari markers
if cls == DEVANAGARI_MARKER_CLASS:
continue
# Skip chapter bookmarks
if cls == CHAPTER_BOOKMARK_CLASS:
continue
# Skip document header and title (only at very start)
if cls == DOC_HEADER_CLASS:
continue
if cls == TITLE_CLASS:
continue
# Skip empty/break classes
if cls in SKIP_CLASSES:
continue
# Section headers: Wordings., Synonyms., Translation., Purport.
if cls in SECTION_HEADER_CLASSES:
text = decode_text(el).strip()
# Normalize section header names
text_lower = text.lower().rstrip(".")
if "wording" in text_lower:
result.append("<b>Wordings</b>")
elif (
"synonym" in text_lower
or "synomym" in text_lower
or "synonysm" in text_lower
):
result.append("<b>Synonyms</b>")
elif "translation" in text_lower:
result.append("<b>Translation</b>")
elif "purport" in text_lower:
result.append("<b>Purport</b>")
else:
# Unknown header — include as bold
html = decode_html_content(el)
if html:
result.append(f"<b>{html}</b>")
continue
# Body paragraphs
if cls in BODY_CLASSES:
html = decode_html_content(el)
if html:
result.append(html)
continue
# Fallback: try to extract content from unknown classes
html = decode_html_content(el)
if html:
result.append(html)
return result
# ---------------------------------------------------------------------------
# Main extraction logic
# ---------------------------------------------------------------------------
def extract_pk(soup: BeautifulSoup) -> dict:
"""Extract the entire Prayers of King Kulaśekhara from the parsed HTML.
Returns a dict with:
"chapters": list of chapter dicts
"chapter_listing": list of {number, title} for intro.json
"""
# We need both <p> and <table> elements in document order
all_elements = list(soup.find_all(["p", "table"]))
# Step 1: Find PK boundaries
pk_start, pk_end = find_pk_boundaries(all_elements)
print(f" PK content: elements {pk_start} to {pk_end}")
print(f" Total elements in range: {pk_end - pk_start}")
# Step 2: Find all chapter bookmarks (p17)
bookmarks = find_chapter_bookmarks(all_elements, pk_start, pk_end)
print(f" Found {len(bookmarks)} PK chapter bookmarks:")
for pos, verse_num in bookmarks:
print(f" Element {pos}: Verse {verse_num}")
# If we didn't find 6 bookmarks, also look for the first verse that starts
# before any p17 bookmark (verse 1 may start with p108 [1])
# Actually, verse 1's bookmark IS in the HTML. Let's verify.
if len(bookmarks) < 6:
print(f" WARNING: Expected 6 bookmarks, found {len(bookmarks)}")
# Step 3: Build chapter ranges
# Each bookmark marks the START of a verse section.
# Content for verse N runs from bookmark N to bookmark N+1 (or pk_end).
chapters = []
for idx, (bk_pos, verse_num) in enumerate(bookmarks):
# Content starts after the bookmark
content_start = bk_pos + 1
# Content ends at next bookmark or pk_end
if idx + 1 < len(bookmarks):
content_end = bookmarks[idx + 1][0]
else:
content_end = pk_end
paragraphs = extract_chapter_paragraphs(
all_elements, content_start, content_end
)
ch = {
"book": "pk",
"chapter": verse_num,
"title": f"Prayer {verse_num}",
"type": "chapter",
"paragraphs": paragraphs,
}
chapters.append(ch)
# Step 4: Build chapter listing for intro.json
chapter_listing = [
{"number": ch["chapter"], "title": ch["title"]} for ch in chapters
]
return {
"chapters": chapters,
"chapter_listing": chapter_listing,
}
# ---------------------------------------------------------------------------
# Output
# ---------------------------------------------------------------------------
def write_json(data, filepath):
"""Write data as formatted JSON to filepath, creating directories as needed."""
os.makedirs(os.path.dirname(filepath), exist_ok=True)
with open(filepath, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False, indent=2)
def main():
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html")
output_base = os.path.join(base_dir, "lib", "jsons", "books", "pk")
if not os.path.exists(html_path):
print(f"ERROR: {html_path} not found!")
sys.exit(1)
print(f"Reading {html_path}...")
with open(html_path, "r", encoding="utf-8") as f:
soup = BeautifulSoup(f, "html.parser")
print("Extracting Prayers of King Kulaśekhara (1958)...\n")
result = extract_pk(soup)
chapters = result["chapters"]
chapter_listing = result["chapter_listing"]
# --- Write intro.json ---
intro_meta = {
"book": "pk",
"title": "Prayers of King Kulaśekhara (1958)",
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
"year": 1958,
"description": "Early handwritten translations and purports of the Mukunda-mālā-stotra prayers (verses 1–6).",
"totalChapters": 6,
"chapters": chapter_listing,
}
intro_path = os.path.join(output_base, "intro.json")
write_json(intro_meta, intro_path)
print(f"\n Written: {intro_path}")
# --- Write chapters ---
print(f"\n--- Chapters ---")
for ch in chapters:
ch_num_str = str(ch["chapter"]).zfill(2)
filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
write_json(ch, filepath)
print(
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
f"{ch['title']}"
)
# --- Summary ---
total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
print(f"\n{'=' * 60}")
print(f" EXTRACTION SUMMARY")
print(f"{'=' * 60}")
print(f" Chapters: {len(chapters)}")
for ch in chapters:
print(
f" Ch {ch['chapter']}: {len(ch['paragraphs'])} paragraphs — {ch['title']}"
)
print(f" Total paragraphs: {total_chapter_paras}")
if chapters:
print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
print(f" Output directory: {output_base}/")
# --- Samples ---
for ch_idx, ch in enumerate(chapters):
print(f"\n--- Sample: Chapter {ch['chapter']} ({ch['title']}) ---")
for i, p in enumerate(ch["paragraphs"][:5]):
preview = re.sub(r"<[^>]+>", "", p)[:120]
print(f" [{i}] {preview}")
if len(ch["paragraphs"]) > 5:
print(f" ... ({len(ch['paragraphs'])} total paragraphs)")
# --- Quality Checks ---
print(f"\n--- Quality Checks ---")
# Check all 6 chapters present
expected_chapters = set(range(1, 7))
actual_chapters = set(ch["chapter"] for ch in chapters)
missing = expected_chapters - actual_chapters
extra = actual_chapters - expected_chapters
if missing:
print(f" ⚠ Missing chapters: {sorted(missing)}")
if extra:
print(f" ⚠ Extra chapters: {sorted(extra)}")
if not missing and not extra:
print(f" ✅ All 6 chapters present")
# Check empty chapters
empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
if empty_chapters:
print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
else:
print(f" ✅ No empty chapters")
# Check each chapter has expected sections
for ch in chapters:
ch_text = " ".join(ch["paragraphs"])
has_wordings = "<b>Wordings</b>" in ch_text
has_synonyms = "<b>Synonyms</b>" in ch_text
has_translation = "<b>Translation</b>" in ch_text
has_purport = "<b>Purport</b>" in ch_text
sections = []
if has_wordings:
sections.append("W")
if has_synonyms:
sections.append("S")
if has_translation:
sections.append("T")
if has_purport:
sections.append("P")
status = "✅" if len(sections) == 4 else "⚠"
print(f" {status} Ch {ch['chapter']}: sections [{', '.join(sections)}]")
# Check synonym entries use em-dash format
synonym_count = 0
for ch in chapters:
for p in ch["paragraphs"]:
if "—" in p and "<b>" not in p:
synonym_count += 1
break
print(f" ℹ Chapters with em-dash synonyms: {synonym_count}/6")
# Verify no Shaktipujah content leaked in
all_text = ""
for ch in chapters:
all_text += " ".join(ch["paragraphs"])
if "Shaktipujah" in all_text or "Baidya Nath" in all_text:
print(f" ⚠ Content from Shaktipujah may have leaked into PK!")
else:
print(f" ✅ No content leak from next work detected")
# Check last chapter ends cleanly
if chapters:
last_ch = chapters[-1]
last_para = last_ch["paragraphs"][-1] if last_ch["paragraphs"] else ""
last_preview = re.sub(r"<[^>]+>", "", last_para)[:80]
print(f' ℹ Last paragraph of Ch {last_ch["chapter"]}: "{last_preview}..."')
print(f"\nDone! ✨")
if __name__ == "__main__":
main()