krispire/spsays-training
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add journals

Browse Source
This commit is contained in:
gosangam
2026-05-03 21:02:41 +05:30
parent 89106efc98
commit 35d1d14f37
47 changed files with 11365 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/mog_extract.py
+539
View File
@@ -0,0 +1,539 @@
#!/usr/bin/env python3
"""Extract Message of Godhead (1955) from others-before-1977.html.
The book has 1 intro segment (front-pages) and 3 chapters:
1. Message of Godhead (Introductory)
2. Knowledge Transcendental
3. Work with Transcendental Effect (Karma Yoga)
Output:
lib/jsons/books/mog/intro.json — Book metadata
lib/jsons/books/mog/intro/*.json — Intro segments (front-pages)
lib/jsons/books/mog/chapters/*.json — Chapters (01–03)
Usage:
python3 scripts/mog_extract.py
"""
import json
import os
import re
import sys
from bs4 import BeautifulSoup, Tag
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def decode_html_content(tag: Tag) -> str:
"""Extract inner HTML from a tag, preserve formatting tags, strip spans."""
inner = "".join(str(child) for child in tag.children)
# Remove span tags with CSS classes (RTF conversion artifacts)
inner = re.sub(
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
)
# Clean up Apple-tab-span leftovers (tab characters)
inner = inner.replace("\t", " ")
# Normalize whitespace
inner = re.sub(r"\s+", " ", inner).strip()
return inner
def decode_text(tag: Tag) -> str:
"""Get plain text from a tag."""
return tag.get_text().strip()
def get_css_class(tag: Tag) -> str:
"""Get the first CSS class of a tag, or empty string."""
classes = tag.get("class", [])
return classes[0] if classes else ""
def is_empty(tag: Tag) -> bool:
"""Check if a tag is empty or whitespace-only."""
text = tag.get_text().strip()
return not text or text == "\xa0"
# ---------------------------------------------------------------------------
# CSS class categorization for MOG
# ---------------------------------------------------------------------------
# Front page classes
FRONT_TITLE_CLASS = "p3" # Book title header (bold)
FRONT_INFO_CLASS = "p19" # Author/address info
FRONT_DATE_CLASS = "p15" # Date line ("From the 30/8/55")
# Chapter heading classes
CHAPTER_BOOKMARK_CLASS = (
"p11" # Chapter bookmark headers (like "MESSAGE OF GODHEAD (Introductory)")
)
CHAPTER_TITLE_CLASS = "p5" # Chapter title (bold, duplicate of p11 — skip)
# Body paragraph classes
BODY_CLASSES = frozenset(["p6", "p38", "p44", "p45", "p46", "p47"])
# Classes to skip entirely (empty/decorative/break)
SKIP_CLASSES = frozenset(["p43", "p24"])
# Boundary detection: classes that mark the start of a different work after MOG
NEXT_WORK_BOUNDARY_CLASSES = frozenset(["p2", "p3"])
# ---------------------------------------------------------------------------
# Boundary detection
# ---------------------------------------------------------------------------
def find_mog_boundaries(all_paragraphs: list) -> tuple:
"""Find the start and end paragraph indices of the MOG content.
Start: <p class="p3"> containing "Message of Godhead"
End: next <p class="p2"> or <p class="p3"> that starts a different book,
or <p class="p6"> containing "Bhakti-rasāmṛta-sindhu" (the next work)
"""
start_idx = None
end_idx = len(all_paragraphs)
for i, p in enumerate(all_paragraphs):
cls = get_css_class(p)
text = decode_text(p)
# Find start: p3 containing "Message of Godhead"
if start_idx is None:
if cls == "p2" and "Message of Godhead" in text:
# The p2 is the TOC entry; the actual start is the next p3
start_idx = i
continue
continue
# Find end: look for the next work boundary
# The p6 containing "Bhakti-rasāmṛta-sindhu" starts a different work
if cls == "p6" and "Bhakti-ras" in text and "sindhu" in text:
end_idx = i
break
# Also check for p2 that starts a different book section
if cls == "p2" and start_idx is not None and i > start_idx + 5:
text_lower = text.lower()
if "message of godhead" not in text_lower:
end_idx = i
break
if start_idx is None:
print("ERROR: Could not find 'Message of Godhead' in others-before-1977.html!")
sys.exit(1)
return start_idx, end_idx
def find_chapter_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
"""Find all MOG chapter bookmark tags (p11) in the MOG range.
Returns list of (paragraph_index, chapter_title).
"""
bookmarks = []
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
if cls == CHAPTER_BOOKMARK_CLASS:
text = decode_text(p).strip()
if text:
bookmarks.append((i, text))
return bookmarks
# ---------------------------------------------------------------------------
# Paragraph extraction
# ---------------------------------------------------------------------------
def extract_paragraphs(
all_paragraphs: list,
start_idx: int,
end_idx: int,
is_front_pages: bool = False,
) -> list:
"""Extract paragraphs for a section (front pages or chapter body).
For front pages: collect p3, p19, p15 lines as paragraphs.
For chapters: collect body paragraphs (p6, p38, p44, p45, p46, p47).
Always skip p43, p24 (empty/break), p11 (bookmark), p5 (duplicate heading).
"""
result = []
for i in range(start_idx, end_idx):
p = all_paragraphs[i]
cls = get_css_class(p)
# Skip empty paragraphs
if is_empty(p):
continue
# Skip known empty/break classes
if cls in SKIP_CLASSES:
continue
# Skip chapter bookmark headers (p11) — used for boundary detection only
if cls == CHAPTER_BOOKMARK_CLASS:
continue
# Skip chapter title duplicate (p5)
if cls == CHAPTER_TITLE_CLASS:
continue
# Skip p2 (TOC entry at start)
if cls == "p2":
continue
if is_front_pages:
# For front pages, collect title, author/address, and date lines
if cls in (FRONT_TITLE_CLASS, FRONT_INFO_CLASS, FRONT_DATE_CLASS):
html = decode_html_content(p)
if html:
result.append(html)
continue
# Also include any other non-skip class if we encounter it
# (shouldn't happen much for front pages, but be permissive)
if cls in BODY_CLASSES:
# Body paragraph before first chapter = still front pages
html = decode_html_content(p)
if html:
result.append(html)
continue
else:
# For chapter body, collect all body paragraph classes
if cls in BODY_CLASSES:
html = decode_html_content(p)
if html:
result.append(html)
continue
# p19 can appear inside chapters (e.g., "(Devanāgarī)" in chapter 2)
if cls == FRONT_INFO_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Title page class inside chapter body (shouldn't happen, but be safe)
if cls == FRONT_TITLE_CLASS:
html = decode_html_content(p)
if html:
result.append(html)
continue
# Fallback: try to extract content from unknown classes
html = decode_html_content(p)
if html:
result.append(html)
return result
# ---------------------------------------------------------------------------
# Main extraction logic
# ---------------------------------------------------------------------------
def extract_mog(soup: BeautifulSoup) -> dict:
"""Extract the entire Message of Godhead from the parsed HTML.
Returns a dict with:
"intro_segments": list of intro segment dicts
"chapters": list of chapter dicts
"chapter_listing": list of {number, title} for intro.json
"""
all_paragraphs = list(soup.find_all("p"))
# Step 1: Find MOG boundaries
mog_start, mog_end = find_mog_boundaries(all_paragraphs)
print(f" MOG content: paragraphs {mog_start} to {mog_end}")
print(f" Total paragraphs in range: {mog_end - mog_start}")
# Step 2: Find all chapter bookmarks (p11)
bookmarks = find_chapter_bookmarks(all_paragraphs, mog_start, mog_end)
print(f" Found {len(bookmarks)} MOG chapter bookmarks:")
for pos, title in bookmarks:
print(f" Paragraph {pos}: {title}")
# Step 3: Map bookmarks to chapters with proper titles
# Expected: 3 chapters
# Map raw bookmark text (case-insensitive) to normalized titles
chapter_titles = {
"message of godhead": "Message of Godhead (Introductory)",
"knowledge transcendental": "Knowledge Transcendental",
"work with transcendental effect": "Work with Transcendental Effect (Karma Yoga)",
}
chapter_bookmarks = [] # (para_idx, chapter_num, normalized_title)
ch_num = 1
for pos, raw_title in bookmarks:
# Try to match known chapter titles (case-insensitive)
matched_title = None
raw_lower = raw_title.lower().strip()
for key, nice_title in chapter_titles.items():
if key in raw_lower:
matched_title = nice_title
break
if matched_title is None:
# Fallback: use the raw title with title case
matched_title = raw_title.strip()
chapter_bookmarks.append((pos, ch_num, matched_title))
ch_num += 1
print(f"\n Mapped {len(chapter_bookmarks)} chapters:")
for pos, num, title in chapter_bookmarks:
print(f" Chapter {num}: {title} (paragraph {pos})")
# Step 4: Extract front pages
# From MOG start to first chapter bookmark
intro_segments = []
if chapter_bookmarks:
front_pages_end = chapter_bookmarks[0][0]
else:
front_pages_end = mog_end
front_pages_paragraphs = extract_paragraphs(
all_paragraphs, mog_start, front_pages_end, is_front_pages=True
)
intro_segments.append(
{
"slug": "front-pages",
"title": "Front Pages",
"type": "intro-segment",
"book": "mog",
"order": 1,
"paragraphs": front_pages_paragraphs,
}
)
# Step 5: Extract chapters
chapters = []
for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_bookmarks):
# Content starts after the bookmark line
# But we also need to skip the p5 duplicate heading that follows p11
content_start = ch_pos + 1
# Content ends at next chapter bookmark or mog_end
if idx + 1 < len(chapter_bookmarks):
content_end = chapter_bookmarks[idx + 1][0]
else:
content_end = mog_end
paragraphs = extract_paragraphs(all_paragraphs, content_start, content_end)
ch = {
"book": "mog",
"chapter": ch_num,
"title": ch_title,
"type": "chapter",
"paragraphs": paragraphs,
}
chapters.append(ch)
# Step 6: Build chapter listing for intro.json
chapter_listing = [
{"number": ch["chapter"], "title": ch["title"]} for ch in chapters
]
return {
"intro_segments": intro_segments,
"chapters": chapters,
"chapter_listing": chapter_listing,
}
# ---------------------------------------------------------------------------
# Output
# ---------------------------------------------------------------------------
def write_json(data, filepath):
"""Write data as formatted JSON to filepath, creating directories as needed."""
os.makedirs(os.path.dirname(filepath), exist_ok=True)
with open(filepath, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False, indent=2)
def main():
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html")
output_base = os.path.join(base_dir, "lib", "jsons", "books", "mog")
if not os.path.exists(html_path):
print(f"ERROR: {html_path} not found!")
sys.exit(1)
print(f"Reading {html_path}...")
with open(html_path, "r", encoding="utf-8") as f:
soup = BeautifulSoup(f, "html.parser")
print("Extracting Message of Godhead (1955)...\n")
result = extract_mog(soup)
intro_segments = result["intro_segments"]
chapters = result["chapters"]
chapter_listing = result["chapter_listing"]
# --- Write intro.json ---
intro_meta = {
"book": "mog",
"title": "Message of Godhead",
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
"year": 1955,
"description": "An early handwritten work presenting the essence of Bhagavad-gītā's teachings on transcendental knowledge, karma-yoga, and the Personality of Godhead.",
"totalChapters": 3,
"chapters": chapter_listing,
}
intro_path = os.path.join(output_base, "intro.json")
write_json(intro_meta, intro_path)
print(f"\n Written: {intro_path}")
# --- Write intro segments ---
print(f"\n--- Intro Segments ---")
for seg in intro_segments:
filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
write_json(seg, filepath)
print(
f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}"
)
# --- Write chapters ---
print(f"\n--- Chapters ---")
for ch in chapters:
ch_num_str = str(ch["chapter"]).zfill(2)
filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
write_json(ch, filepath)
print(
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
f"{ch['title']}"
)
# --- Summary ---
total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments)
total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
total_paras = total_intro_paras + total_chapter_paras
print(f"\n{'=' * 60}")
print(f" EXTRACTION SUMMARY")
print(f"{'=' * 60}")
print(f" Intro segments: {len(intro_segments)}")
for seg in intro_segments:
print(
f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs"
)
print(f" Chapters: {len(chapters)}")
for ch in chapters:
print(
f" Ch {ch['chapter']}: {len(ch['paragraphs'])} paragraphs — {ch['title']}"
)
print(f" Total paragraphs: {total_paras}")
print(f" Intro: {total_intro_paras}")
print(f" Chapters: {total_chapter_paras}")
if chapters:
print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
print(f" Output directory: {output_base}/")
# --- Samples ---
print(f"\n--- Sample: Front Pages ---")
if intro_segments:
front = intro_segments[0]
for i, p in enumerate(front["paragraphs"][:5]):
preview = re.sub(r"<[^>]+>", "", p)[:120]
print(f" [{i}] {preview}")
if chapters:
print(f"\n--- Sample: Chapter 1 ({chapters[0]['title']}) ---")
ch1 = chapters[0]
for i, p in enumerate(ch1["paragraphs"][:3]):
preview = re.sub(r"<[^>]+>", "", p)[:120]
print(f" [{i}] {preview}...")
if len(chapters) >= 2:
print(f"\n--- Sample: Chapter 2 ({chapters[1]['title']}) ---")
ch2 = chapters[1]
for i, p in enumerate(ch2["paragraphs"][:3]):
preview = re.sub(r"<[^>]+>", "", p)[:120]
print(f" [{i}] {preview}...")
if len(chapters) >= 3:
print(f"\n--- Sample: Chapter 3 ({chapters[2]['title']}) ---")
ch3 = chapters[2]
for i, p in enumerate(ch3["paragraphs"][:3]):
preview = re.sub(r"<[^>]+>", "", p)[:120]
print(f" [{i}] {preview}...")
# --- Quality Checks ---
print(f"\n--- Quality Checks ---")
# Check all 3 chapters present
expected_chapters = set(range(1, 4))
actual_chapters = set(ch["chapter"] for ch in chapters)
missing = expected_chapters - actual_chapters
extra = actual_chapters - expected_chapters
if missing:
print(f" ⚠ Missing chapters: {sorted(missing)}")
if extra:
print(f" ⚠ Extra chapters: {sorted(extra)}")
if not missing and not extra:
print(f" ✅ All 3 chapters present")
# Check intro segments
if intro_segments:
print(
f" ✅ Front pages segment present ({len(intro_segments[0]['paragraphs'])} paragraphs)"
)
else:
print(f" ⚠ Front pages segment missing!")
# Check empty chapters
empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
if empty_chapters:
print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
else:
print(f" ✅ No empty chapters")
# Check for "——The Author." in chapter 1
if chapters:
ch1_text = " ".join(chapters[0]["paragraphs"])
if "The Author" in ch1_text:
print(f" ✅ '——The Author.' attribution found in Chapter 1")
else:
print(f" ⚠ '——The Author.' attribution NOT found in Chapter 1")
# Check chapter 2 has "(Devanāgarī)" line
if len(chapters) >= 2:
ch2_text = " ".join(chapters[1]["paragraphs"])
if "Devan" in ch2_text:
print(f" ✅ '(Devanāgarī)' line found in Chapter 2")
else:
print(f" ℹ '(Devanāgarī)' line not found in Chapter 2 (may be normal)")
# Verify no Bhakti-rasāmṛta-sindhu content leaked in
all_text = ""
for ch in chapters:
all_text += " ".join(ch["paragraphs"])
if "Text 22" in all_text or "BHAKTI RASAMRITA" in all_text.upper():
print(f" ⚠ Content from Bhakti-rasāmṛta-sindhu may have leaked into MOG!")
else:
print(f" ✅ No content leak from next work detected")
# Check last chapter ends cleanly
if chapters:
last_ch = chapters[-1]
last_para = last_ch["paragraphs"][-1] if last_ch["paragraphs"] else ""
last_preview = re.sub(r"<[^>]+>", "", last_para)[:80]
print(f' ℹ Last paragraph of Ch {last_ch["chapter"]}: "{last_preview}..."')
print(f"\nDone! ✨")
if __name__ == "__main__":
main()