feat: add journals
This commit is contained in:
gosangam
@@ -0,0 +1,444 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract Śrī Caitanya-bhāgavata Commentary from others-before-1977.html.
|
||||
|
||||
The book has 4 chapters (no intro segments):
|
||||
1. Chapter 1
|
||||
2. Lord Caitanya's Appearance
|
||||
3. Chapter 3
|
||||
4. Chapter 4
|
||||
|
||||
Source: public/html/others-before-1977.html, lines ~4139–4180.
|
||||
Start: <p class="p3"><b>Çré Caitanya-bhägavata-Commentary</b></p>
|
||||
End: <p class="p2">Bhagaväner kathä [Original Documents]</p> (stop before)
|
||||
|
||||
Chapter boundaries: <p class="p17"> tags containing "Caitanya-bhägavata"
|
||||
|
||||
CSS classes:
|
||||
p3 — Book title header (skip)
|
||||
p17 — Chapter bookmark headers (boundary detection, skip from output)
|
||||
p5 — Chapter title headings (skip — we derive titles ourselves)
|
||||
p6 — Body paragraphs (MAIN content)
|
||||
p38 — Body paragraph variant
|
||||
p55 — Body paragraph variant (last paragraph of chapter 4)
|
||||
p9 — End-of-chapter markers like "(End Chapter One)" — include
|
||||
|
||||
Output:
|
||||
lib/jsons/books/cbhc/intro.json — Book metadata
|
||||
lib/jsons/books/cbhc/chapters/01.json — Chapter 1
|
||||
lib/jsons/books/cbhc/chapters/02.json — Chapter 2
|
||||
lib/jsons/books/cbhc/chapters/03.json — Chapter 3
|
||||
lib/jsons/books/cbhc/chapters/04.json — Chapter 4
|
||||
|
||||
Usage:
|
||||
python3 scripts/cbhc_extract.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def decode_html_content(tag: Tag) -> str:
|
||||
"""Extract inner HTML from a tag, preserve formatting tags, strip spans."""
|
||||
inner = "".join(str(child) for child in tag.children)
|
||||
# Remove span tags with CSS classes (RTF conversion artifacts)
|
||||
inner = re.sub(
|
||||
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
|
||||
)
|
||||
# Clean up Apple-tab-span leftovers (tab characters)
|
||||
inner = inner.replace("\t", " ")
|
||||
# Normalize whitespace
|
||||
inner = re.sub(r"\s+", " ", inner).strip()
|
||||
return inner
|
||||
|
||||
|
||||
def decode_text(tag: Tag) -> str:
|
||||
"""Get plain text from a tag."""
|
||||
return tag.get_text().strip()
|
||||
|
||||
|
||||
def get_css_class(tag: Tag) -> str:
|
||||
"""Get the first CSS class of a tag, or empty string."""
|
||||
classes = tag.get("class", [])
|
||||
return classes[0] if classes else ""
|
||||
|
||||
|
||||
def is_empty(tag: Tag) -> bool:
|
||||
"""Check if a tag is empty or whitespace-only."""
|
||||
text = tag.get_text().strip()
|
||||
return not text or text == "\xa0"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CSS class categorization for CBHC
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Book title header
|
||||
BOOK_TITLE_CLASS = "p3"
|
||||
|
||||
# Chapter bookmark headers (used for boundary detection)
|
||||
CHAPTER_BOOKMARK_CLASS = "p17"
|
||||
|
||||
# Chapter title (bold, e.g., "Chapter 1") — skip from body
|
||||
CHAPTER_TITLE_CLASS = "p5"
|
||||
|
||||
# Body paragraph classes
|
||||
BODY_CLASSES = frozenset(["p6", "p38", "p55"])
|
||||
|
||||
# End-of-chapter markers — include as a paragraph
|
||||
END_MARKER_CLASS = "p9"
|
||||
|
||||
# Classes to skip entirely
|
||||
SKIP_CLASSES = frozenset(["p43", "p24"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Boundary detection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def find_cbhc_boundaries(all_paragraphs: list) -> tuple:
|
||||
"""Find the start and end paragraph indices of the CBHC content.
|
||||
|
||||
Start: <p class="p3"> containing "Caitanya-bhägavata-Commentary"
|
||||
End: <p class="p2"> containing "Bhagaväner kathä" (next work)
|
||||
"""
|
||||
start_idx = None
|
||||
end_idx = len(all_paragraphs)
|
||||
|
||||
for i, p in enumerate(all_paragraphs):
|
||||
cls = get_css_class(p)
|
||||
text = decode_text(p)
|
||||
|
||||
# Find start: p3 containing "Caitanya-bhägavata"
|
||||
if start_idx is None:
|
||||
if cls == BOOK_TITLE_CLASS and "Caitanya-bh" in text:
|
||||
start_idx = i
|
||||
continue
|
||||
continue
|
||||
|
||||
# Find end: p2 containing "Bhagaväner" (next work boundary)
|
||||
if cls == "p2" and "Bhag" in text and "kath" in text:
|
||||
end_idx = i
|
||||
break
|
||||
|
||||
if start_idx is None:
|
||||
print(
|
||||
"ERROR: Could not find 'Çré Caitanya-bhägavata-Commentary' "
|
||||
"in others-before-1977.html!"
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
return start_idx, end_idx
|
||||
|
||||
|
||||
def find_chapter_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
|
||||
"""Find all CBHC chapter bookmark tags (p17) in the CBHC range.
|
||||
|
||||
Returns list of (paragraph_index, raw_title_text).
|
||||
"""
|
||||
bookmarks = []
|
||||
for i in range(start_idx, end_idx):
|
||||
p = all_paragraphs[i]
|
||||
cls = get_css_class(p)
|
||||
if cls == CHAPTER_BOOKMARK_CLASS:
|
||||
text = decode_text(p).strip()
|
||||
if text and "Caitanya-bh" in text:
|
||||
bookmarks.append((i, text))
|
||||
return bookmarks
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Paragraph extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def extract_paragraphs(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
|
||||
"""Extract body paragraphs for a chapter section.
|
||||
|
||||
Collects p6, p38, p55, and p9 (end markers).
|
||||
Skips p3 (book header), p17 (bookmarks), p5 (chapter titles), p2, empties.
|
||||
"""
|
||||
result = []
|
||||
|
||||
for i in range(start_idx, end_idx):
|
||||
p = all_paragraphs[i]
|
||||
cls = get_css_class(p)
|
||||
|
||||
# Skip empty paragraphs
|
||||
if is_empty(p):
|
||||
continue
|
||||
|
||||
# Skip known skip classes
|
||||
if cls in SKIP_CLASSES:
|
||||
continue
|
||||
|
||||
# Skip book title header
|
||||
if cls == BOOK_TITLE_CLASS:
|
||||
continue
|
||||
|
||||
# Skip chapter bookmark headers
|
||||
if cls == CHAPTER_BOOKMARK_CLASS:
|
||||
continue
|
||||
|
||||
# Skip chapter title headings (p5)
|
||||
if cls == CHAPTER_TITLE_CLASS:
|
||||
continue
|
||||
|
||||
# Skip p2 (TOC entries)
|
||||
if cls == "p2":
|
||||
continue
|
||||
|
||||
# Body paragraphs
|
||||
if cls in BODY_CLASSES:
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
result.append(html)
|
||||
continue
|
||||
|
||||
# End-of-chapter markers (p9) — include as a paragraph
|
||||
if cls == END_MARKER_CLASS:
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
result.append(html)
|
||||
continue
|
||||
|
||||
# Fallback: try to extract content from unknown classes
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
print(f" ⚠ Unknown class '{cls}' at paragraph {i}: {html[:80]}...")
|
||||
result.append(html)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main extraction logic
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def extract_cbhc(soup: BeautifulSoup) -> dict:
|
||||
"""Extract the entire Śrī Caitanya-bhāgavata Commentary from the parsed HTML.
|
||||
|
||||
Returns a dict with:
|
||||
"chapters": list of chapter dicts
|
||||
"chapter_listing": list of {number, title} for intro.json
|
||||
"""
|
||||
all_paragraphs = list(soup.find_all("p"))
|
||||
|
||||
# Step 1: Find CBHC boundaries
|
||||
cbhc_start, cbhc_end = find_cbhc_boundaries(all_paragraphs)
|
||||
print(f" CBHC content: paragraphs {cbhc_start} to {cbhc_end}")
|
||||
print(f" Total paragraphs in range: {cbhc_end - cbhc_start}")
|
||||
|
||||
# Step 2: Find all chapter bookmarks (p17)
|
||||
bookmarks = find_chapter_bookmarks(all_paragraphs, cbhc_start, cbhc_end)
|
||||
print(f" Found {len(bookmarks)} CBHC chapter bookmarks:")
|
||||
for pos, title in bookmarks:
|
||||
print(f" Paragraph {pos}: {title}")
|
||||
|
||||
# Step 3: Map bookmarks to chapters with proper titles
|
||||
# Expected: 4 chapters
|
||||
# We derive proper titles from the p5 heading that follows each bookmark
|
||||
chapter_titles_map = {
|
||||
1: "Chapter 1",
|
||||
2: "Lord Caitanya's Appearance",
|
||||
3: "Chapter 3",
|
||||
4: "Chapter 4",
|
||||
}
|
||||
|
||||
chapter_bookmarks = [] # (para_idx, chapter_num, normalized_title)
|
||||
for idx, (pos, _raw_title) in enumerate(bookmarks):
|
||||
ch_num = idx + 1
|
||||
ch_title = chapter_titles_map.get(ch_num, f"Chapter {ch_num}")
|
||||
chapter_bookmarks.append((pos, ch_num, ch_title))
|
||||
|
||||
print(f"\n Mapped {len(chapter_bookmarks)} chapters:")
|
||||
for pos, num, title in chapter_bookmarks:
|
||||
print(f" Chapter {num}: {title} (paragraph {pos})")
|
||||
|
||||
# Step 4: Extract chapters
|
||||
chapters = []
|
||||
for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_bookmarks):
|
||||
# Content starts after the bookmark line (skip p17 and p5 heading)
|
||||
content_start = ch_pos + 1
|
||||
|
||||
# Content ends at next chapter bookmark or cbhc_end
|
||||
if idx + 1 < len(chapter_bookmarks):
|
||||
content_end = chapter_bookmarks[idx + 1][0]
|
||||
else:
|
||||
content_end = cbhc_end
|
||||
|
||||
paragraphs = extract_paragraphs(all_paragraphs, content_start, content_end)
|
||||
|
||||
ch = {
|
||||
"book": "cbhc",
|
||||
"chapter": ch_num,
|
||||
"title": ch_title,
|
||||
"type": "chapter",
|
||||
"paragraphs": paragraphs,
|
||||
}
|
||||
chapters.append(ch)
|
||||
|
||||
# Step 5: Build chapter listing for intro.json
|
||||
chapter_listing = [
|
||||
{"number": ch["chapter"], "title": ch["title"]} for ch in chapters
|
||||
]
|
||||
|
||||
return {
|
||||
"chapters": chapters,
|
||||
"chapter_listing": chapter_listing,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Output
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def write_json(data, filepath):
|
||||
"""Write data as formatted JSON to filepath, creating directories as needed."""
|
||||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||||
with open(filepath, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html")
|
||||
output_base = os.path.join(base_dir, "lib", "jsons", "books", "cbhc")
|
||||
|
||||
if not os.path.exists(html_path):
|
||||
print(f"ERROR: {html_path} not found!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Reading {html_path}...")
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
soup = BeautifulSoup(f, "html.parser")
|
||||
|
||||
print("Extracting Śrī Caitanya-bhāgavata Commentary...\n")
|
||||
result = extract_cbhc(soup)
|
||||
|
||||
chapters = result["chapters"]
|
||||
chapter_listing = result["chapter_listing"]
|
||||
|
||||
# --- Write intro.json ---
|
||||
intro_meta = {
|
||||
"book": "cbhc",
|
||||
"title": "Śrī Caitanya-bhāgavata Commentary",
|
||||
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
||||
"year": None,
|
||||
"description": (
|
||||
"Commentary on Śrī Caitanya-bhāgavata by Vṛndāvana dāsa Ṭhākura "
|
||||
"— covering the first four chapters of Lord Caitanya's pastimes."
|
||||
),
|
||||
"totalChapters": 4,
|
||||
"chapters": chapter_listing,
|
||||
}
|
||||
intro_path = os.path.join(output_base, "intro.json")
|
||||
write_json(intro_meta, intro_path)
|
||||
print(f"\n Written: {intro_path}")
|
||||
|
||||
# --- Write chapters ---
|
||||
print(f"\n--- Chapters ---")
|
||||
for ch in chapters:
|
||||
ch_num_str = str(ch["chapter"]).zfill(2)
|
||||
filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
|
||||
write_json(ch, filepath)
|
||||
print(
|
||||
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
|
||||
f"{ch['title']}"
|
||||
)
|
||||
|
||||
# --- Summary ---
|
||||
total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
|
||||
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f" EXTRACTION SUMMARY")
|
||||
print(f"{'=' * 60}")
|
||||
print(f" Chapters: {len(chapters)}")
|
||||
for ch in chapters:
|
||||
print(
|
||||
f" Ch {ch['chapter']}: {len(ch['paragraphs'])} paragraphs — {ch['title']}"
|
||||
)
|
||||
print(f" Total paragraphs: {total_chapter_paras}")
|
||||
if chapters:
|
||||
print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
|
||||
print(f" Output directory: {output_base}/")
|
||||
|
||||
# --- Samples ---
|
||||
for ch in chapters:
|
||||
ch_num = ch["chapter"]
|
||||
print(f"\n--- Sample: Chapter {ch_num} ({ch['title']}) ---")
|
||||
for i, p in enumerate(ch["paragraphs"][:3]):
|
||||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
||||
print(f" [{i}] {preview}...")
|
||||
if len(ch["paragraphs"]) > 3:
|
||||
print(f" ... ({len(ch['paragraphs']) - 3} more paragraphs)")
|
||||
|
||||
# --- Quality Checks ---
|
||||
print(f"\n--- Quality Checks ---")
|
||||
|
||||
# Check all 4 chapters present
|
||||
expected_chapters = set(range(1, 5))
|
||||
actual_chapters = set(ch["chapter"] for ch in chapters)
|
||||
missing = expected_chapters - actual_chapters
|
||||
extra = actual_chapters - expected_chapters
|
||||
if missing:
|
||||
print(f" ⚠ Missing chapters: {sorted(missing)}")
|
||||
if extra:
|
||||
print(f" ⚠ Extra chapters: {sorted(extra)}")
|
||||
if not missing and not extra:
|
||||
print(f" ✅ All 4 chapters present")
|
||||
|
||||
# Check empty chapters
|
||||
empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
|
||||
if empty_chapters:
|
||||
print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
|
||||
else:
|
||||
print(f" ✅ No empty chapters")
|
||||
|
||||
# Check chapter 1 has "(End Chapter One)"
|
||||
if chapters:
|
||||
ch1_text = " ".join(chapters[0]["paragraphs"])
|
||||
if "End Chapter One" in ch1_text:
|
||||
print(f" ✅ '(End Chapter One)' end marker found in Chapter 1")
|
||||
else:
|
||||
print(f" ⚠ '(End Chapter One)' end marker NOT found in Chapter 1")
|
||||
|
||||
# Check chapter 2 mentions Lord Caitanya's appearance
|
||||
if len(chapters) >= 2:
|
||||
ch2_text = " ".join(chapters[1]["paragraphs"])
|
||||
if "appearance" in ch2_text.lower() or "born" in ch2_text.lower():
|
||||
print(f" ✅ Chapter 2 contains appearance/birth content")
|
||||
else:
|
||||
print(f" ⚠ Chapter 2 may not contain appearance content")
|
||||
|
||||
# Check for content leak from next work (Bhagaväner kathä)
|
||||
all_text = ""
|
||||
for ch in chapters:
|
||||
all_text += " ".join(ch["paragraphs"])
|
||||
if "Original Documents" in all_text:
|
||||
print(f" ⚠ Content from next work (Bhagaväner kathä) may have leaked!")
|
||||
else:
|
||||
print(f" ✅ No content leak from next work detected")
|
||||
|
||||
# Check last chapter ends cleanly
|
||||
if chapters:
|
||||
last_ch = chapters[-1]
|
||||
last_para = last_ch["paragraphs"][-1] if last_ch["paragraphs"] else ""
|
||||
last_preview = re.sub(r"<[^>]+>", "", last_para)[:100]
|
||||
print(f' ℹ Last paragraph of Ch {last_ch["chapter"]}: "{last_preview}..."')
|
||||
|
||||
print(f"\nDone! ✨")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user