feat: add journals

2026-05-03 21:02:41 +05:30
parent 89106efc98
commit 35d1d14f37
47 changed files with 11365 additions and 37 deletions
@@ -0,0 +1,590 @@
+#!/usr/bin/env python3
+"""Extract Bhagavāner Kathā (part of Renunciation through Wisdom) from others-before-1977.html.
+
+The book has 2 chapters (no intro segments):
+  1. This World of Misery
+  2. The Cause of Misery
+
+Source: public/html/others-before-1977.html, lines ~4182–4289.
+  Start: <p class="p56"> containing "Bhagaväner kathä"
+  End:   <p class="p35"> containing "Indian Bhagavatams" (stop before)
+
+Chapter boundaries: <p class="p17"> tags containing "Bk:Chapter"
+
+IMPORTANT: Unlike MOG/CBHC, this book's body text uses ScaGoudy font encoding.
+Characters like ä, å, ñ, ö, ë, ò, ç, ì, ï, ü, é, à, ù must be decoded to
+proper IAST Unicode (ā, ṛ, ṣ, ṭ, ṇ, ḍ, ś, ṅ, ñ, ū, ī, ṁ, ḥ).
+
+CSS classes:
+  p56 — Book title header (skip)
+  p17 — Chapter bookmark headers ("Bk:Chapter N") — boundary detection, skip
+  p5  — Chapter title headings (skip — we derive titles ourselves)
+  p6  — Body paragraphs (MAIN content) — HAS ScaGoudy encoding
+  p12 — Body paragraph variant (longer text) — HAS ScaGoudy encoding
+  p38 — Body paragraph variant — HAS ScaGoudy encoding
+  p9  — Sanskrit verse lines (italic, with <span class="s3">) — centered italic
+  p10 — Reference markers like [RTW 1.1] — include as paragraphs
+  p61 — Reference marker variant [RTW 1.2] — include
+  p57 — Footnote header ("Footnote") — include
+  p58 — Footnote text — include
+  p19 — Sanskrit verse block (multi-line) — centered italic
+  p59 — Sanskrit verse block variant — centered italic
+  p23 — Sanskrit verse line variant — centered italic
+  p36 — Sanskrit verse line variant — centered italic
+  p60 — Sanskrit verse line variant — centered italic
+
+Output:
+    lib/jsons/books/bk/intro.json          — Book metadata
+    lib/jsons/books/bk/chapters/01.json    — Chapter 1: This World of Misery
+    lib/jsons/books/bk/chapters/02.json    — Chapter 2: The Cause of Misery
+
+Usage:
+    python3 scripts/bk_extract.py
+"""
+
+import json
+import os
+import re
+import sys
+
+from bs4 import BeautifulSoup, Tag
+
+# ---------------------------------------------------------------------------
+# ScaGoudy → IAST Unicode mapping
+# ---------------------------------------------------------------------------
+
+SCAGOUDY_TO_IAST = {
+    "ä": "ā",
+    "Ä": "Ā",
+    "å": "ṛ",
+    "Å": "Ṛ",
+    "ñ": "ṣ",
+    "Ñ": "Ṣ",
+    "ö": "ṭ",
+    "Ö": "Ṭ",
+    "ë": "ṇ",
+    "Ë": "Ṇ",
+    "ò": "ḍ",
+    "Ò": "Ḍ",
+    "ç": "ś",
+    "Ç": "Ś",
+    "ì": "ṅ",
+    "Ì": "Ṅ",
+    "ï": "ñ",
+    "Ï": "Ñ",
+    "ü": "ū",
+    "Ü": "Ū",
+    "é": "ī",
+    "É": "Ī",
+    "à": "ṁ",
+    "À": "Ṁ",
+    "ù": "ḥ",
+    "Ù": "Ḥ",
+    "è": "ṝ",
+    "û": "ṁl",
+    "ÿ": "ḷ",
+}
+
+# Build a translation table for fast character-by-character replacement
+_SCAGOUDY_TRANS = str.maketrans(
+    {k: v for k, v in SCAGOUDY_TO_IAST.items() if len(k) == 1}
+)
+
+
+def scagoudy_decode(text: str) -> str:
+    """Decode ScaGoudy-encoded text to proper IAST Unicode."""
+    return text.translate(_SCAGOUDY_TRANS)
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def decode_html_content(tag: Tag, apply_scagoudy: bool = True) -> str:
+    """Extract inner HTML from a tag, preserve formatting tags, strip spans,
+    and optionally apply ScaGoudy decoding.
+
+    Steps:
+      1. Get inner HTML
+      2. Strip <span class="sN"> wrappers (keep content)
+      3. Apply ScaGoudy decode to the entire result (if requested)
+      4. Normalize whitespace
+    """
+    inner = "".join(str(child) for child in tag.children)
+    # Remove span tags with CSS classes (RTF conversion artifacts) — keep content
+    inner = re.sub(
+        r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
+    )
+    # Clean up Apple-tab-span leftovers (tab characters)
+    inner = inner.replace("\t", " ")
+    # Normalize whitespace
+    inner = re.sub(r"\s+", " ", inner).strip()
+    # Apply ScaGoudy decoding
+    if apply_scagoudy:
+        inner = scagoudy_decode(inner)
+    return inner
+
+
+def decode_html_content_as_verse(tag: Tag) -> str:
+    """Extract inner HTML, strip spans, ScaGoudy decode, wrap as centered italic.
+
+    For Sanskrit verse lines (p9, p19, p59, p23, p36, p60).
+    Preserves <br> and reference tags like [Bg. 7.14].
+    """
+    inner = "".join(str(child) for child in tag.children)
+    # Remove span tags with CSS classes — keep content
+    inner = re.sub(
+        r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
+    )
+    # Clean up tab characters
+    inner = inner.replace("\t", " ")
+    # Normalize spaces (but preserve <br> tags)
+    # Replace multiple spaces with single space
+    inner = re.sub(r"[ ]+", " ", inner).strip()
+    # Remove trailing <br> tags
+    inner = re.sub(r"(<br\s*/?\s*>)+\s*$", "", inner).strip()
+    # Apply ScaGoudy decoding
+    inner = scagoudy_decode(inner)
+    # Wrap as centered italic
+    if inner:
+        return f'<div style="text-align:center"><i>{inner}</i></div>'
+    return ""
+
+
+def decode_text(tag: Tag) -> str:
+    """Get plain text from a tag."""
+    return tag.get_text().strip()
+
+
+def get_css_class(tag: Tag) -> str:
+    """Get the first CSS class of a tag, or empty string."""
+    classes = tag.get("class", [])
+    return classes[0] if classes else ""
+
+
+def is_empty(tag: Tag) -> bool:
+    """Check if a tag is empty or whitespace-only."""
+    text = tag.get_text().strip()
+    return not text or text == "\xa0"
+
+
+# ---------------------------------------------------------------------------
+# CSS class categorization for BK
+# ---------------------------------------------------------------------------
+
+# Book title header
+BOOK_TITLE_CLASS = "p56"
+
+# Chapter bookmark headers (used for boundary detection)
+CHAPTER_BOOKMARK_CLASS = "p17"
+
+# Chapter title (bold) — skip from body
+CHAPTER_TITLE_CLASS = "p5"
+
+# Body paragraph classes (all need ScaGoudy decoding)
+BODY_CLASSES = frozenset(["p6", "p12", "p38"])
+
+# Sanskrit verse line classes (wrap as centered italic)
+VERSE_CLASSES = frozenset(["p9", "p19", "p59", "p23", "p36", "p60"])
+
+# Reference marker classes — include as paragraphs
+REFERENCE_CLASSES = frozenset(["p10", "p61"])
+
+# Footnote classes — include as paragraphs
+FOOTNOTE_CLASSES = frozenset(["p57", "p58"])
+
+# Classes to skip entirely
+SKIP_CLASSES = frozenset(["p43", "p24"])
+
+# TOC entry class
+TOC_CLASS = "p2"
+
+
+# ---------------------------------------------------------------------------
+# Boundary detection
+# ---------------------------------------------------------------------------
+
+
+def find_bk_boundaries(all_paragraphs: list) -> tuple:
+    """Find the start and end paragraph indices of the BK content.
+
+    Start: <p class="p56"> containing "Bhagaväner kathä"
+    End:   <p class="p35"> containing "Indian Bhagavatams" (stop before)
+    """
+    start_idx = None
+    end_idx = len(all_paragraphs)
+
+    for i, p in enumerate(all_paragraphs):
+        cls = get_css_class(p)
+        text = decode_text(p)
+
+        # Find start: p56 containing "Bhagaväner" or p2 TOC entry
+        if start_idx is None:
+            if cls == BOOK_TITLE_CLASS and "Bhag" in text and "kath" in text:
+                start_idx = i
+                continue
+            continue
+
+        # Find end: p35 containing "Indian Bhagavatams"
+        if cls == "p35" and "Indian" in text and "Bhagavatam" in text:
+            end_idx = i
+            break
+
+    if start_idx is None:
+        print("ERROR: Could not find 'Bhagaväner kathä' in others-before-1977.html!")
+        sys.exit(1)
+
+    return start_idx, end_idx
+
+
+def find_chapter_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
+    """Find all BK chapter bookmark tags (p17) in the BK range.
+
+    Returns list of (paragraph_index, raw_title_text).
+    """
+    bookmarks = []
+    for i in range(start_idx, end_idx):
+        p = all_paragraphs[i]
+        cls = get_css_class(p)
+        if cls == CHAPTER_BOOKMARK_CLASS:
+            text = decode_text(p).strip()
+            if text and "Bk:" in text:
+                bookmarks.append((i, text))
+    return bookmarks
+
+
+# ---------------------------------------------------------------------------
+# Paragraph extraction
+# ---------------------------------------------------------------------------
+
+
+def extract_paragraphs(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
+    """Extract body paragraphs for a chapter section.
+
+    Collects:
+      - p6, p12, p38 (body paragraphs — ScaGoudy decoded)
+      - p9, p19, p59, p23, p36, p60 (verse lines — centered italic, ScaGoudy decoded)
+      - p10, p61 (reference markers — ScaGoudy decoded)
+      - p57, p58 (footnotes — ScaGoudy decoded)
+    Skips:
+      - p56 (book title), p17 (bookmarks), p5 (chapter titles), p2 (TOC), empties
+    """
+    result = []
+
+    for i in range(start_idx, end_idx):
+        p = all_paragraphs[i]
+        cls = get_css_class(p)
+
+        # Skip empty paragraphs
+        if is_empty(p):
+            continue
+
+        # Skip known skip classes
+        if cls in SKIP_CLASSES:
+            continue
+
+        # Skip book title header
+        if cls == BOOK_TITLE_CLASS:
+            continue
+
+        # Skip chapter bookmark headers
+        if cls == CHAPTER_BOOKMARK_CLASS:
+            continue
+
+        # Skip chapter title headings (p5)
+        if cls == CHAPTER_TITLE_CLASS:
+            continue
+
+        # Skip TOC entries (p2)
+        if cls == TOC_CLASS:
+            continue
+
+        # Body paragraphs (ScaGoudy decoded)
+        if cls in BODY_CLASSES:
+            html = decode_html_content(p, apply_scagoudy=True)
+            if html:
+                result.append(html)
+            continue
+
+        # Sanskrit verse lines (centered italic, ScaGoudy decoded)
+        if cls in VERSE_CLASSES:
+            html = decode_html_content_as_verse(p)
+            if html:
+                result.append(html)
+            continue
+
+        # Reference markers (ScaGoudy decoded)
+        if cls in REFERENCE_CLASSES:
+            html = decode_html_content(p, apply_scagoudy=True)
+            if html:
+                result.append(html)
+            continue
+
+        # Footnote classes (ScaGoudy decoded)
+        if cls in FOOTNOTE_CLASSES:
+            html = decode_html_content(p, apply_scagoudy=True)
+            if html:
+                result.append(html)
+            continue
+
+        # Fallback: try to extract content from unknown classes
+        html = decode_html_content(p, apply_scagoudy=True)
+        if html:
+            print(f"    ⚠ Unknown class '{cls}' at paragraph {i}: {html[:80]}...")
+            result.append(html)
+
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Main extraction logic
+# ---------------------------------------------------------------------------
+
+
+def extract_bk(soup: BeautifulSoup) -> dict:
+    """Extract the entire Bhagavāner Kathā from the parsed HTML.
+
+    Returns a dict with:
+        "chapters": list of chapter dicts
+        "chapter_listing": list of {number, title} for intro.json
+    """
+    all_paragraphs = list(soup.find_all("p"))
+
+    # Step 1: Find BK boundaries
+    bk_start, bk_end = find_bk_boundaries(all_paragraphs)
+    print(f"  BK content: paragraphs {bk_start} to {bk_end}")
+    print(f"  Total paragraphs in range: {bk_end - bk_start}")
+
+    # Step 2: Find all chapter bookmarks (p17 with "Bk:")
+    bookmarks = find_chapter_bookmarks(all_paragraphs, bk_start, bk_end)
+    print(f"  Found {len(bookmarks)} BK chapter bookmarks:")
+    for pos, title in bookmarks:
+        print(f"    Paragraph {pos}: {title}")
+
+    # Step 3: Map bookmarks to chapters with proper titles
+    # Expected: 2 chapters
+    chapter_titles_map = {
+        1: "This World of Misery",
+        2: "The Cause of Misery",
+    }
+
+    chapter_bookmarks = []  # (para_idx, chapter_num, normalized_title)
+    for idx, (pos, _raw_title) in enumerate(bookmarks):
+        ch_num = idx + 1
+        ch_title = chapter_titles_map.get(ch_num, f"Chapter {ch_num}")
+        chapter_bookmarks.append((pos, ch_num, ch_title))
+
+    print(f"\n  Mapped {len(chapter_bookmarks)} chapters:")
+    for pos, num, title in chapter_bookmarks:
+        print(f"    Chapter {num}: {title} (paragraph {pos})")
+
+    # Step 4: Extract chapters
+    chapters = []
+    for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_bookmarks):
+        # Content starts after the bookmark line (skip p17 and p5 heading)
+        content_start = ch_pos + 1
+
+        # Content ends at next chapter bookmark or bk_end
+        if idx + 1 < len(chapter_bookmarks):
+            content_end = chapter_bookmarks[idx + 1][0]
+        else:
+            content_end = bk_end
+
+        paragraphs = extract_paragraphs(all_paragraphs, content_start, content_end)
+
+        ch = {
+            "book": "bk",
+            "chapter": ch_num,
+            "title": ch_title,
+            "type": "chapter",
+            "paragraphs": paragraphs,
+        }
+        chapters.append(ch)
+
+    # Step 5: Build chapter listing for intro.json
+    chapter_listing = [
+        {"number": ch["chapter"], "title": ch["title"]} for ch in chapters
+    ]
+
+    return {
+        "chapters": chapters,
+        "chapter_listing": chapter_listing,
+    }
+
+
+# ---------------------------------------------------------------------------
+# Output
+# ---------------------------------------------------------------------------
+
+
+def write_json(data, filepath):
+    """Write data as formatted JSON to filepath, creating directories as needed."""
+    os.makedirs(os.path.dirname(filepath), exist_ok=True)
+    with open(filepath, "w", encoding="utf-8") as f:
+        json.dump(data, f, ensure_ascii=False, indent=2)
+
+
+def main():
+    base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html")
+    output_base = os.path.join(base_dir, "lib", "jsons", "books", "bk")
+
+    if not os.path.exists(html_path):
+        print(f"ERROR: {html_path} not found!")
+        sys.exit(1)
+
+    print(f"Reading {html_path}...")
+    with open(html_path, "r", encoding="utf-8") as f:
+        soup = BeautifulSoup(f, "html.parser")
+
+    print("Extracting Bhagavāner Kathā...\n")
+    result = extract_bk(soup)
+
+    chapters = result["chapters"]
+    chapter_listing = result["chapter_listing"]
+
+    # --- Write intro.json ---
+    intro_meta = {
+        "book": "bk",
+        "title": "Bhagavāner Kathā",
+        "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
+        "year": None,
+        "description": (
+            "Early Bengali writings on the topics of Godhead "
+            "— part of 'Renunciation through Wisdom' (Vairāgya-vidyā)."
+        ),
+        "totalChapters": 2,
+        "chapters": chapter_listing,
+    }
+    intro_path = os.path.join(output_base, "intro.json")
+    write_json(intro_meta, intro_path)
+    print(f"\n  Written: {intro_path}")
+
+    # --- Write chapters ---
+    print(f"\n--- Chapters ---")
+    for ch in chapters:
+        ch_num_str = str(ch["chapter"]).zfill(2)
+        filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
+        write_json(ch, filepath)
+        print(
+            f"  Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
+            f"{ch['title']}"
+        )
+
+    # --- Summary ---
+    total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
+
+    print(f"\n{'=' * 60}")
+    print(f"  EXTRACTION SUMMARY")
+    print(f"{'=' * 60}")
+    print(f"  Chapters:           {len(chapters)}")
+    for ch in chapters:
+        print(
+            f"    Ch {ch['chapter']}: {len(ch['paragraphs'])} paragraphs — {ch['title']}"
+        )
+    print(f"  Total paragraphs:   {total_chapter_paras}")
+    if chapters:
+        print(f"  Avg paras/chapter:  {total_chapter_paras / len(chapters):.1f}")
+    print(f"  Output directory:   {output_base}/")
+
+    # --- Samples ---
+    for ch in chapters:
+        ch_num = ch["chapter"]
+        print(f"\n--- Sample: Chapter {ch_num} ({ch['title']}) ---")
+        for i, p in enumerate(ch["paragraphs"][:3]):
+            preview = re.sub(r"<[^>]+>", "", p)[:120]
+            print(f"  [{i}] {preview}...")
+        if len(ch["paragraphs"]) > 3:
+            print(f"  ... ({len(ch['paragraphs']) - 3} more paragraphs)")
+
+    # --- Quality Checks ---
+    print(f"\n--- Quality Checks ---")
+
+    # Check all 2 chapters present
+    expected_chapters = set(range(1, 3))
+    actual_chapters = set(ch["chapter"] for ch in chapters)
+    missing = expected_chapters - actual_chapters
+    extra = actual_chapters - expected_chapters
+    if missing:
+        print(f"  ⚠ Missing chapters: {sorted(missing)}")
+    if extra:
+        print(f"  ⚠ Extra chapters: {sorted(extra)}")
+    if not missing and not extra:
+        print(f"  ✅ All 2 chapters present")
+
+    # Check empty chapters
+    empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
+    if empty_chapters:
+        print(f"  ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
+    else:
+        print(f"  ✅ No empty chapters")
+
+    # Check ScaGoudy decoding worked — look for common ScaGoudy characters
+    scagoudy_chars = set("äåñöëòçìïüéàù")
+    all_text = ""
+    for ch in chapters:
+        all_text += " ".join(ch["paragraphs"])
+    remaining_scagoudy = [c for c in all_text if c in scagoudy_chars]
+    if remaining_scagoudy:
+        # Count occurrences
+        from collections import Counter
+
+        counts = Counter(remaining_scagoudy)
+        print(
+            f"  ⚠ Remaining ScaGoudy characters found (may be false positives): {dict(counts)}"
+        )
+    else:
+        print(f"  ✅ No ScaGoudy characters remaining — decoding looks clean")
+
+    # Check for IAST characters (positive signal)
+    iast_chars = set("āṛṣṭṇḍśṅñūīṁḥ")
+    iast_found = [c for c in all_text if c in iast_chars]
+    if iast_found:
+        from collections import Counter
+
+        counts = Counter(iast_found)
+        print(f"  ✅ IAST Unicode characters present: {dict(counts)}")
+    else:
+        print(f"  ⚠ No IAST Unicode characters found — decoding may have failed")
+
+    # Check chapter 1 has [RTW 1.1] reference
+    if chapters:
+        ch1_text = " ".join(chapters[0]["paragraphs"])
+        if "RTW" in ch1_text and "1.1" in ch1_text:
+            print(f"  ✅ '[RTW 1.1]' reference found in Chapter 1")
+        else:
+            print(f"  ⚠ '[RTW 1.1]' reference NOT found in Chapter 1")
+
+    # Check chapter 2 has [RTW 1.2] reference
+    if len(chapters) >= 2:
+        ch2_text = " ".join(chapters[1]["paragraphs"])
+        if "RTW" in ch2_text and "1.2" in ch2_text:
+            print(f"  ✅ '[RTW 1.2]' reference found in Chapter 2")
+        else:
+            print(f"  ⚠ '[RTW 1.2]' reference NOT found in Chapter 2")
+
+    # Check for content leak from next work (Indian Bhagavatams)
+    if "Indian Bhagavatam" in all_text:
+        print(f"  ⚠ Content from next work (Indian Bhagavatams) may have leaked!")
+    else:
+        print(f"  ✅ No content leak from next work detected")
+
+    # Check Kṛṣṇa appears (decoded from Kåñëa)
+    if "Kṛṣṇa" in all_text:
+        print(f"  ✅ 'Kṛṣṇa' properly decoded (from 'Kåñëa')")
+    else:
+        print(f"  ⚠ 'Kṛṣṇa' not found — ScaGoudy decoding may need review")
+
+    # Check last chapter ends cleanly
+    if chapters:
+        last_ch = chapters[-1]
+        last_para = last_ch["paragraphs"][-1] if last_ch["paragraphs"] else ""
+        last_preview = re.sub(r"<[^>]+>", "", last_para)[:100]
+        print(f'  ℹ Last paragraph of Ch {last_ch["chapter"]}: "{last_preview}..."')
+
+    print(f"\nDone! ✨")
+
+
+if __name__ == "__main__":
+    main()