feat: add ssr

scripts/mg_extract.py

@@ -0,0 +1,644 @@
+#!/usr/bin/env python3
+"""Extract Kṛṣṇa Consciousness: The Matchless Gift (1974) from other.html.
+
+The book has 1 intro segment (front-pages) and 7 chapters.
+
+Output:
+    lib/jsons/books/mg/intro.json          — Book metadata
+    lib/jsons/books/mg/intro/*.json         — Intro segments
+    lib/jsons/books/mg/chapters/*.json      — Chapters
+
+Usage:
+    python3 scripts/mg_extract.py
+"""
+
+import json
+import os
+import re
+import sys
+
+from bs4 import BeautifulSoup, Tag
+
+# ---------------------------------------------------------------------------
+# ScaGoudy → IAST Unicode mapping
+# ---------------------------------------------------------------------------
+SCAGOUDY_TO_IAST = {
+    "ä": "ā",
+    "Ä": "Ā",
+    "å": "ṛ",
+    "Å": "Ṛ",
+    "ñ": "ṣ",
+    "Ñ": "Ṣ",
+    "ö": "ṭ",
+    "Ö": "Ṭ",
+    "ë": "ṇ",
+    "Ë": "Ṇ",
+    "ò": "ḍ",
+    "Ò": "Ḍ",
+    "ç": "ś",
+    "Ç": "Ś",
+    "ì": "ṅ",
+    "ï": "ñ",
+    "ü": "ū",
+    "Ü": "Ū",
+    "é": "ī",
+    "É": "Ī",
+    "à": "ṁ",
+    "ù": "ḥ",
+    "è": "ṝ",
+    "û": "ṁl",
+    "ÿ": "ḷ",
+}
+
+
+def decode_scagoudy(text: str) -> str:
+    """Replace ScaGoudy-encoded characters with proper IAST Unicode."""
+    for src, dst in SCAGOUDY_TO_IAST.items():
+        text = text.replace(src, dst)
+    return text
+
+
+def decode_html_content(tag: Tag) -> str:
+    """Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
+    inner = "".join(str(child) for child in tag.children)
+    inner = decode_scagoudy(inner)
+    # Remove span tags with CSS classes (RTF conversion artifacts)
+    inner = re.sub(
+        r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
+    )
+    # Clean up Apple-tab-span leftovers (tab characters)
+    inner = inner.replace("\t", " ")
+    # Normalize whitespace (but preserve <br> tags)
+    inner = re.sub(r"\s+", " ", inner).strip()
+    return inner
+
+
+def decode_text(tag: Tag) -> str:
+    """Get plain text from a tag with ScaGoudy decoding."""
+    return decode_scagoudy(tag.get_text().strip())
+
+
+def get_css_class(tag: Tag) -> str:
+    """Get the first CSS class of a tag, or empty string."""
+    classes = tag.get("class", [])
+    return classes[0] if classes else ""
+
+
+def is_empty(tag: Tag) -> bool:
+    """Check if a tag is empty or whitespace-only."""
+    text = tag.get_text().strip()
+    return not text or text == "\xa0"
+
+
+def is_dash_line(text: str) -> bool:
+    """Return True if text is only dashes, spaces, and/or whitespace."""
+    cleaned = text.strip()
+    if not cleaned:
+        return False
+    return all(c in "-—–\u2014\u2013 \t" for c in cleaned)
+
+
+# ---------------------------------------------------------------------------
+# CSS class categorization for MG
+# ---------------------------------------------------------------------------
+
+# Book title header class
+BOOK_HEADER_CLASS = "p12"
+
+# Chapter bookmark class
+CHAPTER_BOOKMARK_CLASS = "p13"
+
+# Body paragraph classes (primary content)
+BODY_CLASSES = frozenset(["p6", "p78", "p30"])
+
+# Sanskrit verse pāda classes (italic, centered) — BOTH p53 AND p24
+VERSE_PADA_CLASSES = frozenset(["p53", "p24"])
+
+# Verse translations / body text
+TRANSLATION_CLASS = "p44"
+
+# Chapter heading (bold)
+CHAPTER_HEADING_CLASS = "p27"
+
+# Title page text
+TITLE_PAGE_CLASS = "p3"
+
+# Book list header/footer
+BOOK_LIST_CLASSES = frozenset(["p76", "p77"])
+
+# Classes to skip entirely (empty/decorative)
+SKIP_CLASSES = frozenset(["p9", "p70", "p79"])
+
+
+# ---------------------------------------------------------------------------
+# Extraction helpers
+# ---------------------------------------------------------------------------
+
+
+def find_mg_boundaries(all_paragraphs: list) -> tuple:
+    """Find the start and end paragraph indices of the MG content.
+
+    Start: <p class="p12"> containing "Matchless Gift" and "1974"
+    End: next <p class="p12"> after start, or <p class="p27"> containing "APPENDIX"
+    """
+    start_idx = None
+    end_idx = len(all_paragraphs)
+
+    for i, p in enumerate(all_paragraphs):
+        cls = get_css_class(p)
+        if cls != BOOK_HEADER_CLASS:
+            continue
+        text = decode_text(p)
+        if "Matchless Gift" in text and "1974" in text:
+            start_idx = i
+        elif start_idx is not None:
+            # Next p12 after MG start = end of MG
+            end_idx = i
+            break
+
+    if start_idx is None:
+        print(
+            "ERROR: Could not find 'Kṛṣṇa Consciousness: The Matchless Gift (1974)' header in other.html!"
+        )
+        sys.exit(1)
+
+    # Also check for APPENDIXES/APPENDIX as end boundary (p27 with bold text)
+    for i in range(start_idx + 1, end_idx):
+        p = all_paragraphs[i]
+        cls = get_css_class(p)
+        if cls == CHAPTER_HEADING_CLASS:
+            text = p.get_text().strip().upper()
+            if text.startswith("APPENDIX") or text.startswith("APPENDIXES"):
+                end_idx = i
+                break
+
+    return start_idx, end_idx
+
+
+def find_chapter_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
+    """Find all MG chapter bookmark tags (p13) in the MG range.
+
+    Returns list of (paragraph_index, chapter_number, chapter_title).
+    """
+    bookmarks = []
+    for i in range(start_idx, end_idx):
+        p = all_paragraphs[i]
+        cls = get_css_class(p)
+        if cls == CHAPTER_BOOKMARK_CLASS:
+            text = decode_text(p)
+            # Use re.DOTALL to handle titles that span multiple lines
+            m = re.match(r"MG\s+(\d+)\s*:\s*(.+)", text, re.DOTALL)
+            if m:
+                ch_num = int(m.group(1))
+                # Clean the title: replace newlines with spaces, normalize whitespace
+                ch_title = re.sub(r"\s+", " ", m.group(2)).strip()
+                # Capitalize "love" in "Learning to love" -> "Learning to Love"
+                if ch_title == "Learning to love":
+                    ch_title = "Learning to Love"
+                bookmarks.append((i, ch_num, ch_title))
+    return bookmarks
+
+
+def extract_paragraphs(
+    all_paragraphs: list,
+    start_idx: int,
+    end_idx: int,
+    is_front_pages: bool = False,
+) -> list:
+    """Extract paragraphs for a section (chapter or intro segment).
+
+    Handles all CSS classes within the MG content.
+    Groups consecutive p53/p24 verse pādas into centered italic divs.
+    """
+    result = []
+    verse_accumulator = []
+
+    def flush_verse():
+        if verse_accumulator:
+            joined = "<br>".join(verse_accumulator)
+            result.append(f'<div style="text-align:center"><i>{joined}</i></div>')
+            verse_accumulator.clear()
+
+    for i in range(start_idx, end_idx):
+        p = all_paragraphs[i]
+        cls = get_css_class(p)
+
+        # Skip empty paragraphs
+        if is_empty(p):
+            continue
+
+        # Skip noise classes
+        if cls in SKIP_CLASSES:
+            continue
+
+        # Skip bookmark tags (p13)
+        if cls == CHAPTER_BOOKMARK_CLASS:
+            continue
+
+        # Skip book title header (p12)
+        if cls == BOOK_HEADER_CLASS:
+            continue
+
+        # Check for dash lines — skip them
+        text = decode_text(p)
+        if is_dash_line(text):
+            continue
+
+        # Sanskrit verse pādas (p53 and p24) — accumulate into centered block
+        if cls in VERSE_PADA_CLASSES:
+            html = decode_html_content(p)
+            if html:
+                # Remove inner <i> tags before re-wrapping
+                stripped = re.sub(r"</?i>", "", html)
+                verse_accumulator.append(stripped)
+            continue
+
+        # Non-verse paragraph — flush any accumulated verse first
+        flush_verse()
+
+        # Chapter heading (p27) — include as paragraph
+        if cls == CHAPTER_HEADING_CLASS:
+            html = decode_html_content(p)
+            if html:
+                result.append(html)
+            continue
+
+        # Title page text (p3) — include
+        if cls == TITLE_PAGE_CLASS:
+            html = decode_html_content(p)
+            if html:
+                result.append(html)
+            continue
+
+        # Book list classes (p76, p77) — include in front-pages
+        if cls in BOOK_LIST_CLASSES:
+            if is_front_pages:
+                html = decode_html_content(p)
+                if html:
+                    result.append(html)
+            continue
+
+        # Verse translation / body text (p44)
+        if cls == TRANSLATION_CLASS:
+            html = decode_html_content(p)
+            if html:
+                result.append(html)
+            continue
+
+        # Body paragraphs (p6, p78, p30)
+        if cls in BODY_CLASSES:
+            html = decode_html_content(p)
+            if html:
+                result.append(html)
+            continue
+
+        # Any other class — try to extract content (be permissive)
+        html = decode_html_content(p)
+        if html and not is_dash_line(text):
+            result.append(html)
+
+    # Flush any trailing verse
+    flush_verse()
+
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Main extraction logic
+# ---------------------------------------------------------------------------
+
+
+def extract_mg(soup: BeautifulSoup) -> dict:
+    """Extract the entire Kṛṣṇa Consciousness: The Matchless Gift (1974) from the parsed HTML.
+
+    Returns a dict with:
+        "intro_segments": list of intro segment dicts
+        "chapters": list of chapter dicts
+        "chapter_listing": list of {number, title} for intro.json
+    """
+    all_paragraphs = list(soup.find_all("p"))
+
+    # Step 1: Find MG boundaries
+    mg_start, mg_end = find_mg_boundaries(all_paragraphs)
+    print(f"  MG content: paragraphs {mg_start} to {mg_end}")
+    print(f"  Total paragraphs in range: {mg_end - mg_start}")
+
+    # Step 2: Find all chapter bookmarks
+    chapter_bookmarks = find_chapter_bookmarks(all_paragraphs, mg_start, mg_end)
+    print(f"  Found {len(chapter_bookmarks)} MG chapter bookmarks")
+    for pos, ch_num, ch_title in chapter_bookmarks:
+        print(f"    [chapter {ch_num}] paragraph {pos}: {ch_title}")
+
+    # Step 3: Extract front pages
+    # From book start (mg_start) to first chapter bookmark (MG 1)
+    intro_segments = []
+
+    if chapter_bookmarks:
+        front_pages_end = chapter_bookmarks[0][0]
+    else:
+        front_pages_end = mg_end
+
+    front_pages_paragraphs = extract_paragraphs(
+        all_paragraphs, mg_start, front_pages_end, is_front_pages=True
+    )
+    intro_segments.append(
+        {
+            "slug": "front-pages",
+            "title": "Front Pages",
+            "type": "intro-segment",
+            "book": "mg",
+            "order": 1,
+            "paragraphs": front_pages_paragraphs,
+        }
+    )
+
+    # Step 4: Extract chapters
+    chapters = []
+    for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_bookmarks):
+        content_start = ch_pos + 1
+
+        # Content ends at next chapter bookmark or mg_end
+        if idx + 1 < len(chapter_bookmarks):
+            content_end = chapter_bookmarks[idx + 1][0]
+        else:
+            content_end = mg_end
+
+        paragraphs = extract_paragraphs(all_paragraphs, content_start, content_end)
+
+        ch = {
+            "book": "mg",
+            "chapter": ch_num,
+            "title": ch_title,
+            "type": "chapter",
+            "paragraphs": paragraphs,
+        }
+        chapters.append(ch)
+
+    # Step 5: Build chapter listing for intro.json
+    chapter_listing = [
+        {"number": ch["chapter"], "title": ch["title"]} for ch in chapters
+    ]
+
+    return {
+        "intro_segments": intro_segments,
+        "chapters": chapters,
+        "chapter_listing": chapter_listing,
+    }
+
+
+# ---------------------------------------------------------------------------
+# Output
+# ---------------------------------------------------------------------------
+
+
+def write_json(data, filepath):
+    """Write data as formatted JSON to filepath, creating directories as needed."""
+    os.makedirs(os.path.dirname(filepath), exist_ok=True)
+    with open(filepath, "w", encoding="utf-8") as f:
+        json.dump(data, f, ensure_ascii=False, indent=2)
+
+
+def main():
+    base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    html_path = os.path.join(base_dir, "public", "html", "other.html")
+    output_base = os.path.join(base_dir, "lib", "jsons", "books", "mg")
+
+    if not os.path.exists(html_path):
+        print(f"ERROR: {html_path} not found!")
+        sys.exit(1)
+
+    print(f"Reading {html_path}...")
+    with open(html_path, "r", encoding="utf-8") as f:
+        soup = BeautifulSoup(f, "html.parser")
+
+    print("Extracting Kṛṣṇa Consciousness: The Matchless Gift (1974)...\n")
+    result = extract_mg(soup)
+
+    intro_segments = result["intro_segments"]
+    chapters = result["chapters"]
+    chapter_listing = result["chapter_listing"]
+
+    # --- Write intro.json ---
+    intro_meta = {
+        "book": "mg",
+        "title": "Kṛṣṇa Consciousness: The Matchless Gift",
+        "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
+        "year": 1974,
+        "description": "A profound introduction to the science of Kṛṣṇa consciousness and the matchless gift of love of God.",
+        "totalChapters": 7,
+        "chapters": chapter_listing,
+    }
+    intro_path = os.path.join(output_base, "intro.json")
+    write_json(intro_meta, intro_path)
+    print(f"\n  Written: {intro_path}")
+
+    # --- Write intro segments ---
+    print(f"\n--- Intro Segments ---")
+    for seg in intro_segments:
+        filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
+        write_json(seg, filepath)
+        print(
+            f"  {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}"
+        )
+
+    # --- Write chapters ---
+    print(f"\n--- Chapters ---")
+    for ch in chapters:
+        ch_num_str = str(ch["chapter"]).zfill(2)
+        filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
+        write_json(ch, filepath)
+        print(
+            f"  Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
+            f"{ch['title']}"
+        )
+
+    # --- Summary ---
+    total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments)
+    total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
+    total_paras = total_intro_paras + total_chapter_paras
+
+    print(f"\n{'=' * 60}")
+    print(f"  EXTRACTION SUMMARY")
+    print(f"{'=' * 60}")
+    print(f"  Intro segments:     {len(intro_segments)}")
+    print(f"  Chapters:           {len(chapters)}")
+    print(f"  Total paragraphs:   {total_paras}")
+    print(f"    Intro:            {total_intro_paras}")
+    print(f"    Chapters:         {total_chapter_paras}")
+    if chapters:
+        print(f"  Avg paras/chapter:  {total_chapter_paras / len(chapters):.1f}")
+    print(f"  Output directory:   {output_base}/")
+
+    # --- Samples ---
+    print(f"\n--- Sample: Front Pages ---")
+    if intro_segments:
+        front = intro_segments[0]
+        for i, p in enumerate(front["paragraphs"][:5]):
+            preview = re.sub(r"<[^>]+>", "", p)[:100]
+            print(f"  [{i}] {preview}...")
+
+    if chapters:
+        print(f"\n--- Sample: Chapter 1 ({chapters[0]['title']}) ---")
+        ch1 = chapters[0]
+        for i, p in enumerate(ch1["paragraphs"][:5]):
+            preview = re.sub(r"<[^>]+>", "", p)[:100]
+            print(f"  [{i}] {preview}...")
+
+    if chapters and len(chapters) > 1:
+        print(
+            f"\n--- Sample: Chapter {chapters[-1]['chapter']} ({chapters[-1]['title']}) ---"
+        )
+        last = chapters[-1]
+        for i, p in enumerate(last["paragraphs"][:5]):
+            preview = re.sub(r"<[^>]+>", "", p)[:100]
+            print(f"  [{i}] {preview}...")
+
+    # --- Quality Checks ---
+    print(f"\n--- Quality Checks ---")
+
+    # Check all 7 chapters present
+    expected_chapters = set(range(1, 8))
+    actual_chapters = set(ch["chapter"] for ch in chapters)
+    missing = expected_chapters - actual_chapters
+    extra = actual_chapters - expected_chapters
+    if missing:
+        print(f"  ⚠ Missing chapters: {sorted(missing)}")
+    if extra:
+        print(f"  ⚠ Extra chapters: {sorted(extra)}")
+    if not missing and not extra:
+        print(f"  ✅ All 7 chapters present")
+
+    # Check all 1 intro segment present
+    expected_intros = {"front-pages"}
+    actual_intros = {s["slug"] for s in intro_segments}
+    missing_intros = expected_intros - actual_intros
+    extra_intros = actual_intros - expected_intros
+    if missing_intros:
+        print(f"  ⚠ Missing intro segments: {missing_intros}")
+    if extra_intros:
+        print(f"  ⚠ Extra intro segments: {extra_intros}")
+    if not missing_intros and not extra_intros:
+        print(f"  ✅ All 1 intro segments present")
+
+    # Check empty chapters
+    empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
+    if empty_chapters:
+        print(f"  ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
+    else:
+        print(f"  ✅ No empty chapters")
+
+    # Check empty intro segments
+    empty_intros = [s["slug"] for s in intro_segments if len(s["paragraphs"]) == 0]
+    if empty_intros:
+        print(f"  ⚠ Empty intro segments: {empty_intros}")
+    else:
+        print(f"  ✅ No empty intro segments")
+
+    # Check ScaGoudy decoding
+    print(f"\n--- ScaGoudy Decoding Check ---")
+    all_text = ""
+    for ch in chapters:
+        all_text += " ".join(ch["paragraphs"][:20])
+    for seg in intro_segments:
+        all_text += " ".join(seg["paragraphs"][:10])
+
+    # Check for properly decoded words
+    if "Kṛṣṇa" in all_text:
+        print(f"  ✅ 'Kṛṣṇa' found (ScaGoudy properly decoded)")
+    else:
+        print(f"  ⚠ 'Kṛṣṇa' not found — ScaGoudy decoding may have failed")
+
+    if "Bhagavad-gītā" in all_text:
+        print(f"  ✅ 'Bhagavad-gītā' found (ScaGoudy properly decoded)")
+    elif "gītā" in all_text or "Gītā" in all_text:
+        print(f"  ✅ 'gītā' found (ScaGoudy properly decoded)")
+    else:
+        print(f"  ⚠ 'Gītā' not found — ScaGoudy decoding may have failed")
+
+    if "Prabhupāda" in all_text:
+        print(f"  ✅ 'Prabhupāda' found (ScaGoudy properly decoded)")
+
+    # Check for un-decoded ScaGoudy remnants (common encoded chars)
+    scagoudy_remnants = 0
+    for ch in chapters:
+        for p in ch["paragraphs"]:
+            if "Kåñëa" in p or "Prabhupäda" in p:
+                scagoudy_remnants += 1
+    for seg in intro_segments:
+        for p in seg["paragraphs"]:
+            if "Kåñëa" in p or "Prabhupäda" in p:
+                scagoudy_remnants += 1
+    if scagoudy_remnants == 0:
+        print(f"  ✅ No ScaGoudy remnants detected")
+    else:
+        print(f"  ⚠ {scagoudy_remnants} paragraphs with ScaGoudy remnants")
+
+    # Check for centered verse blocks
+    verse_block_count = 0
+    for ch in chapters:
+        for p in ch["paragraphs"]:
+            if '<div style="text-align:center"><i>' in p:
+                verse_block_count += 1
+    if verse_block_count > 0:
+        print(f"  ✅ {verse_block_count} centered Sanskrit verse blocks found")
+    else:
+        print(f"  ⚠ No centered verse blocks found — verse grouping may have failed")
+
+    # Check for residual span tags
+    span_count = 0
+    for ch in chapters:
+        for p in ch["paragraphs"]:
+            if 'class="' in p and "<span" in p:
+                span_count += 1
+    for seg in intro_segments:
+        for p in seg["paragraphs"]:
+            if 'class="' in p and "<span" in p:
+                span_count += 1
+    if span_count == 0:
+        print(f"  ✅ No residual span class artifacts found")
+    else:
+        print(f"  ⚠ {span_count} paragraphs still contain span class artifacts")
+
+    # Check for dash-line paragraphs that shouldn't be there
+    dash_count = 0
+    for ch in chapters:
+        for p in ch["paragraphs"]:
+            plain = re.sub(r"<[^>]+>", "", p)
+            if is_dash_line(plain):
+                dash_count += 1
+    for seg in intro_segments:
+        for p in seg["paragraphs"]:
+            plain = re.sub(r"<[^>]+>", "", p)
+            if is_dash_line(plain):
+                dash_count += 1
+    if dash_count == 0:
+        print(f"  ✅ No dashed-line paragraphs in output")
+    else:
+        print(f"  ⚠ {dash_count} dashed-line paragraphs found in output")
+
+    # Show a verse block sample
+    print(f"\n--- Sample: Verse Block ---")
+    for ch in chapters:
+        for p in ch["paragraphs"]:
+            if '<div style="text-align:center"><i>' in p:
+                preview = re.sub(r"<[^>]+>", "", p)[:120]
+                print(f"  {preview}...")
+                break
+        else:
+            continue
+        break
+
+    # Chapter title decoding check
+    print(f"\n--- Chapter Title Decoding Check ---")
+    for ch in chapters:
+        title = ch["title"]
+        if "å" in title or "ë" in title or "ñ" in title or "ä" in title:
+            print(f"  ⚠ Chapter {ch['chapter']} title has un-decoded ScaGoudy: {title}")
+        else:
+            print(f"  ✅ Ch {ch['chapter']}: {title}")
+
+    print(f"\nDone! ✨")
+
+
+if __name__ == "__main__":
+    main()