spsays-training/scripts/mog_extract.py

#!/usr/bin/env python3
"""Extract Message of Godhead (1955) from others-before-1977.html.

The book has 1 intro segment (front-pages) and 3 chapters:
  1. Message of Godhead (Introductory)
  2. Knowledge Transcendental
  3. Work with Transcendental Effect (Karma Yoga)

Output:
    lib/jsons/books/mog/intro.json          — Book metadata
    lib/jsons/books/mog/intro/*.json         — Intro segments (front-pages)
    lib/jsons/books/mog/chapters/*.json      — Chapters (01–03)

Usage:
    python3 scripts/mog_extract.py
"""

import json
import os
import re
import sys

from bs4 import BeautifulSoup, Tag

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------


def decode_html_content(tag: Tag) -> str:
    """Extract inner HTML from a tag, preserve formatting tags, strip spans."""
    inner = "".join(str(child) for child in tag.children)
    # Remove span tags with CSS classes (RTF conversion artifacts)
    inner = re.sub(
        r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
    )
    # Clean up Apple-tab-span leftovers (tab characters)
    inner = inner.replace("\t", " ")
    # Normalize whitespace
    inner = re.sub(r"\s+", " ", inner).strip()
    return inner


def decode_text(tag: Tag) -> str:
    """Get plain text from a tag."""
    return tag.get_text().strip()


def get_css_class(tag: Tag) -> str:
    """Get the first CSS class of a tag, or empty string."""
    classes = tag.get("class", [])
    return classes[0] if classes else ""


def is_empty(tag: Tag) -> bool:
    """Check if a tag is empty or whitespace-only."""
    text = tag.get_text().strip()
    return not text or text == "\xa0"


# ---------------------------------------------------------------------------
# CSS class categorization for MOG
# ---------------------------------------------------------------------------

# Front page classes
FRONT_TITLE_CLASS = "p3"  # Book title header (bold)
FRONT_INFO_CLASS = "p19"  # Author/address info
FRONT_DATE_CLASS = "p15"  # Date line ("From the 30/8/55")

# Chapter heading classes
CHAPTER_BOOKMARK_CLASS = (
    "p11"  # Chapter bookmark headers (like "MESSAGE OF GODHEAD (Introductory)")
)
CHAPTER_TITLE_CLASS = "p5"  # Chapter title (bold, duplicate of p11 — skip)

# Body paragraph classes
BODY_CLASSES = frozenset(["p6", "p38", "p44", "p45", "p46", "p47"])

# Classes to skip entirely (empty/decorative/break)
SKIP_CLASSES = frozenset(["p43", "p24"])

# Boundary detection: classes that mark the start of a different work after MOG
NEXT_WORK_BOUNDARY_CLASSES = frozenset(["p2", "p3"])


# ---------------------------------------------------------------------------
# Boundary detection
# ---------------------------------------------------------------------------


def find_mog_boundaries(all_paragraphs: list) -> tuple:
    """Find the start and end paragraph indices of the MOG content.

    Start: <p class="p3"> containing "Message of Godhead"
    End: next <p class="p2"> or <p class="p3"> that starts a different book,
         or <p class="p6"> containing "Bhakti-rasāmṛta-sindhu" (the next work)
    """
    start_idx = None
    end_idx = len(all_paragraphs)

    for i, p in enumerate(all_paragraphs):
        cls = get_css_class(p)
        text = decode_text(p)

        # Find start: p3 containing "Message of Godhead"
        if start_idx is None:
            if cls == "p2" and "Message of Godhead" in text:
                # The p2 is the TOC entry; the actual start is the next p3
                start_idx = i
                continue
            continue

        # Find end: look for the next work boundary
        # The p6 containing "Bhakti-rasāmṛta-sindhu" starts a different work
        if cls == "p6" and "Bhakti-ras" in text and "sindhu" in text:
            end_idx = i
            break

        # Also check for p2 that starts a different book section
        if cls == "p2" and start_idx is not None and i > start_idx + 5:
            text_lower = text.lower()
            if "message of godhead" not in text_lower:
                end_idx = i
                break

    if start_idx is None:
        print("ERROR: Could not find 'Message of Godhead' in others-before-1977.html!")
        sys.exit(1)

    return start_idx, end_idx


def find_chapter_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
    """Find all MOG chapter bookmark tags (p11) in the MOG range.

    Returns list of (paragraph_index, chapter_title).
    """
    bookmarks = []
    for i in range(start_idx, end_idx):
        p = all_paragraphs[i]
        cls = get_css_class(p)
        if cls == CHAPTER_BOOKMARK_CLASS:
            text = decode_text(p).strip()
            if text:
                bookmarks.append((i, text))
    return bookmarks


# ---------------------------------------------------------------------------
# Paragraph extraction
# ---------------------------------------------------------------------------


def extract_paragraphs(
    all_paragraphs: list,
    start_idx: int,
    end_idx: int,
    is_front_pages: bool = False,
) -> list:
    """Extract paragraphs for a section (front pages or chapter body).

    For front pages: collect p3, p19, p15 lines as paragraphs.
    For chapters: collect body paragraphs (p6, p38, p44, p45, p46, p47).
    Always skip p43, p24 (empty/break), p11 (bookmark), p5 (duplicate heading).
    """
    result = []

    for i in range(start_idx, end_idx):
        p = all_paragraphs[i]
        cls = get_css_class(p)

        # Skip empty paragraphs
        if is_empty(p):
            continue

        # Skip known empty/break classes
        if cls in SKIP_CLASSES:
            continue

        # Skip chapter bookmark headers (p11) — used for boundary detection only
        if cls == CHAPTER_BOOKMARK_CLASS:
            continue

        # Skip chapter title duplicate (p5)
        if cls == CHAPTER_TITLE_CLASS:
            continue

        # Skip p2 (TOC entry at start)
        if cls == "p2":
            continue

        if is_front_pages:
            # For front pages, collect title, author/address, and date lines
            if cls in (FRONT_TITLE_CLASS, FRONT_INFO_CLASS, FRONT_DATE_CLASS):
                html = decode_html_content(p)
                if html:
                    result.append(html)
                continue
            # Also include any other non-skip class if we encounter it
            # (shouldn't happen much for front pages, but be permissive)
            if cls in BODY_CLASSES:
                # Body paragraph before first chapter = still front pages
                html = decode_html_content(p)
                if html:
                    result.append(html)
                continue
        else:
            # For chapter body, collect all body paragraph classes
            if cls in BODY_CLASSES:
                html = decode_html_content(p)
                if html:
                    result.append(html)
                continue

            # p19 can appear inside chapters (e.g., "(Devanāgarī)" in chapter 2)
            if cls == FRONT_INFO_CLASS:
                html = decode_html_content(p)
                if html:
                    result.append(html)
                continue

            # Title page class inside chapter body (shouldn't happen, but be safe)
            if cls == FRONT_TITLE_CLASS:
                html = decode_html_content(p)
                if html:
                    result.append(html)
                continue

        # Fallback: try to extract content from unknown classes
        html = decode_html_content(p)
        if html:
            result.append(html)

    return result


# ---------------------------------------------------------------------------
# Main extraction logic
# ---------------------------------------------------------------------------


def extract_mog(soup: BeautifulSoup) -> dict:
    """Extract the entire Message of Godhead from the parsed HTML.

    Returns a dict with:
        "intro_segments": list of intro segment dicts
        "chapters": list of chapter dicts
        "chapter_listing": list of {number, title} for intro.json
    """
    all_paragraphs = list(soup.find_all("p"))

    # Step 1: Find MOG boundaries
    mog_start, mog_end = find_mog_boundaries(all_paragraphs)
    print(f"  MOG content: paragraphs {mog_start} to {mog_end}")
    print(f"  Total paragraphs in range: {mog_end - mog_start}")

    # Step 2: Find all chapter bookmarks (p11)
    bookmarks = find_chapter_bookmarks(all_paragraphs, mog_start, mog_end)
    print(f"  Found {len(bookmarks)} MOG chapter bookmarks:")
    for pos, title in bookmarks:
        print(f"    Paragraph {pos}: {title}")

    # Step 3: Map bookmarks to chapters with proper titles
    # Expected: 3 chapters
    # Map raw bookmark text (case-insensitive) to normalized titles
    chapter_titles = {
        "message of godhead": "Message of Godhead (Introductory)",
        "knowledge transcendental": "Knowledge Transcendental",
        "work with transcendental effect": "Work with Transcendental Effect (Karma Yoga)",
    }

    chapter_bookmarks = []  # (para_idx, chapter_num, normalized_title)
    ch_num = 1
    for pos, raw_title in bookmarks:
        # Try to match known chapter titles (case-insensitive)
        matched_title = None
        raw_lower = raw_title.lower().strip()
        for key, nice_title in chapter_titles.items():
            if key in raw_lower:
                matched_title = nice_title
                break

        if matched_title is None:
            # Fallback: use the raw title with title case
            matched_title = raw_title.strip()

        chapter_bookmarks.append((pos, ch_num, matched_title))
        ch_num += 1

    print(f"\n  Mapped {len(chapter_bookmarks)} chapters:")
    for pos, num, title in chapter_bookmarks:
        print(f"    Chapter {num}: {title} (paragraph {pos})")

    # Step 4: Extract front pages
    # From MOG start to first chapter bookmark
    intro_segments = []

    if chapter_bookmarks:
        front_pages_end = chapter_bookmarks[0][0]
    else:
        front_pages_end = mog_end

    front_pages_paragraphs = extract_paragraphs(
        all_paragraphs, mog_start, front_pages_end, is_front_pages=True
    )
    intro_segments.append(
        {
            "slug": "front-pages",
            "title": "Front Pages",
            "type": "intro-segment",
            "book": "mog",
            "order": 1,
            "paragraphs": front_pages_paragraphs,
        }
    )

    # Step 5: Extract chapters
    chapters = []
    for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_bookmarks):
        # Content starts after the bookmark line
        # But we also need to skip the p5 duplicate heading that follows p11
        content_start = ch_pos + 1

        # Content ends at next chapter bookmark or mog_end
        if idx + 1 < len(chapter_bookmarks):
            content_end = chapter_bookmarks[idx + 1][0]
        else:
            content_end = mog_end

        paragraphs = extract_paragraphs(all_paragraphs, content_start, content_end)

        ch = {
            "book": "mog",
            "chapter": ch_num,
            "title": ch_title,
            "type": "chapter",
            "paragraphs": paragraphs,
        }
        chapters.append(ch)

    # Step 6: Build chapter listing for intro.json
    chapter_listing = [
        {"number": ch["chapter"], "title": ch["title"]} for ch in chapters
    ]

    return {
        "intro_segments": intro_segments,
        "chapters": chapters,
        "chapter_listing": chapter_listing,
    }


# ---------------------------------------------------------------------------
# Output
# ---------------------------------------------------------------------------


def write_json(data, filepath):
    """Write data as formatted JSON to filepath, creating directories as needed."""
    os.makedirs(os.path.dirname(filepath), exist_ok=True)
    with open(filepath, "w", encoding="utf-8") as f:
        json.dump(data, f, ensure_ascii=False, indent=2)


def main():
    base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html")
    output_base = os.path.join(base_dir, "lib", "jsons", "books", "mog")

    if not os.path.exists(html_path):
        print(f"ERROR: {html_path} not found!")
        sys.exit(1)

    print(f"Reading {html_path}...")
    with open(html_path, "r", encoding="utf-8") as f:
        soup = BeautifulSoup(f, "html.parser")

    print("Extracting Message of Godhead (1955)...\n")
    result = extract_mog(soup)

    intro_segments = result["intro_segments"]
    chapters = result["chapters"]
    chapter_listing = result["chapter_listing"]

    # --- Write intro.json ---
    intro_meta = {
        "book": "mog",
        "title": "Message of Godhead",
        "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
        "year": 1955,
        "description": "An early handwritten work presenting the essence of Bhagavad-gītā's teachings on transcendental knowledge, karma-yoga, and the Personality of Godhead.",
        "totalChapters": 3,
        "chapters": chapter_listing,
    }
    intro_path = os.path.join(output_base, "intro.json")
    write_json(intro_meta, intro_path)
    print(f"\n  Written: {intro_path}")

    # --- Write intro segments ---
    print(f"\n--- Intro Segments ---")
    for seg in intro_segments:
        filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
        write_json(seg, filepath)
        print(
            f"  {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}"
        )

    # --- Write chapters ---
    print(f"\n--- Chapters ---")
    for ch in chapters:
        ch_num_str = str(ch["chapter"]).zfill(2)
        filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
        write_json(ch, filepath)
        print(
            f"  Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
            f"{ch['title']}"
        )

    # --- Summary ---
    total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments)
    total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
    total_paras = total_intro_paras + total_chapter_paras

    print(f"\n{'=' * 60}")
    print(f"  EXTRACTION SUMMARY")
    print(f"{'=' * 60}")
    print(f"  Intro segments:     {len(intro_segments)}")
    for seg in intro_segments:
        print(
            f"    {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs"
        )
    print(f"  Chapters:           {len(chapters)}")
    for ch in chapters:
        print(
            f"    Ch {ch['chapter']}: {len(ch['paragraphs'])} paragraphs — {ch['title']}"
        )
    print(f"  Total paragraphs:   {total_paras}")
    print(f"    Intro:            {total_intro_paras}")
    print(f"    Chapters:         {total_chapter_paras}")
    if chapters:
        print(f"  Avg paras/chapter:  {total_chapter_paras / len(chapters):.1f}")
    print(f"  Output directory:   {output_base}/")

    # --- Samples ---
    print(f"\n--- Sample: Front Pages ---")
    if intro_segments:
        front = intro_segments[0]
        for i, p in enumerate(front["paragraphs"][:5]):
            preview = re.sub(r"<[^>]+>", "", p)[:120]
            print(f"  [{i}] {preview}")

    if chapters:
        print(f"\n--- Sample: Chapter 1 ({chapters[0]['title']}) ---")
        ch1 = chapters[0]
        for i, p in enumerate(ch1["paragraphs"][:3]):
            preview = re.sub(r"<[^>]+>", "", p)[:120]
            print(f"  [{i}] {preview}...")

    if len(chapters) >= 2:
        print(f"\n--- Sample: Chapter 2 ({chapters[1]['title']}) ---")
        ch2 = chapters[1]
        for i, p in enumerate(ch2["paragraphs"][:3]):
            preview = re.sub(r"<[^>]+>", "", p)[:120]
            print(f"  [{i}] {preview}...")

    if len(chapters) >= 3:
        print(f"\n--- Sample: Chapter 3 ({chapters[2]['title']}) ---")
        ch3 = chapters[2]
        for i, p in enumerate(ch3["paragraphs"][:3]):
            preview = re.sub(r"<[^>]+>", "", p)[:120]
            print(f"  [{i}] {preview}...")

    # --- Quality Checks ---
    print(f"\n--- Quality Checks ---")

    # Check all 3 chapters present
    expected_chapters = set(range(1, 4))
    actual_chapters = set(ch["chapter"] for ch in chapters)
    missing = expected_chapters - actual_chapters
    extra = actual_chapters - expected_chapters
    if missing:
        print(f"  ⚠ Missing chapters: {sorted(missing)}")
    if extra:
        print(f"  ⚠ Extra chapters: {sorted(extra)}")
    if not missing and not extra:
        print(f"  ✅ All 3 chapters present")

    # Check intro segments
    if intro_segments:
        print(
            f"  ✅ Front pages segment present ({len(intro_segments[0]['paragraphs'])} paragraphs)"
        )
    else:
        print(f"  ⚠ Front pages segment missing!")

    # Check empty chapters
    empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
    if empty_chapters:
        print(f"  ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
    else:
        print(f"  ✅ No empty chapters")

    # Check for "——The Author." in chapter 1
    if chapters:
        ch1_text = " ".join(chapters[0]["paragraphs"])
        if "The Author" in ch1_text:
            print(f"  ✅ '——The Author.' attribution found in Chapter 1")
        else:
            print(f"  ⚠ '——The Author.' attribution NOT found in Chapter 1")

    # Check chapter 2 has "(Devanāgarī)" line
    if len(chapters) >= 2:
        ch2_text = " ".join(chapters[1]["paragraphs"])
        if "Devan" in ch2_text:
            print(f"  ✅ '(Devanāgarī)' line found in Chapter 2")
        else:
            print(f"  ℹ '(Devanāgarī)' line not found in Chapter 2 (may be normal)")

    # Verify no Bhakti-rasāmṛta-sindhu content leaked in
    all_text = ""
    for ch in chapters:
        all_text += " ".join(ch["paragraphs"])
    if "Text 22" in all_text or "BHAKTI RASAMRITA" in all_text.upper():
        print(f"  ⚠ Content from Bhakti-rasāmṛta-sindhu may have leaked into MOG!")
    else:
        print(f"  ✅ No content leak from next work detected")

    # Check last chapter ends cleanly
    if chapters:
        last_ch = chapters[-1]
        last_para = last_ch["paragraphs"][-1] if last_ch["paragraphs"] else ""
        last_preview = re.sub(r"<[^>]+>", "", last_para)[:80]
        print(f'  ℹ Last paragraph of Ch {last_ch["chapter"]}: "{last_preview}..."')

    print(f"\nDone! ✨")


if __name__ == "__main__":
    main()