#!/usr/bin/env python3
"""Extract Śrī Caitanya-bhāgavata Commentary from others-before-1977.html.

The book has 4 chapters (no intro segments):
  1. Chapter 1
  2. Lord Caitanya's Appearance
  3. Chapter 3
  4. Chapter 4

Source: public/html/others-before-1977.html, lines ~4139–4180.
  Start: <p class="p3"><b>Çré Caitanya-bhägavata-Commentary</b></p>
  End:   <p class="p2">Bhagaväner kathä [Original Documents]</p> (stop before)

Chapter boundaries: <p class="p17"> tags containing "Caitanya-bhägavata"

CSS classes:
  p3  — Book title header (skip)
  p17 — Chapter bookmark headers (boundary detection, skip from output)
  p5  — Chapter title headings (skip — we derive titles ourselves)
  p6  — Body paragraphs (MAIN content)
  p38 — Body paragraph variant
  p55 — Body paragraph variant (last paragraph of chapter 4)
  p9  — End-of-chapter markers like "(End Chapter One)" — include

Output:
    lib/jsons/books/cbhc/intro.json          — Book metadata
    lib/jsons/books/cbhc/chapters/01.json     — Chapter 1
    lib/jsons/books/cbhc/chapters/02.json     — Chapter 2
    lib/jsons/books/cbhc/chapters/03.json     — Chapter 3
    lib/jsons/books/cbhc/chapters/04.json     — Chapter 4

Usage:
    python3 scripts/cbhc_extract.py
"""

import json
import os
import re
import sys

from bs4 import BeautifulSoup, Tag

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------


def decode_html_content(tag: Tag) -> str:
    """Extract inner HTML from a tag, preserve formatting tags, strip spans."""
    inner = "".join(str(child) for child in tag.children)
    # Remove span tags with CSS classes (RTF conversion artifacts)
    inner = re.sub(
        r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
    )
    # Clean up Apple-tab-span leftovers (tab characters)
    inner = inner.replace("\t", " ")
    # Normalize whitespace
    inner = re.sub(r"\s+", " ", inner).strip()
    return inner


def decode_text(tag: Tag) -> str:
    """Get plain text from a tag."""
    return tag.get_text().strip()


def get_css_class(tag: Tag) -> str:
    """Get the first CSS class of a tag, or empty string."""
    classes = tag.get("class", [])
    return classes[0] if classes else ""


def is_empty(tag: Tag) -> bool:
    """Check if a tag is empty or whitespace-only."""
    text = tag.get_text().strip()
    return not text or text == "\xa0"


# ---------------------------------------------------------------------------
# CSS class categorization for CBHC
# ---------------------------------------------------------------------------

# Book title header
BOOK_TITLE_CLASS = "p3"

# Chapter bookmark headers (used for boundary detection)
CHAPTER_BOOKMARK_CLASS = "p17"

# Chapter title (bold, e.g., "Chapter 1") — skip from body
CHAPTER_TITLE_CLASS = "p5"

# Body paragraph classes
BODY_CLASSES = frozenset(["p6", "p38", "p55"])

# End-of-chapter markers — include as a paragraph
END_MARKER_CLASS = "p9"

# Classes to skip entirely
SKIP_CLASSES = frozenset(["p43", "p24"])


# ---------------------------------------------------------------------------
# Boundary detection
# ---------------------------------------------------------------------------


def find_cbhc_boundaries(all_paragraphs: list) -> tuple:
    """Find the start and end paragraph indices of the CBHC content.

    Start: <p class="p3"> containing "Caitanya-bhägavata-Commentary"
    End:   <p class="p2"> containing "Bhagaväner kathä" (next work)
    """
    start_idx = None
    end_idx = len(all_paragraphs)

    for i, p in enumerate(all_paragraphs):
        cls = get_css_class(p)
        text = decode_text(p)

        # Find start: p3 containing "Caitanya-bhägavata"
        if start_idx is None:
            if cls == BOOK_TITLE_CLASS and "Caitanya-bh" in text:
                start_idx = i
                continue
            continue

        # Find end: p2 containing "Bhagaväner" (next work boundary)
        if cls == "p2" and "Bhag" in text and "kath" in text:
            end_idx = i
            break

    if start_idx is None:
        print(
            "ERROR: Could not find 'Çré Caitanya-bhägavata-Commentary' "
            "in others-before-1977.html!"
        )
        sys.exit(1)

    return start_idx, end_idx


def find_chapter_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
    """Find all CBHC chapter bookmark tags (p17) in the CBHC range.

    Returns list of (paragraph_index, raw_title_text).
    """
    bookmarks = []
    for i in range(start_idx, end_idx):
        p = all_paragraphs[i]
        cls = get_css_class(p)
        if cls == CHAPTER_BOOKMARK_CLASS:
            text = decode_text(p).strip()
            if text and "Caitanya-bh" in text:
                bookmarks.append((i, text))
    return bookmarks


# ---------------------------------------------------------------------------
# Paragraph extraction
# ---------------------------------------------------------------------------


def extract_paragraphs(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
    """Extract body paragraphs for a chapter section.

    Collects p6, p38, p55, and p9 (end markers).
    Skips p3 (book header), p17 (bookmarks), p5 (chapter titles), p2, empties.
    """
    result = []

    for i in range(start_idx, end_idx):
        p = all_paragraphs[i]
        cls = get_css_class(p)

        # Skip empty paragraphs
        if is_empty(p):
            continue

        # Skip known skip classes
        if cls in SKIP_CLASSES:
            continue

        # Skip book title header
        if cls == BOOK_TITLE_CLASS:
            continue

        # Skip chapter bookmark headers
        if cls == CHAPTER_BOOKMARK_CLASS:
            continue

        # Skip chapter title headings (p5)
        if cls == CHAPTER_TITLE_CLASS:
            continue

        # Skip p2 (TOC entries)
        if cls == "p2":
            continue

        # Body paragraphs
        if cls in BODY_CLASSES:
            html = decode_html_content(p)
            if html:
                result.append(html)
            continue

        # End-of-chapter markers (p9) — include as a paragraph
        if cls == END_MARKER_CLASS:
            html = decode_html_content(p)
            if html:
                result.append(html)
            continue

        # Fallback: try to extract content from unknown classes
        html = decode_html_content(p)
        if html:
            print(f"    ⚠ Unknown class '{cls}' at paragraph {i}: {html[:80]}...")
            result.append(html)

    return result


# ---------------------------------------------------------------------------
# Main extraction logic
# ---------------------------------------------------------------------------


def extract_cbhc(soup: BeautifulSoup) -> dict:
    """Extract the entire Śrī Caitanya-bhāgavata Commentary from the parsed HTML.

    Returns a dict with:
        "chapters": list of chapter dicts
        "chapter_listing": list of {number, title} for intro.json
    """
    all_paragraphs = list(soup.find_all("p"))

    # Step 1: Find CBHC boundaries
    cbhc_start, cbhc_end = find_cbhc_boundaries(all_paragraphs)
    print(f"  CBHC content: paragraphs {cbhc_start} to {cbhc_end}")
    print(f"  Total paragraphs in range: {cbhc_end - cbhc_start}")

    # Step 2: Find all chapter bookmarks (p17)
    bookmarks = find_chapter_bookmarks(all_paragraphs, cbhc_start, cbhc_end)
    print(f"  Found {len(bookmarks)} CBHC chapter bookmarks:")
    for pos, title in bookmarks:
        print(f"    Paragraph {pos}: {title}")

    # Step 3: Map bookmarks to chapters with proper titles
    # Expected: 4 chapters
    # We derive proper titles from the p5 heading that follows each bookmark
    chapter_titles_map = {
        1: "Chapter 1",
        2: "Lord Caitanya's Appearance",
        3: "Chapter 3",
        4: "Chapter 4",
    }

    chapter_bookmarks = []  # (para_idx, chapter_num, normalized_title)
    for idx, (pos, _raw_title) in enumerate(bookmarks):
        ch_num = idx + 1
        ch_title = chapter_titles_map.get(ch_num, f"Chapter {ch_num}")
        chapter_bookmarks.append((pos, ch_num, ch_title))

    print(f"\n  Mapped {len(chapter_bookmarks)} chapters:")
    for pos, num, title in chapter_bookmarks:
        print(f"    Chapter {num}: {title} (paragraph {pos})")

    # Step 4: Extract chapters
    chapters = []
    for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_bookmarks):
        # Content starts after the bookmark line (skip p17 and p5 heading)
        content_start = ch_pos + 1

        # Content ends at next chapter bookmark or cbhc_end
        if idx + 1 < len(chapter_bookmarks):
            content_end = chapter_bookmarks[idx + 1][0]
        else:
            content_end = cbhc_end

        paragraphs = extract_paragraphs(all_paragraphs, content_start, content_end)

        ch = {
            "book": "cbhc",
            "chapter": ch_num,
            "title": ch_title,
            "type": "chapter",
            "paragraphs": paragraphs,
        }
        chapters.append(ch)

    # Step 5: Build chapter listing for intro.json
    chapter_listing = [
        {"number": ch["chapter"], "title": ch["title"]} for ch in chapters
    ]

    return {
        "chapters": chapters,
        "chapter_listing": chapter_listing,
    }


# ---------------------------------------------------------------------------
# Output
# ---------------------------------------------------------------------------


def write_json(data, filepath):
    """Write data as formatted JSON to filepath, creating directories as needed."""
    os.makedirs(os.path.dirname(filepath), exist_ok=True)
    with open(filepath, "w", encoding="utf-8") as f:
        json.dump(data, f, ensure_ascii=False, indent=2)


def main():
    base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html")
    output_base = os.path.join(base_dir, "lib", "jsons", "books", "cbhc")

    if not os.path.exists(html_path):
        print(f"ERROR: {html_path} not found!")
        sys.exit(1)

    print(f"Reading {html_path}...")
    with open(html_path, "r", encoding="utf-8") as f:
        soup = BeautifulSoup(f, "html.parser")

    print("Extracting Śrī Caitanya-bhāgavata Commentary...\n")
    result = extract_cbhc(soup)

    chapters = result["chapters"]
    chapter_listing = result["chapter_listing"]

    # --- Write intro.json ---
    intro_meta = {
        "book": "cbhc",
        "title": "Śrī Caitanya-bhāgavata Commentary",
        "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
        "year": None,
        "description": (
            "Commentary on Śrī Caitanya-bhāgavata by Vṛndāvana dāsa Ṭhākura "
            "— covering the first four chapters of Lord Caitanya's pastimes."
        ),
        "totalChapters": 4,
        "chapters": chapter_listing,
    }
    intro_path = os.path.join(output_base, "intro.json")
    write_json(intro_meta, intro_path)
    print(f"\n  Written: {intro_path}")

    # --- Write chapters ---
    print(f"\n--- Chapters ---")
    for ch in chapters:
        ch_num_str = str(ch["chapter"]).zfill(2)
        filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
        write_json(ch, filepath)
        print(
            f"  Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
            f"{ch['title']}"
        )

    # --- Summary ---
    total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)

    print(f"\n{'=' * 60}")
    print(f"  EXTRACTION SUMMARY")
    print(f"{'=' * 60}")
    print(f"  Chapters:           {len(chapters)}")
    for ch in chapters:
        print(
            f"    Ch {ch['chapter']}: {len(ch['paragraphs'])} paragraphs — {ch['title']}"
        )
    print(f"  Total paragraphs:   {total_chapter_paras}")
    if chapters:
        print(f"  Avg paras/chapter:  {total_chapter_paras / len(chapters):.1f}")
    print(f"  Output directory:   {output_base}/")

    # --- Samples ---
    for ch in chapters:
        ch_num = ch["chapter"]
        print(f"\n--- Sample: Chapter {ch_num} ({ch['title']}) ---")
        for i, p in enumerate(ch["paragraphs"][:3]):
            preview = re.sub(r"<[^>]+>", "", p)[:120]
            print(f"  [{i}] {preview}...")
        if len(ch["paragraphs"]) > 3:
            print(f"  ... ({len(ch['paragraphs']) - 3} more paragraphs)")

    # --- Quality Checks ---
    print(f"\n--- Quality Checks ---")

    # Check all 4 chapters present
    expected_chapters = set(range(1, 5))
    actual_chapters = set(ch["chapter"] for ch in chapters)
    missing = expected_chapters - actual_chapters
    extra = actual_chapters - expected_chapters
    if missing:
        print(f"  ⚠ Missing chapters: {sorted(missing)}")
    if extra:
        print(f"  ⚠ Extra chapters: {sorted(extra)}")
    if not missing and not extra:
        print(f"  ✅ All 4 chapters present")

    # Check empty chapters
    empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
    if empty_chapters:
        print(f"  ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
    else:
        print(f"  ✅ No empty chapters")

    # Check chapter 1 has "(End Chapter One)"
    if chapters:
        ch1_text = " ".join(chapters[0]["paragraphs"])
        if "End Chapter One" in ch1_text:
            print(f"  ✅ '(End Chapter One)' end marker found in Chapter 1")
        else:
            print(f"  ⚠ '(End Chapter One)' end marker NOT found in Chapter 1")

    # Check chapter 2 mentions Lord Caitanya's appearance
    if len(chapters) >= 2:
        ch2_text = " ".join(chapters[1]["paragraphs"])
        if "appearance" in ch2_text.lower() or "born" in ch2_text.lower():
            print(f"  ✅ Chapter 2 contains appearance/birth content")
        else:
            print(f"  ⚠ Chapter 2 may not contain appearance content")

    # Check for content leak from next work (Bhagaväner kathä)
    all_text = ""
    for ch in chapters:
        all_text += " ".join(ch["paragraphs"])
    if "Original Documents" in all_text:
        print(f"  ⚠ Content from next work (Bhagaväner kathä) may have leaked!")
    else:
        print(f"  ✅ No content leak from next work detected")

    # Check last chapter ends cleanly
    if chapters:
        last_ch = chapters[-1]
        last_para = last_ch["paragraphs"][-1] if last_ch["paragraphs"] else ""
        last_preview = re.sub(r"<[^>]+>", "", last_para)[:100]
        print(f'  ℹ Last paragraph of Ch {last_ch["chapter"]}: "{last_preview}..."')

    print(f"\nDone! ✨")


if __name__ == "__main__":
    main()