#!/usr/bin/env python3
"""Extract Two Essays (1966) from other.html.

The book contains 2 essays:
  1. "Krishna The Reservoir Of Pleasure"
  2. "Who Is Crazy?"

These are plain English essays from 1966 — NO ScaGoudy encoding.

Output:
    lib/jsons/books/te/intro.json                              — Book metadata
    lib/jsons/books/te/essays/krishna-reservoir-of-pleasure.json
    lib/jsons/books/te/essays/who-is-crazy.json

Usage:
    python3 scripts/te_extract.py
"""

import json
import os
import re
import sys

from bs4 import BeautifulSoup, Tag

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------


def get_css_class(tag: Tag) -> str:
    """Get the first CSS class of a tag, or empty string."""
    classes = tag.get("class", [])
    return classes[0] if classes else ""


def is_empty(tag: Tag) -> bool:
    """Check if a tag is empty or whitespace-only."""
    text = tag.get_text().strip()
    return not text or text == "\xa0"


def clean_html_content(tag: Tag) -> str:
    """Extract inner HTML from a tag, strip span class artifacts, preserve <i>, <b>, <br>."""
    inner = "".join(str(child) for child in tag.children)
    # Remove span tags with CSS classes (RTF conversion artifacts)
    inner = re.sub(
        r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
    )
    # Clean up Apple-tab-span leftovers (tab characters)
    inner = inner.replace("\t", " ")
    # Normalize whitespace (but preserve <br> tags)
    inner = re.sub(r"\s+", " ", inner).strip()
    return inner


def write_json(data, filepath):
    """Write data as formatted JSON to filepath, creating directories as needed."""
    os.makedirs(os.path.dirname(filepath), exist_ok=True)
    with open(filepath, "w", encoding="utf-8") as f:
        json.dump(data, f, ensure_ascii=False, indent=2)


# ---------------------------------------------------------------------------
# CSS classes to SKIP entirely (ad block, structural headers, etc.)
# ---------------------------------------------------------------------------
SKIP_CLASSES = {"p12", "p121", "p122", "p13", "p123", "p124", "p23", "p18"}

# CSS classes that contain essay body content
BODY_CLASSES = {"p28", "p6", "p22", "p3"}


# ---------------------------------------------------------------------------
# Boundary detection
# ---------------------------------------------------------------------------


def find_te_boundaries(all_paragraphs: list[Tag]) -> tuple[int, int]:
    """Find the start and end paragraph indices for Two Essays content.

    Start: <p class="p12"><b>TWO ESSAYS</b></p>
    End: Next <p class="p12"> or <p class="p24"> or <p class="p34"> after our content,
         or just past the last essay paragraph.

    Returns (start_idx, end_idx) — exclusive end.
    """
    te_start = None
    te_end = None

    for i, p in enumerate(all_paragraphs):
        css = get_css_class(p)
        text = p.get_text().strip().upper()

        # Find the TWO ESSAYS header
        if css == "p12" and "TWO ESSAYS" in text:
            te_start = i
            continue

        # Once we've found the start, look for the end
        if te_start is not None and te_end is None:
            # A new p12 header means we've left Two Essays
            if css == "p12" and i > te_start:
                te_end = i
                break
            # Collier end markers
            if css in ("p24", "p34"):
                te_end = i
                break

    if te_start is None:
        raise ValueError("Could not find TWO ESSAYS header in HTML")

    # If we didn't find an explicit end marker, scan for the last body paragraph
    if te_end is None:
        # Walk backwards from the end to find the last relevant paragraph
        for i in range(len(all_paragraphs) - 1, te_start, -1):
            css = get_css_class(all_paragraphs[i])
            if css in BODY_CLASSES:
                te_end = i + 1
                break
        if te_end is None:
            te_end = len(all_paragraphs)

    return te_start, te_end


def find_essay_headings(
    all_paragraphs: list[Tag], start: int, end: int
) -> list[tuple[int, str]]:
    """Find p27 headings within the Two Essays range.

    Returns list of (paragraph_index, title_text).
    """
    headings = []
    for i in range(start, end):
        p = all_paragraphs[i]
        css = get_css_class(p)
        if css == "p27":
            title = p.get_text().strip()
            headings.append((i, title))
    return headings


# ---------------------------------------------------------------------------
# Paragraph extraction
# ---------------------------------------------------------------------------


def extract_essay_paragraphs(
    all_paragraphs: list[Tag], start: int, end: int
) -> list[str]:
    """Extract body paragraphs from the given range.

    Includes: p28, p6, p22, p3
    Skips: SKIP_CLASSES, p27 (headings), empty paragraphs
    """
    paragraphs = []

    for i in range(start, end):
        p = all_paragraphs[i]
        css = get_css_class(p)

        # Skip classes we don't want
        if css in SKIP_CLASSES:
            continue

        # Skip headings (already handled)
        if css == "p27":
            continue

        # Only accept body classes
        if css not in BODY_CLASSES:
            continue

        # Skip empty paragraphs
        if is_empty(p):
            continue

        html = clean_html_content(p)
        if html and html != "&nbsp;":
            paragraphs.append(html)

    return paragraphs


# ---------------------------------------------------------------------------
# Main extraction
# ---------------------------------------------------------------------------


def extract_te(soup: BeautifulSoup) -> dict:
    """Extract Two Essays from the parsed HTML.

    Returns a dict with:
        "essays": list of essay dicts
        "essay_listing": list of {slug, title, order} for intro.json
    """
    all_paragraphs = list(soup.find_all("p"))

    # Step 1: Find Two Essays boundaries
    te_start, te_end = find_te_boundaries(all_paragraphs)
    print(f"  Two Essays content: paragraphs {te_start} to {te_end}")
    print(f"  Total paragraphs in range: {te_end - te_start}")

    # Step 2: Find essay headings (p27)
    headings = find_essay_headings(all_paragraphs, te_start, te_end)
    print(f"  Found {len(headings)} essay headings")
    for pos, title in headings:
        print(f"    paragraph {pos}: {title}")

    if len(headings) != 2:
        print(f"  WARNING: Expected 2 essay headings, found {len(headings)}")

    # Step 3: Extract essays
    essays = []

    essay_configs = [
        {
            "slug": "krishna-reservoir-of-pleasure",
            "title": "Krishna The Reservoir Of Pleasure",
            "order": 1,
        },
        {
            "slug": "who-is-crazy",
            "title": "Who Is Crazy?",
            "order": 2,
        },
    ]

    for idx, (heading_pos, heading_title) in enumerate(headings):
        config = essay_configs[idx] if idx < len(essay_configs) else None

        # Content starts after the heading
        content_start = heading_pos + 1

        # Content ends at next heading or te_end
        if idx + 1 < len(headings):
            content_end = headings[idx + 1][0]
        else:
            content_end = te_end

        paragraphs = extract_essay_paragraphs(
            all_paragraphs, content_start, content_end
        )

        if config:
            slug = config["slug"]
            title = config["title"]
            order = config["order"]
        else:
            # Fallback for unexpected essays
            slug = re.sub(r"[^a-z0-9]+", "-", heading_title.lower()).strip("-")
            title = heading_title
            order = idx + 1

        essay = {
            "slug": slug,
            "title": title,
            "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
            "year": 1966,
            "group": "two-essays",
            "order": order,
            "paragraphs": paragraphs,
        }
        essays.append(essay)

    # Step 4: Build essay listing for intro.json
    essay_listing = [
        {"slug": e["slug"], "title": e["title"], "order": e["order"]} for e in essays
    ]

    return {
        "essays": essays,
        "essay_listing": essay_listing,
    }


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------


def main():
    base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    html_path = os.path.join(base_dir, "public", "html", "other.html")
    output_base = os.path.join(base_dir, "lib", "jsons", "books", "te")

    if not os.path.exists(html_path):
        print(f"ERROR: {html_path} not found!")
        sys.exit(1)

    print(f"Reading {html_path}...")
    with open(html_path, "r", encoding="utf-8") as f:
        soup = BeautifulSoup(f, "html.parser")

    print("Extracting Two Essays (1966)...\n")
    result = extract_te(soup)

    essays = result["essays"]
    essay_listing = result["essay_listing"]

    # --- Write intro.json ---
    intro_meta = {
        "group": "two-essays",
        "title": "Two Essays",
        "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
        "year": 1966,
        "description": "Two foundational essays: 'Krishna The Reservoir Of Pleasure' and 'Who Is Crazy?'",
        "essays": essay_listing,
    }
    intro_path = os.path.join(output_base, "intro.json")
    write_json(intro_meta, intro_path)
    print(f"\n  Written: {intro_path}")

    # --- Write essay files ---
    print(f"\n--- Essays ---")
    for essay in essays:
        filepath = os.path.join(output_base, "essays", f"{essay['slug']}.json")
        write_json(essay, filepath)
        print(
            f"  {essay['order']}. {essay['title']}: "
            f"{len(essay['paragraphs'])} paragraphs → {filepath}"
        )

    # --- Summary ---
    total_paras = sum(len(e["paragraphs"]) for e in essays)

    print(f"\n{'=' * 60}")
    print(f"  EXTRACTION SUMMARY")
    print(f"{'=' * 60}")
    print(f"  Essays:             {len(essays)}")
    print(f"  Total paragraphs:   {total_paras}")
    for e in essays:
        print(f"    {e['order']}. {e['title']}: {len(e['paragraphs'])} paragraphs")
    if essays:
        print(f"  Avg paras/essay:    {total_paras / len(essays):.1f}")
    print(f"  Output directory:   {output_base}/")

    # --- Samples ---
    for essay in essays:
        print(f"\n--- Sample: {essay['title']} ---")
        for i, p in enumerate(essay["paragraphs"][:3]):
            preview = re.sub(r"<[^>]+>", "", p)[:120]
            print(f"  [{i}] {preview}...")

    # --- Quality Checks ---
    print(f"\n--- Quality Checks ---")

    # Check both essays present
    expected_slugs = {"krishna-reservoir-of-pleasure", "who-is-crazy"}
    actual_slugs = {e["slug"] for e in essays}
    missing = expected_slugs - actual_slugs
    extra = actual_slugs - expected_slugs
    if missing:
        print(f"  ⚠ Missing essays: {missing}")
    if extra:
        print(f"  ⚠ Extra essays: {extra}")
    if not missing and not extra:
        print(f"  ✅ Both essays present")

    # Check empty essays
    empty_essays = [e["slug"] for e in essays if len(e["paragraphs"]) == 0]
    if empty_essays:
        print(f"  ⚠ Empty essays (0 paragraphs): {empty_essays}")
    else:
        print(f"  ✅ No empty essays")

    # Check paragraph counts are reasonable
    for e in essays:
        count = len(e["paragraphs"])
        if count < 5:
            print(
                f"  ⚠ Essay '{e['slug']}' has only {count} paragraphs (suspiciously low)"
            )
        elif count > 100:
            print(f"  ⚠ Essay '{e['slug']}' has {count} paragraphs (suspiciously high)")
        else:
            print(f"  ✅ Essay '{e['slug']}' has {count} paragraphs (reasonable)")

    # Check for residual span class artifacts
    span_count = 0
    for e in essays:
        for p in e["paragraphs"]:
            if 'class="' in p and "<span" in p:
                span_count += 1
    if span_count == 0:
        print(f"  ✅ No residual span class artifacts found")
    else:
        print(f"  ⚠ {span_count} paragraphs still contain span class artifacts")

    # Check no ScaGoudy encoding present (these are plain English essays)
    # But verify key words appear as plain English (not IAST)
    all_text = " ".join(" ".join(e["paragraphs"][:10]) for e in essays)
    if "Krishna" in all_text:
        print(f"  ✅ 'Krishna' found (plain English, as expected for 1966)")
    else:
        print(f"  ⚠ 'Krishna' not found in text")

    if "Bhagavad Gita" in all_text or "Bhagavad-Gita" in all_text:
        print(f"  ✅ 'Bhagavad Gita' found (plain English)")

    # Check for ad content leaking in
    ad_markers = ["ISKCON", "26 2nd Ave", "SRIMAD-BHAGAWATAM", "$3.25", "$16.80"]
    leaked_ads = []
    for e in essays:
        for p in e["paragraphs"]:
            plain = re.sub(r"<[^>]+>", "", p)
            for marker in ad_markers:
                if marker in plain:
                    leaked_ads.append((e["slug"], marker, plain[:80]))
                    break
    if leaked_ads:
        print(f"  ⚠ {len(leaked_ads)} paragraphs may contain ad content:")
        for slug, marker, preview in leaked_ads[:3]:
            print(f"    [{slug}] matched '{marker}': {preview}...")
    else:
        print(f"  ✅ No ad content detected in essays")

    # Check for SKIP_CLASSES content leaking in
    # (p18 paragraphs contain ad text like "BLISS—what was once only a word...")
    bliss_check = False
    for e in essays:
        for p in e["paragraphs"]:
            plain = re.sub(r"<[^>]+>", "", p)
            if plain.startswith("BLISS") or "SOUNDS OF KRISHNA CONSCIOUSNESS" in plain:
                bliss_check = True
                break
    if bliss_check:
        print(f"  ⚠ Ad text ('BLISS...') leaked into essay paragraphs")
    else:
        print(f"  ✅ Ad text properly excluded")

    # Show last paragraph of each essay
    print(f"\n--- Last Paragraph of Each Essay ---")
    for e in essays:
        if e["paragraphs"]:
            last = re.sub(r"<[^>]+>", "", e["paragraphs"][-1])[:150]
            print(f"  {e['title']}:")
            print(f"    ...{last}")

    print(f"\nDone! ✨")


if __name__ == "__main__":
    main()