#!/usr/bin/env python3
"""Extract five "Essays on Śrī Caitanya-caritāmṛta" from others-before-1977.html.

Essay 1: "The Spiritual Masters"   (cc-essays, order 1, 1960)
Essay 2: "The Absolute Truth"      (cc-essays, order 2, 1957)
Essay 3: "The Causes of Lord Caitanya's Incarnation" (cc-essays, order 3, 1957)
Essay 4: "The Confidential Reasons for Lord Caitanya's Appearance" (cc-essays, order 4, 1957)
Essay 5: "The Glories of Lord Nityānanda" (cc-essays, order 5, 1957)

All five are plain English (NO ScaGoudy decoding needed).

Output:
    lib/jsons/essays/the-spiritual-masters.json
    lib/jsons/essays/the-absolute-truth.json
    lib/jsons/essays/causes-of-lord-caitanyas-incarnation.json
    lib/jsons/essays/confidential-reasons-for-lord-caitanyas-appearance.json
    lib/jsons/essays/glories-of-lord-nityananda.json

Usage:
    python3 scripts/cc_essays_extract.py
"""

import json
import os
import re
import sys

from bs4 import BeautifulSoup, Tag

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------


def get_css_class(tag: Tag) -> str:
    """Get the first CSS class of a tag, or empty string."""
    classes = tag.get("class", [])
    return classes[0] if classes else ""


def is_empty(tag: Tag) -> bool:
    """Check if a tag is empty or whitespace-only."""
    text = tag.get_text().strip()
    return not text or text == "\xa0"


def clean_html_content(tag: Tag) -> str:
    """Extract inner HTML from a tag, strip span class artifacts, preserve <i>, <b>, <br>."""
    inner = "".join(str(child) for child in tag.children)
    # Remove span tags with CSS classes (RTF conversion artifacts) but keep their content
    inner = re.sub(
        r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
    )
    # Remove any remaining bare <span> or </span> tags without classes
    inner = re.sub(r"</?span[^>]*>", "", inner)
    # Clean up Apple-tab-span leftovers (tab characters)
    inner = inner.replace("\t", " ")
    # Normalize whitespace (but preserve <br> tags)
    inner = re.sub(r"\s+", " ", inner).strip()
    return inner


def write_json(data, filepath):
    """Write data as formatted JSON to filepath, creating directories as needed."""
    os.makedirs(os.path.dirname(filepath), exist_ok=True)
    with open(filepath, "w", encoding="utf-8") as f:
        json.dump(data, f, ensure_ascii=False, indent=2)


def is_divider(tag: Tag) -> bool:
    """Check if a p9 tag is just a divider line (underscores) or a page header."""
    text = tag.get_text().strip()
    # Dividers are lines of underscores
    if re.match(r"^_+$", text):
        return True
    return False


def is_page_header(tag: Tag) -> bool:
    """Check if a p9 tag is a page header like 'Essays on Shri Chaitanya Charitamrita  Page 30'."""
    text = tag.get_text().strip()
    if re.search(r"Page\s+\d+", text):
        return True
    return False


def is_editor_note(tag: Tag) -> bool:
    """Check if a paragraph is an editor note like '[This file has been proofread...]'."""
    text = tag.get_text().strip()
    if text.startswith("[") and (
        "proofread" in text.lower() or "—Ed" in text or "- Ed" in text or ".—Ed" in text
    ):
        return True
    return False


# ---------------------------------------------------------------------------
# Essay configurations
# ---------------------------------------------------------------------------

AUTHOR = "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda"

ESSAYS = [
    {
        "slug": "the-spiritual-masters",
        "title": "The Spiritual Masters",
        "author": AUTHOR,
        "year": 1960,
        "group": "cc-essays",
        "order": 1,
    },
    {
        "slug": "the-absolute-truth",
        "title": "The Absolute Truth",
        "author": AUTHOR,
        "year": 1957,
        "group": "cc-essays",
        "order": 2,
    },
    {
        "slug": "causes-of-lord-caitanyas-incarnation",
        "title": "The Causes of Lord Caitanya's Incarnation",
        "author": AUTHOR,
        "year": 1957,
        "group": "cc-essays",
        "order": 3,
    },
    {
        "slug": "confidential-reasons-for-lord-caitanyas-appearance",
        "title": "The Confidential Reasons for Lord Caitanya's Appearance",
        "author": AUTHOR,
        "year": 1957,
        "group": "cc-essays",
        "order": 4,
    },
    {
        "slug": "glories-of-lord-nityananda",
        "title": "The Glories of Lord Nityānanda",
        "author": AUTHOR,
        "year": 1957,
        "group": "cc-essays",
        "order": 5,
    },
]

# ---------------------------------------------------------------------------
# Extraction logic
# ---------------------------------------------------------------------------


def parse_all_paragraphs(html_path: str) -> list[Tag]:
    """Parse the entire HTML and return all <p> tags."""
    with open(html_path, "r", encoding="utf-8") as f:
        content = f.read()
    soup = BeautifulSoup(content, "html.parser")
    return list(soup.find_all("p"))


def find_tag_index(tags: list[Tag], start_from: int, predicate) -> int | None:
    """Find index of first tag matching predicate, starting from start_from."""
    for i in range(start_from, len(tags)):
        if predicate(tags[i]):
            return i
    return None


def extract_body_paragraphs(
    tags: list[Tag], start_idx: int, end_idx: int, include_p9_content: bool = True
) -> list[str]:
    """Extract body paragraphs from tags[start_idx] to tags[end_idx] (exclusive).

    Includes p6 paragraphs (main body) and optionally p9 paragraphs that
    contain actual content (Sanskrit verses, section markers) but skips
    p9 paragraphs that are dividers or page headers.
    """
    paragraphs = []
    body_classes = {"p6"}
    if include_p9_content:
        body_classes.add("p9")

    for i in range(start_idx, end_idx):
        tag = tags[i]
        css = get_css_class(tag)

        if css not in body_classes:
            continue

        if is_empty(tag):
            continue

        # For p9 tags, skip dividers and page headers
        if css == "p9":
            if is_divider(tag) or is_page_header(tag):
                continue

        html = clean_html_content(tag)
        if html and html != "&nbsp;":
            paragraphs.append(html)

    return paragraphs


def extract_body_paragraphs_extended(
    tags: list[Tag],
    start_idx: int,
    end_idx: int,
    body_classes: set[str] | None = None,
    skip_editor_notes: bool = False,
) -> list[str]:
    """Extract body paragraphs from tags[start_idx] to tags[end_idx] (exclusive).

    Like extract_body_paragraphs but with configurable body classes and
    optional editor-note skipping.
    """
    if body_classes is None:
        body_classes = {"p6"}

    paragraphs = []

    for i in range(start_idx, end_idx):
        tag = tags[i]
        css = get_css_class(tag)

        if css not in body_classes:
            continue

        if is_empty(tag):
            continue

        # For p9 tags, skip dividers and page headers
        if css == "p9":
            if is_divider(tag) or is_page_header(tag):
                continue

        # Skip editor notes if requested
        if skip_editor_notes and is_editor_note(tag):
            continue

        html = clean_html_content(tag)
        if html and html != "&nbsp;":
            paragraphs.append(html)

    return paragraphs


def extract_essay_1(tags: list[Tag]) -> dict:
    """Extract 'The Spiritual Masters' essay.

    Start: <p class="p8"><b>(The Spiritual Masters).</b></p>
    Body: p6 and p9 paragraphs after start
    End: Next <p class="p10"> (contains [BTG 1960])
    """
    # Find the title tag
    start_idx = find_tag_index(
        tags,
        0,
        lambda t: get_css_class(t) == "p8" and "Spiritual Masters" in t.get_text(),
    )
    if start_idx is None:
        raise ValueError("Could not find Essay 1 start marker (p8 'Spiritual Masters')")

    # Find the end marker: next p10
    end_idx = find_tag_index(tags, start_idx + 1, lambda t: get_css_class(t) == "p10")
    if end_idx is None:
        raise ValueError("Could not find Essay 1 end marker (p10)")

    print(f"  Essay 1: tags[{start_idx}] → tags[{end_idx}] (exclusive)")

    # Body starts after the p8 title tag
    paragraphs = extract_body_paragraphs(tags, start_idx + 1, end_idx)

    essay = dict(ESSAYS[0])
    essay["paragraphs"] = paragraphs
    return essay


def extract_essay_2(tags: list[Tag]) -> dict:
    """Extract 'The Absolute Truth' essay.

    Start: <p class="p9">(Second Part)</p>
    Body: p6 paragraphs after start
    End: Next <p class="p11"> after start
    """
    # Find the start marker
    start_idx = find_tag_index(
        tags,
        0,
        lambda t: get_css_class(t) == "p9" and "(Second Part)" in t.get_text(),
    )
    if start_idx is None:
        raise ValueError("Could not find Essay 2 start marker (p9 '(Second Part)')")

    # Find the end marker: next p11
    end_idx = find_tag_index(tags, start_idx + 1, lambda t: get_css_class(t) == "p11")
    if end_idx is None:
        raise ValueError("Could not find Essay 2 end marker (p11)")

    print(f"  Essay 2: tags[{start_idx}] → tags[{end_idx}] (exclusive)")

    # Body starts after the "(Second Part)" header
    paragraphs = extract_body_paragraphs(tags, start_idx + 1, end_idx)

    essay = dict(ESSAYS[1])
    essay["paragraphs"] = paragraphs
    return essay


def extract_essay_3(tags: list[Tag]) -> dict:
    """Extract 'The Causes of Lord Caitanya's Incarnation' essay.

    Start: <p class="p9"> containing "Essays on Shri Chaitanya Charitamrita. III"
    Body: p6 paragraphs after start
    End: Next <p class="p11"> after start.
         The <p class="p9">____</p> divider is skipped.
         But the <p class="p6">Thus ends the III part...</p> IS included.
    """
    # Find the start marker — note the ". III" with period
    start_idx = find_tag_index(
        tags,
        0,
        lambda t: (
            get_css_class(t) == "p9"
            and "Essays on Shri Chaitanya Charitamrita. III" in t.get_text()
        ),
    )
    if start_idx is None:
        raise ValueError(
            "Could not find Essay 3 start marker (p9 'Essays on Shri Chaitanya Charitamrita. III')"
        )

    # Find the end marker: next p11
    end_idx = find_tag_index(tags, start_idx + 1, lambda t: get_css_class(t) == "p11")
    if end_idx is None:
        raise ValueError("Could not find Essay 3 end marker (p11)")

    print(f"  Essay 3: tags[{start_idx}] → tags[{end_idx}] (exclusive)")

    # Body starts after the header tag.
    # We collect p6 paragraphs. The divider (p9 with underscores) is skipped
    # automatically by extract_body_paragraphs. The "Thus ends the III part..."
    # paragraph is a p6, so it will be included.
    paragraphs = extract_body_paragraphs(tags, start_idx + 1, end_idx)

    essay = dict(ESSAYS[2])
    essay["paragraphs"] = paragraphs
    return essay


def extract_essay_4(tags: list[Tag]) -> dict:
    """Extract 'The Confidential Reasons for Lord Caitanya's Appearance' essay.

    Start: <p class="p30"> containing "4th Chapter"
    Body: p6 paragraphs after start (skip editor note, skip p4/p5/p29/p30 headers)
    End: Next <p class="p4"> after start (next chapter header "Ädi-lélä, Chapter 5")
    """
    # Find the start marker: p30 containing "4th Chapter"
    start_idx = find_tag_index(
        tags,
        0,
        lambda t: get_css_class(t) == "p30" and "4th Chapter" in t.get_text(),
    )
    if start_idx is None:
        raise ValueError("Could not find Essay 4 start marker (p30 '4th Chapter')")

    # Find the end marker: next p4 after the start (next chapter header)
    end_idx = find_tag_index(tags, start_idx + 1, lambda t: get_css_class(t) == "p4")
    if end_idx is None:
        raise ValueError("Could not find Essay 4 end marker (p4)")

    print(f"  Essay 4: tags[{start_idx}] → tags[{end_idx}] (exclusive)")

    # Collect p6 paragraphs, skipping editor notes
    paragraphs = extract_body_paragraphs_extended(
        tags,
        start_idx + 1,
        end_idx,
        body_classes={"p6"},
        skip_editor_notes=True,
    )

    essay = dict(ESSAYS[3])
    essay["paragraphs"] = paragraphs
    return essay


def extract_essay_5(tags: list[Tag]) -> dict:
    """Extract 'The Glories of Lord Nityānanda' essay.

    Start: <p class="p31"> containing "1.V" (i.e. "Shi Chaitanya Charitamrita 1.V")
    Body: p6 and p34 paragraphs after start (skip editor note, skip p4/p5/p31 headers,
          skip p9 divider underscores)
    End: Next <p class="p35"> after start ("Easy Journey To Other Planets")
    """
    # Find the start marker: p31 containing "1.V"
    start_idx = find_tag_index(
        tags,
        0,
        lambda t: get_css_class(t) == "p31" and "1.V" in t.get_text(),
    )
    if start_idx is None:
        raise ValueError("Could not find Essay 5 start marker (p31 containing '1.V')")

    # Find the end marker: next p35 after the start
    end_idx = find_tag_index(tags, start_idx + 1, lambda t: get_css_class(t) == "p35")
    if end_idx is None:
        raise ValueError("Could not find Essay 5 end marker (p35)")

    print(f"  Essay 5: tags[{start_idx}] → tags[{end_idx}] (exclusive)")

    # Collect p6 and p34 paragraphs, skipping editor notes and p9 dividers
    paragraphs = extract_body_paragraphs_extended(
        tags,
        start_idx + 1,
        end_idx,
        body_classes={"p6", "p34"},
        skip_editor_notes=True,
    )

    essay = dict(ESSAYS[4])
    essay["paragraphs"] = paragraphs
    return essay


# ---------------------------------------------------------------------------
# Quality checks
# ---------------------------------------------------------------------------


def quality_check(essay: dict, essay_num: int) -> None:
    """Run quality checks on an extracted essay."""
    paragraphs = essay["paragraphs"]
    title = essay["title"]

    print(f"\n--- Quality Checks: Essay {essay_num} ({title}) ---")

    # Check paragraph count is reasonable
    if len(paragraphs) < 5:
        print(f"  ⚠ Only {len(paragraphs)} paragraphs (suspiciously low)")
    elif len(paragraphs) > 150:
        print(f"  ⚠ {len(paragraphs)} paragraphs (suspiciously high)")
    else:
        print(f"  ✅ {len(paragraphs)} paragraphs (reasonable)")

    # Check for residual span class artifacts
    span_count = sum(1 for p in paragraphs if 'class="' in p and "<span" in p)
    if span_count == 0:
        print(f"  ✅ No residual span class artifacts found")
    else:
        print(f"  ⚠ {span_count} paragraphs still contain span class artifacts")

    # Check key terms
    all_text = " ".join(paragraphs)
    if "Krishna" in all_text or "Chaitanya" in all_text:
        print(f"  ✅ 'Krishna'/'Chaitanya' found (plain English, as expected)")
    else:
        print(f"  ⚠ Key terms not found")

    # Check that editor notes were excluded
    for p in paragraphs:
        plain = re.sub(r"<[^>]+>", "", p).strip()
        if plain.startswith("[") and ("proofread" in plain.lower() or "—Ed" in plain):
            print(f"  ⚠ Editor note leaked into paragraphs")
            break
    else:
        print(f"  ✅ No editor notes leaked into paragraphs")

    # Check that divider lines were excluded
    for p in paragraphs:
        plain = re.sub(r"<[^>]+>", "", p).strip()
        if re.match(r"^_+$", plain):
            print(f"  ⚠ Divider line leaked into paragraphs")
            break
    else:
        print(f"  ✅ No divider lines leaked into paragraphs")

    # Essay-specific checks
    if essay_num == 1:
        if "spiritual master" in all_text.lower():
            print(f"  ✅ 'spiritual master' found in text")
        else:
            print(f"  ⚠ 'spiritual master' not found")

        # Check that reference line [BTG 1960] was excluded
        for p in paragraphs:
            plain = re.sub(r"<[^>]+>", "", p).strip()
            if "BTG 1960" in plain and plain.startswith("["):
                print(f"  ⚠ Reference line '[BTG 1960]' leaked into paragraphs")
                break
        else:
            print(f"  ✅ Reference line properly excluded")

        # Check that title was excluded
        for p in paragraphs:
            plain = re.sub(r"<[^>]+>", "", p).strip()
            if (
                plain == "(The Spiritual Masters)."
                or plain == "(The Spiritual Masters)."
            ):
                print(f"  ⚠ Title line leaked into paragraphs")
                break
        else:
            print(f"  ✅ Title line properly excluded")

    elif essay_num == 2:
        if "impersonal Brahman" in all_text or "Absolute Truth" in all_text:
            print(f"  ✅ 'Absolute Truth' / 'impersonal Brahman' found")
        else:
            print(f"  ⚠ Key terms for Essay 2 not found")

        # Check "(Second Part)" header was excluded
        for p in paragraphs:
            plain = re.sub(r"<[^>]+>", "", p).strip()
            if plain == "(Second Part)":
                print(f"  ⚠ Header '(Second Part)' leaked into paragraphs")
                break
        else:
            print(f"  ✅ Header '(Second Part)' properly excluded")

    elif essay_num == 3:
        if "incarnation" in all_text.lower() or "descent" in all_text.lower():
            print(f"  ✅ 'incarnation'/'descent' found")
        else:
            print(f"  ⚠ Key terms for Essay 3 not found")

        # Check that "Thus ends the III part..." is included
        found_ending = False
        for p in paragraphs:
            if "Thus ends the III part" in p:
                found_ending = True
                break
        if found_ending:
            print(f"  ✅ 'Thus ends the III part...' paragraph included")
        else:
            print(f"  ⚠ 'Thus ends the III part...' paragraph missing")

    elif essay_num == 4:
        if "Radharani" in all_text or "confidential" in all_text.lower():
            print(f"  ✅ 'Radharani'/'confidential' found")
        else:
            print(f"  ⚠ Key terms for Essay 4 not found")

        # Check that the summary paragraph is included (last paragraph)
        found_summary = False
        for p in paragraphs:
            if "author then summarize" in p.lower():
                found_summary = True
                break
        if found_summary:
            print(f"  ✅ Summary paragraph ('The author then summarize...') included")
        else:
            print(f"  ⚠ Summary paragraph missing")

        # Check that the first body paragraph mentions "three principal purposes"
        if paragraphs:
            if "three principal purposes" in paragraphs[0]:
                print(f"  ✅ First paragraph mentions 'three principal purposes'")
            else:
                print(f"  ⚠ First paragraph doesn't mention 'three principal purposes'")

        # Check that p4/p5/p29/p30 header lines were excluded
        for p in paragraphs:
            plain = re.sub(r"<[^>]+>", "", p).strip()
            if "4th Chapter" in plain and len(plain) < 50:
                print(f"  ⚠ Header '4th Chapter' leaked into paragraphs")
                break
        else:
            print(f"  ✅ Header lines properly excluded")

    elif essay_num == 5:
        if "Nityananda" in all_text:
            print(f"  ✅ 'Nityananda' found")
        else:
            print(f"  ⚠ 'Nityananda' not found")

        # Check that the final p34 description paragraph is included
        if paragraphs:
            last_plain = re.sub(r"<[^>]+>", "", paragraphs[-1]).strip()
            if (
                "lusture" in last_plain
                or "turban" in last_plain
                or "wonderful feature" in last_plain
            ):
                print(f"  ✅ Final description paragraph (p34) included")
            else:
                print(f"  ⚠ Final description paragraph (p34) may be missing")
                print(f"     Last paragraph preview: {last_plain[:80]}...")

        # Check that the first body paragraph mentions "5th chapter"
        if paragraphs:
            if "5th chapter" in paragraphs[0].lower():
                print(f"  ✅ First paragraph mentions '5th chapter'")
            else:
                print(f"  ⚠ First paragraph doesn't mention '5th chapter'")

        # Check that p31 header was excluded
        for p in paragraphs:
            plain = re.sub(r"<[^>]+>", "", p).strip()
            if "Shi Chaitanya Charitamrita 1.V" in plain:
                print(
                    f"  ⚠ Header 'Shi Chaitanya Charitamrita 1.V' leaked into paragraphs"
                )
                break
        else:
            print(f"  ✅ Header lines properly excluded")

    # Sample output
    print(f"\n  First paragraph (preview):")
    if paragraphs:
        preview = re.sub(r"<[^>]+>", "", paragraphs[0])[:120]
        print(f"    {preview}...")
    print(f"  Last paragraph (preview):")
    if paragraphs:
        last = re.sub(r"<[^>]+>", "", paragraphs[-1])[:120]
        print(f"    {last}...")


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------


def main():
    base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html")
    output_dir = os.path.join(base_dir, "lib", "jsons", "essays")

    if not os.path.exists(html_path):
        print(f"ERROR: {html_path} not found!")
        sys.exit(1)

    print(f"Reading {html_path}...")
    tags = parse_all_paragraphs(html_path)
    print(f"  Parsed {len(tags)} <p> tags total\n")

    # Extract all five essays
    print("Locating essays...")
    essay_1 = extract_essay_1(tags)
    essay_2 = extract_essay_2(tags)
    essay_3 = extract_essay_3(tags)
    essay_4 = extract_essay_4(tags)
    essay_5 = extract_essay_5(tags)

    all_essays = [essay_1, essay_2, essay_3, essay_4, essay_5]

    # Write JSON files
    print(f"\nWriting JSON files...")
    for essay in all_essays:
        output_path = os.path.join(output_dir, f"{essay['slug']}.json")
        write_json(essay, output_path)
        print(f"  Written: {output_path}")

    # Quality checks
    for i, essay in enumerate(all_essays, 1):
        quality_check(essay, i)

    # Summary
    print(f"\n{'=' * 60}")
    print(f"  EXTRACTION SUMMARY")
    print(f"{'=' * 60}")
    for i, essay in enumerate(all_essays, 1):
        print(
            f"  Essay {i}: {essay['title']}"
            f"  ({len(essay['paragraphs'])} paragraphs, {essay['year']})"
        )
        print(
            f"           slug={essay['slug']}, group={essay['group']}, order={essay['order']}"
        )
    print(f"{'=' * 60}")
    print(f"  Total essays: {len(all_essays)}")
    print(f"  Total paragraphs: {sum(len(e['paragraphs']) for e in all_essays)}")
    print(f"\nDone! ✨")


if __name__ == "__main__":
    main()