#!/usr/bin/env python3
"""Extract letters from letters.html.

Each letter starts with a p7 tag containing "Letter to: <recipient>".
The header contains recipient, location, and date information.

Structure:
    - p7 = Letter header (recipient, dash, location, date)
    - p8 = Reference code (YY-MM-DD) and also postscript/closing lines
    - p9 = Recipient address block
    - p10 = Salutation ("Dear...")
    - p11 = Body paragraphs
    - p12 = Centered text (verses, centered quotes)
    - p13 = Closing/signature block (color: #408080)
    - p14 = Indented/numbered items
    - p16-p22, p28-p37 = Various body text styles

Output:
    lib/jsons/letters/letters.json  (array of all letters with metadata)

Usage:
    python3 scripts/letters_extract.py
"""

import json
import os
import re
import sys
from collections import Counter

from bs4 import BeautifulSoup, Tag

# ---------------------------------------------------------------------------
# ScaGoudy → IAST Unicode mapping
# ---------------------------------------------------------------------------
SCAGOUDY_TO_IAST = {
    "ä": "ā",
    "Ä": "Ā",
    "å": "ṛ",
    "Å": "Ṛ",
    "ñ": "ṣ",
    "Ñ": "Ṣ",
    "ö": "ṭ",
    "Ö": "Ṭ",
    "ë": "ṇ",
    "Ë": "Ṇ",
    "ò": "ḍ",
    "Ò": "Ḍ",
    "ç": "ś",
    "Ç": "Ś",
    "ì": "ṅ",
    "Ì": "Ṅ",
    "ï": "ñ",
    "Ï": "Ñ",
    "ü": "ū",
    "Ü": "Ū",
    "é": "ī",
    "É": "Ī",
    "à": "ṁ",
    "ù": "ḥ",
    "è": "ṝ",
    "ÿ": "ḷ",
}


def decode_scagoudy(text: str) -> str:
    """Replace ScaGoudy-encoded characters with proper IAST Unicode."""
    for src, dst in SCAGOUDY_TO_IAST.items():
        text = text.replace(src, dst)
    return text


def get_css_class(tag: Tag) -> str:
    classes = tag.get("class", [])
    return classes[0] if classes else ""


def inner_html(tag: Tag) -> str:
    """Get inner HTML of a tag, decode ScaGoudy, clean up spans."""
    html = "".join(str(child) for child in tag.children)
    html = decode_scagoudy(html)

    # <span class="s1">text</span> → keep inner text (reference style)
    html = re.sub(
        r'<span[^>]*class="s\d+"[^>]*>(.*?)</span>', r"\1", html, flags=re.DOTALL
    )
    # Remove any remaining spans but keep content
    html = re.sub(r"<span[^>]*>(.*?)</span>", r"\1", html, flags=re.DOTALL)
    # Remove tab characters
    html = html.replace("\t", " ")
    # Collapse multiple spaces
    html = re.sub(r"[ ]+", " ", html)
    html = html.strip()
    # Remove leading <br> tags
    html = re.sub(r"^(<br\s*/?>[\s]*)+", "", html)
    # Remove trailing <br> tags
    html = re.sub(r"(<br\s*/?>[\s]*)+$", "", html)
    return html


def write_json(data, path: str):
    os.makedirs(os.path.dirname(path), exist_ok=True)
    with open(path, "w", encoding="utf-8") as f:
        json.dump(data, f, indent=2, ensure_ascii=False)
        f.write("\n")


# ---------------------------------------------------------------------------
# Month name → number mapping
# ---------------------------------------------------------------------------
MONTH_MAP = {
    "january": 1,
    "february": 2,
    "march": 3,
    "april": 4,
    "may": 5,
    "june": 6,
    "july": 7,
    "august": 8,
    "september": 9,
    "october": 10,
    "november": 11,
    "december": 12,
    "jan": 1,
    "feb": 2,
    "mar": 3,
    "apr": 4,
    "jun": 6,
    "jul": 7,
    "aug": 8,
    "sep": 9,
    "sept": 9,
    "oct": 10,
    "nov": 11,
    "dec": 12,
}


def parse_date(date_str: str):
    """Parse a date string like '12 July, 1947' or 'May 1949'.

    Returns (year, month, day) where month and day can be None.
    """
    if not date_str:
        return None, None, None

    date_str = date_str.strip().rstrip(".")

    # Try "DD Month, YYYY" or "DD Month YYYY"
    m = re.match(r"(\d{1,2})\s+([A-Za-z]+),?\s*(\d{4})", date_str)
    if m:
        day = int(m.group(1))
        month_name = m.group(2).lower()
        year = int(m.group(3))
        month = MONTH_MAP.get(month_name)
        if year >= 1900 and year <= 2000:
            return year, month, day

    # Try "Month DD, YYYY" or "Month DD YYYY"
    m = re.match(r"([A-Za-z]+)\s+(\d{1,2}),?\s*(\d{4})", date_str)
    if m:
        month_name = m.group(1).lower()
        day = int(m.group(2))
        year = int(m.group(3))
        month = MONTH_MAP.get(month_name)
        if year >= 1900 and year <= 2000:
            return year, month, day

    # Try "Month, YYYY" or "Month YYYY"
    m = re.match(r"([A-Za-z]+),?\s*(\d{4})", date_str)
    if m:
        month_name = m.group(1).lower()
        year = int(m.group(2))
        month = MONTH_MAP.get(month_name)
        if month and year >= 1900 and year <= 2000:
            return year, month, None

    # Try just "YYYY"
    m = re.match(r"(\d{4})$", date_str.strip())
    if m:
        year = int(m.group(1))
        if year >= 1900 and year <= 2000:
            return year, None, None

    return None, None, None


def parse_header(tag: Tag):
    """Parse a p7 letter header tag.

    Expected format:
        Letter to: <recipient>
        —
        <location>
        <date>

    Or sometimes:
        Letter to: <recipient>
        <date>

    Returns (recipient, location, date_str).
    """
    text = tag.get_text().strip()
    lines = [l.strip() for l in text.split("\n") if l.strip()]

    recipient = None
    location = None
    date_str = None

    if not lines:
        return recipient, location, date_str

    # First line: "Letter to: <recipient>"
    first = lines[0]
    m = re.match(r"Letter to:\s*(.*)", first, re.IGNORECASE)
    if m:
        recipient = m.group(1).strip().rstrip(",.")
        recipient = decode_scagoudy(recipient)

    # Remove "—" lines
    content_lines = [l for l in lines[1:] if l != "—" and l != "-"]

    if len(content_lines) >= 2:
        # Last line is usually the date
        potential_date = content_lines[-1]
        year, month, day = parse_date(potential_date)
        if year:
            date_str = potential_date
            # Everything before the date is location
            loc_lines = content_lines[:-1]
            if loc_lines:
                location = loc_lines[0].strip().rstrip(",.")
                location = decode_scagoudy(location)
        else:
            # Maybe the second-to-last is location and last is something else
            # Try to find a date anywhere
            for cl in reversed(content_lines):
                yr, mo, dy = parse_date(cl)
                if yr:
                    date_str = cl
                    idx = content_lines.index(cl)
                    if idx > 0:
                        location = content_lines[0].strip().rstrip(",.")
                        location = decode_scagoudy(location)
                    break
    elif len(content_lines) == 1:
        # Could be just a date, or just a location
        potential_date = content_lines[0]
        year, month, day = parse_date(potential_date)
        if year:
            date_str = potential_date
        else:
            location = potential_date.strip().rstrip(",.")
            location = decode_scagoudy(location)

    return recipient, location, date_str


def make_slug(recipient: str, date_ref: str, year, month, day, index: int) -> str:
    """Create a unique slug for a letter."""
    parts = []

    # Date part
    if date_ref and re.match(r"\d{2}-\d{2}", date_ref):
        parts.append(date_ref)
    elif year:
        yr = str(year)[2:]  # Last 2 digits
        if month:
            if day:
                parts.append(f"{yr}-{month:02d}-{day:02d}")
            else:
                parts.append(f"{yr}-{month:02d}-00")
        else:
            parts.append(f"{yr}-00-00")
    else:
        parts.append(f"unknown-{index}")

    # Recipient part
    if recipient:
        # Slugify recipient name
        name = recipient.lower()
        name = re.sub(r"[^a-z0-9]+", "-", name)
        name = name.strip("-")[:40]
        parts.append(name)

    slug = "-".join(parts)
    return slug


# ---------------------------------------------------------------------------
# Main extraction
# ---------------------------------------------------------------------------


def main():
    base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    html_path = os.path.join(base_dir, "public", "html", "letters.html")
    output_dir = os.path.join(base_dir, "lib", "jsons", "letters")

    if not os.path.exists(html_path):
        print(f"ERROR: {html_path} not found!")
        sys.exit(1)

    print(f"Reading {html_path}...")
    with open(html_path, "r", encoding="utf-8") as f:
        content = f.read()

    soup = BeautifulSoup(content, "html.parser")
    all_tags = soup.find_all("p")
    print(f"  Parsed {len(all_tags)} <p> tags total")

    # ---------------------------------------------------------------------------
    # Phase 1: Find all letter boundaries
    # ---------------------------------------------------------------------------
    letter_starts = []
    for i, tag in enumerate(all_tags):
        cls = get_css_class(tag)
        text = tag.get_text().strip()
        if cls == "p7" and text.startswith("Letter to:"):
            letter_starts.append(i)

    print(f"\n  Found {len(letter_starts)} letters")

    # ---------------------------------------------------------------------------
    # Phase 2: Extract each letter
    # ---------------------------------------------------------------------------

    # Classes that are just empty spacers
    SKIP_CLASSES = {"p2", "p15", "p23", "p24", "p26"}

    # p12 = centered (verse-like)
    CENTERED_CLASSES = {"p12", "p18", "p19", "p20", "p21", "p27"}

    # p13, p34, p38 = signature/closing blocks (colored)
    SIGNATURE_CLASSES = {"p13", "p34", "p38"}

    letters = []
    slug_counts = Counter()

    for li, start_idx in enumerate(letter_starts):
        # Determine end boundary
        end_idx = (
            letter_starts[li + 1] if li + 1 < len(letter_starts) else len(all_tags)
        )

        # Parse header
        header_tag = all_tags[start_idx]
        recipient, location, date_str = parse_header(header_tag)

        # Parse date into components
        year, month, day = parse_date(date_str) if date_str else (None, None, None)

        # Find date reference code (first p8 after header that matches YY-MM-DD)
        date_ref = None
        salutation = None
        paragraphs = []
        first_p8_seen = False

        for i in range(start_idx + 1, end_idx):
            tag = all_tags[i]
            cls = get_css_class(tag)
            text = tag.get_text().strip()

            if cls in SKIP_CLASSES:
                continue
            if not text:
                continue

            # p8 = reference code or body text
            if cls == "p8":
                if not first_p8_seen:
                    # First p8 after header — check if it's a date reference
                    if re.match(r"\d{2}-\d{2}", text):
                        date_ref = text.strip()
                        first_p8_seen = True

                        # Try to extract year/month/day from ref if not already parsed
                        if not year and date_ref:
                            ref_parts = date_ref.split("-")
                            if len(ref_parts) >= 3:
                                try:
                                    yr = int(ref_parts[0])
                                    mo = int(ref_parts[1])
                                    dy = int(ref_parts[2])
                                    year = 1900 + yr if yr < 100 else yr
                                    month = mo if 1 <= mo <= 12 else None
                                    day = dy if 1 <= dy <= 31 else None
                                except ValueError:
                                    pass
                        continue
                    else:
                        first_p8_seen = True
                        # Not a date ref — treat as body text
                        html = inner_html(tag)
                        if html:
                            paragraphs.append(html)
                        continue
                else:
                    # Subsequent p8 tags — body/closing text
                    html = inner_html(tag)
                    if html:
                        paragraphs.append(html)
                    continue

            # p9 = recipient address block
            if cls == "p9":
                html = inner_html(tag)
                if html:
                    paragraphs.append(html)
                continue

            # p10 = salutation
            if cls == "p10":
                salutation = decode_scagoudy(text)
                html = inner_html(tag)
                if html:
                    paragraphs.append(html)
                continue

            # Centered classes (verse blocks)
            if cls in CENTERED_CLASSES:
                html = inner_html(tag)
                if html:
                    # Check if it has line breaks (verse-like)
                    if "<br" in html:
                        html = f'<div style="text-align:center">{html}</div>'
                    paragraphs.append(html)
                continue

            # Signature/closing blocks
            if cls in SIGNATURE_CLASSES:
                html = inner_html(tag)
                if html:
                    paragraphs.append(html)
                continue

            # All other body text classes (p11, p14, p16, p22, p28, p29, etc.)
            html = inner_html(tag)
            if html and html != "<br>":
                # Check for verse-like content
                stripped = html.strip()
                if (
                    stripped.startswith("<i>")
                    and "<br" in stripped
                    and '<div style="text-align:center">' not in stripped
                ):
                    plain = re.sub(r"</?[bi]>", "", stripped)
                    lines = [
                        l.strip() for l in re.split(r"<br\s*/?>", plain) if l.strip()
                    ]
                    if len(lines) >= 2:
                        html = f'<div style="text-align:center">{stripped}</div>'

                paragraphs.append(html)

        # Clean recipient name — strip trailing punctuation
        if recipient:
            recipient = recipient.rstrip(":,.; ").strip()
            # Also clean " :" (space before colon)
            recipient = re.sub(r"\s+:$", "", recipient).strip()

        # Split multi-recipient strings into individual names
        # Patterns: "Mukunda, Janaki" or "Mukunda and Janaki"
        recipient_display = recipient or "Unknown"
        if recipient:
            # Split by comma, semicolon, " and ", " & "
            parts = re.split(r"\s*[,;]\s*|\s+and\s+|\s+&\s+", recipient)
            recipients_list = []
            for p in parts:
                name = p.strip().rstrip(":,.; ").strip()
                # Clean " :" pattern
                name = re.sub(r"\s+:$", "", name).strip()
                if name:
                    recipients_list.append(name)
            if not recipients_list:
                recipients_list = ["Unknown"]
        else:
            recipients_list = ["Unknown"]

        # Build slug
        slug = make_slug(recipient_display, date_ref, year, month, day, li)

        # Ensure unique slug
        if slug in slug_counts:
            slug_counts[slug] += 1
            slug = f"{slug}-{slug_counts[slug]}"
        else:
            slug_counts[slug] = 1

        # Normalize location
        if location:
            # Clean up common variants
            location = location.rstrip(",.")
            # Normalize "Vrndavana" variants
            if location.lower() in ("vrndavana", "vrndävana"):
                location = "Vrindaban"

        letter = {
            "slug": slug,
            "recipients": recipients_list,
            "recipientDisplay": recipient_display,
            "date": date_str,
            "dateRef": date_ref,
            "year": year,
            "month": month,
            "day": day,
            "location": location,
            "salutation": salutation,
            "paragraphs": paragraphs,
        }
        letters.append(letter)

    print(f"\n  Extracted {len(letters)} letters")

    # ---------------------------------------------------------------------------
    # Phase 3: Statistics
    # ---------------------------------------------------------------------------

    # Year distribution
    year_counts = Counter()
    for letter in letters:
        if letter["year"]:
            year_counts[letter["year"]] += 1
    print(f"\n  Year distribution:")
    for yr in sorted(year_counts.keys()):
        if 1940 <= yr <= 1980:
            print(f"    {yr}: {year_counts[yr]} letters")

    # Location distribution (top 20)
    loc_counts = Counter()
    for letter in letters:
        if letter["location"]:
            loc_counts[letter["location"]] += 1
    print(f"\n  Top 20 locations:")
    for loc, count in loc_counts.most_common(20):
        print(f"    {loc}: {count}")

    # Recipient distribution (by individual name, not group)
    recip_counts = Counter()
    for letter in letters:
        for r in letter["recipients"]:
            recip_counts[r] += 1
    print(f"\n  Top 20 recipients (individual):")
    for recip, count in recip_counts.most_common(20):
        print(f"    {recip}: {count}")

    # Letters with missing data
    no_year = sum(1 for l in letters if not l["year"])
    no_location = sum(1 for l in letters if not l["location"])
    no_date = sum(1 for l in letters if not l["date"])
    no_recipient = sum(1 for l in letters if l["recipients"] == ["Unknown"])
    print(f"\n  Missing data:")
    print(f"    No year: {no_year}")
    print(f"    No location: {no_location}")
    print(f"    No date: {no_date}")
    print(f"    No recipient (Unknown): {no_recipient}")

    # Unique counts
    all_individual_recipients = set()
    for l in letters:
        for r in l["recipients"]:
            all_individual_recipients.add(r)
    unique_recipients = len(all_individual_recipients)
    unique_locations = len(set(l["location"] for l in letters if l["location"]))
    unique_years = len(set(l["year"] for l in letters if l["year"]))
    print(f"\n  Unique counts:")
    print(f"    Recipients: {unique_recipients}")
    print(f"    Locations: {unique_locations}")
    print(f"    Years: {unique_years}")

    # ---------------------------------------------------------------------------
    # Phase 4: Write output
    # ---------------------------------------------------------------------------

    # Write individual letter files grouped by year for manageable sizes
    years_with_letters = sorted(
        set(l["year"] for l in letters if l["year"] and 1940 <= l["year"] <= 1980)
    )

    # Write a summary/index file
    index_data = {
        "totalLetters": len(letters),
        "uniqueRecipients": unique_recipients,
        "uniqueLocations": unique_locations,
        "years": [],
        "topRecipients": [
            {"name": r, "count": c} for r, c in recip_counts.most_common(50)
        ],
        "topLocations": [
            {"name": l, "count": c} for l, c in loc_counts.most_common(50)
        ],
    }

    for yr in years_with_letters:
        yr_letters = [l for l in letters if l["year"] == yr]
        index_data["years"].append(
            {
                "year": yr,
                "count": len(yr_letters),
            }
        )

    # Also count letters without a valid year
    undated = [
        l for l in letters if not l["year"] or l["year"] < 1940 or l["year"] > 1980
    ]
    if undated:
        index_data["years"].append(
            {
                "year": 0,
                "count": len(undated),
            }
        )

    write_json(index_data, os.path.join(output_dir, "index.json"))
    print(f"\n  Written: index.json")

    # Write letters grouped by year (one JSON file per year)
    for yr in years_with_letters:
        yr_letters = [l for l in letters if l["year"] == yr]
        yr_letters.sort(key=lambda l: (l["month"] or 0, l["day"] or 0))
        write_json(yr_letters, os.path.join(output_dir, f"{yr}.json"))

    # Write undated letters
    if undated:
        write_json(undated, os.path.join(output_dir, "undated.json"))

    print(f"  Written: {len(years_with_letters)} year files + undated.json")

    # ---------------------------------------------------------------------------
    # Phase 5: Quality checks
    # ---------------------------------------------------------------------------
    print(f"\n{'=' * 60}")
    print(f"  QUALITY CHECKS")
    print(f"{'=' * 60}")

    # Check for ScaGoudy remnants
    sca_chars = {"å", "ë", "ö"}
    sca_remnants = 0
    for letter in letters[:100]:  # Check first 100
        for p in letter["paragraphs"]:
            if any(c in p for c in sca_chars):
                sca_remnants += 1
                break
    if sca_remnants == 0:
        print(f"  ✅ No ScaGoudy remnants detected (checked first 100)")
    else:
        print(f"  ⚠ {sca_remnants} letters may have ScaGoudy remnants")

    # Check empty letters
    empty = [l for l in letters if len(l["paragraphs"]) == 0]
    if empty:
        print(f"  ⚠ {len(empty)} empty letters (no paragraphs)")
    else:
        print(f"  ✅ No empty letters")

    # Check duplicate slugs
    slug_list = [l["slug"] for l in letters]
    dupes = len(slug_list) - len(set(slug_list))
    if dupes:
        print(f"  ⚠ {dupes} duplicate slugs")
    else:
        print(f"  ✅ No duplicate slugs")

    # Verse count
    verse_count = 0
    for letter in letters:
        for p in letter["paragraphs"]:
            if '<div style="text-align:center">' in p:
                verse_count += 1
    print(f"  ℹ️  {verse_count} center-aligned verse paragraphs")

    # ---------------------------------------------------------------------------
    # Summary
    # ---------------------------------------------------------------------------
    print(f"\n{'=' * 60}")
    print(f"  EXTRACTION SUMMARY")
    print(f"{'=' * 60}")
    print(f"  Total letters: {len(letters)}")
    print(f"  Unique recipients: {unique_recipients}")
    print(f"  Unique locations: {unique_locations}")
    print(f"  Year range: {min(years_with_letters)}–{max(years_with_letters)}")
    print(f"  Total paragraphs: {sum(len(l['paragraphs']) for l in letters)}")
    print(f"{'=' * 60}")
    print(f"\nDone! ✨")


if __name__ == "__main__":
    main()