#!/usr/bin/env python3
"""Extract Dialectic Spiritualism (1985) from other.html.

Hierarchical sub-chapter extraction: each philosopher sub-chapter
becomes its OWN chapter JSON file with a parent-child relationship.

Parent chapters (13 total with Roman numeral prefixes) are containers only.

Sub-chapter bookmarks use class p49 with "DS:" prefix.
Parent chapter bookmarks use class p13 with concatenated Roman numerals.

Output:
    lib/jsons/books/ds/intro.json          — Book metadata with hierarchical chapters
    lib/jsons/books/ds/intro/*.json         — Intro segments (front-pages, foreword, preface)
    lib/jsons/books/ds/chapters/*.json      — Sub-chapter files (1-1.json, 1-2.json, ...)

Usage:
    python3 scripts/ds_extract.py
"""

import json
import os
import re
import sys
from collections import defaultdict

from bs4 import BeautifulSoup, Tag

# ---------------------------------------------------------------------------
# ScaGoudy → IAST Unicode mapping
# ---------------------------------------------------------------------------
SCAGOUDY_TO_IAST = {
    "ä": "ā",
    "Ä": "Ā",
    "å": "ṛ",
    "Å": "Ṛ",
    "ñ": "ṣ",
    "Ñ": "Ṣ",
    "ö": "ṭ",
    "Ö": "Ṭ",
    "ë": "ṇ",
    "Ë": "Ṇ",
    "ò": "ḍ",
    "Ò": "Ḍ",
    "ç": "ś",
    "Ç": "Ś",
    "ì": "ṅ",
    "ï": "ñ",
    "ü": "ū",
    "Ü": "Ū",
    "é": "ī",
    "É": "Ī",
    "à": "ṁ",
    "ù": "ḥ",
    "è": "ṝ",
    "û": "ṁl",
    "ÿ": "ḷ",
}


def decode_scagoudy(text: str) -> str:
    """Replace ScaGoudy-encoded characters with proper IAST Unicode."""
    for src, dst in SCAGOUDY_TO_IAST.items():
        text = text.replace(src, dst)
    return text


def decode_html_content(tag: Tag) -> str:
    """Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
    inner = "".join(str(child) for child in tag.children)
    inner = decode_scagoudy(inner)
    # Remove span tags with CSS classes (RTF conversion artifacts)
    inner = re.sub(
        r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
    )
    # Clean up Apple-tab-span leftovers (tab characters)
    inner = inner.replace("\t", " ")
    # Normalize whitespace (but preserve <br> tags)
    inner = re.sub(r"\s+", " ", inner).strip()
    return inner


def decode_text(tag: Tag) -> str:
    """Get plain text from a tag with ScaGoudy decoding."""
    return decode_scagoudy(tag.get_text().strip())


def get_css_class(tag: Tag) -> str:
    """Get the first CSS class of a tag, or empty string."""
    classes = tag.get("class", [])
    return classes[0] if classes else ""


def is_empty(tag: Tag) -> bool:
    """Check if a tag is empty or whitespace-only."""
    text = tag.get_text().strip()
    return not text or text == "\xa0"


def is_dash_line(text: str) -> bool:
    """Return True if text is only dashes, spaces, and/or whitespace."""
    cleaned = text.strip()
    if not cleaned:
        return False
    return all(c in "-—–\u2014\u2013 \t*" for c in cleaned)


# ---------------------------------------------------------------------------
# CSS class constants
# ---------------------------------------------------------------------------

BOOK_HEADER_CLASS = "p12"

# Parent chapter bookmark class (p13) — Roman numeral concatenated with title
CHAPTER_BOOKMARK_CLASS = "p13"

# Sub-chapter bookmark class (p49) — "DS:Philosopher Name (dates)"
SUBCHAPTER_BOOKMARK_CLASS = "p49"

# Verse/quote classes — group consecutive into centered italic divs
VERSE_QUOTE_CLASSES = frozenset(["p24", "p66", "p121", "p134", "p157", "p163"])

# Body/dialogue classes — include as content
BODY_CLASSES = frozenset(
    [
        "p118",  # Main dialogue lines (bold speaker + text)
        "p156",  # Body/dialogue
        "p6",  # Body
        "p159",  # Body
        "p160",  # Body
        "p140",  # Body
        "p44",  # Verse translations
        "p23",  # Misc body
        "p162",  # Dialogue (bold formatting variant)
        "p161",  # Quote/verse translations
        "p112",  # Dialogue variant
        "p158",  # Quote
        "p56",  # Quote/translation
        "p55",  # TRANSLATION/PURPORT labels
    ]
)

# Front matter classes
FRONT_MATTER_CLASSES = frozenset(
    [
        "p5",  # Title/front pages
        "p25",  # Contact/front
        "p18",  # Front pages
        "p154",  # Back cover label
        "p155",  # Foreword heading
        "p26",  # Misc front
    ]
)

# Skip classes — empty or noise
SKIP_CLASSES = frozenset(["p15", "p153"])

# Parent chapter heading class (p27) — Roman numeral headings, philosopher headings
PARENT_HEADING_CLASS = "p27"

# Sub-chapter title class (p31) — duplicate of bookmark title
SUBCHAPTER_TITLE_CLASS = "p31"


# ---------------------------------------------------------------------------
# Roman numeral parsing
# ---------------------------------------------------------------------------

# Ordered longest-first to prevent partial matches
ROMAN_NUMERALS_ORDERED = [
    "XIII",
    "XII",
    "XI",
    "X",
    "IX",
    "VIII",
    "VII",
    "VI",
    "V",
    "IV",
    "III",
    "II",
    "I",
]

ROMAN_TO_ARABIC = {
    "I": 1,
    "II": 2,
    "III": 3,
    "IV": 4,
    "V": 5,
    "VI": 6,
    "VII": 7,
    "VIII": 8,
    "IX": 9,
    "X": 10,
    "XI": 11,
    "XII": 12,
    "XIII": 13,
}

# Clean chapter titles (pre-decoded IAST)
CLEAN_PARENT_TITLES = {
    1: "The Greek Foundation",
    2: "The Early Theists",
    3: "Scholasticism",
    4: "Renaissance Thought",
    5: "Rationalism",
    6: "British Empiricism",
    7: "German Idealism",
    8: "Evolutionary Naturalism",
    9: "Utilitarianism and Positivism",
    10: "Communism",
    11: "American Pragmatism",
    12: "Existentialism",
    13: "Psychoanalysis",
}


def parse_roman_chapter(text: str):
    """Parse a p13 bookmark text like 'ITHE GREEK FOUNDATION'.

    Returns (chapter_number, raw_title) or None if not a valid chapter heading.
    """
    text = text.strip()
    for roman in ROMAN_NUMERALS_ORDERED:
        if text.startswith(roman):
            remainder = text[len(roman) :]
            if remainder:
                return ROMAN_TO_ARABIC[roman], remainder.strip()
    return None


def parse_ds_sub_bookmark(text: str):
    """Parse a p49 bookmark text like 'DS:Socrates (470?-399 B.C.)'.

    Returns the title after 'DS:' or None if not a DS bookmark.
    """
    m = re.match(r"DS:(.+)", text.strip())
    if m:
        return m.group(1).strip()
    return None


# ---------------------------------------------------------------------------
# Boundary detection
# ---------------------------------------------------------------------------


def find_ds_boundaries(all_paragraphs: list) -> tuple:
    """Find the start and end paragraph indices of the DS content.

    Start: <p class="p12"> containing "Dialectic Spiritualism" and "1985"
    End: next <p class="p12"> after DS content.
    """
    start_idx = None
    end_idx = len(all_paragraphs)

    for i, p in enumerate(all_paragraphs):
        cls = get_css_class(p)
        text = p.get_text().strip()

        if (
            cls == BOOK_HEADER_CLASS
            and "Dialectic Spiritualism" in text
            and "1985" in text
        ):
            start_idx = i

        if start_idx is not None and i > start_idx and cls == BOOK_HEADER_CLASS:
            end_idx = i
            break

    if start_idx is None:
        print("ERROR: Could not find 'Dialectic Spiritualism (1985)' header!")
        sys.exit(1)

    return start_idx, end_idx


def find_chapter_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
    """Find all DS main chapter bookmark tags (p13) in range.

    Returns list of (paragraph_index, chapter_number, raw_title).
    """
    bookmarks = []
    for i in range(start_idx, end_idx):
        p = all_paragraphs[i]
        cls = get_css_class(p)
        if cls != CHAPTER_BOOKMARK_CLASS:
            continue
        text = decode_text(p)
        result = parse_roman_chapter(text)
        if result:
            ch_num, raw_title = result
            bookmarks.append((i, ch_num, raw_title))
    return bookmarks


def find_sub_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
    """Find all DS sub-chapter bookmark tags (p49 with 'DS:' prefix) in range.

    Returns list of (paragraph_index, title).
    """
    bookmarks = []
    for i in range(start_idx, end_idx):
        p = all_paragraphs[i]
        cls = get_css_class(p)
        if cls != SUBCHAPTER_BOOKMARK_CLASS:
            continue
        text = decode_text(p)
        title = parse_ds_sub_bookmark(text)
        if title is not None:
            bookmarks.append((i, title))
    return bookmarks


# ---------------------------------------------------------------------------
# Paragraph extraction
# ---------------------------------------------------------------------------


def _normalize_for_match(text: str) -> str:
    """Normalize text for fuzzy title matching: lowercase, collapse whitespace, strip quotes."""
    text = re.sub(r"\s+", " ", text.strip().lower())
    text = text.strip('""\u201c\u201d\'')
    return text


def extract_paragraphs(
    all_paragraphs: list,
    start_idx: int,
    end_idx: int,
    skip_bookmarks: bool = True,
    skip_p27: bool = True,
    skip_first_p31_matching: str = None,
    is_front_matter: bool = False,
) -> list:
    """Extract paragraphs for a section.

    Handles all CSS classes within DS content.
    Groups consecutive verse quote pādas (p24, p66, etc.) into centered italic divs.

    Args:
        all_paragraphs: full list of <p> tags
        start_idx: first paragraph index (inclusive)
        end_idx: last paragraph index (exclusive)
        skip_bookmarks: if True, skip p13 and p49 bookmark tags
        skip_p27: if True, skip the p27 parent chapter heading
        skip_first_p31_matching: if set, skip the FIRST p31 whose text matches
            this string (case-insensitive, normalized). Used to remove duplicate
            sub-chapter title headings.
        is_front_matter: if True, include front matter classes
    """
    result = []
    verse_accumulator = []
    first_p31_skipped = False

    def flush_verse():
        if verse_accumulator:
            joined = "<br>".join(verse_accumulator)
            result.append(f'<div style="text-align:center"><i>{joined}</i></div>')
            verse_accumulator.clear()

    for i in range(start_idx, end_idx):
        p = all_paragraphs[i]
        cls = get_css_class(p)

        # Skip empty paragraphs
        if is_empty(p):
            continue

        # Skip noise classes
        if cls in SKIP_CLASSES:
            continue

        # Skip book title header (p12)
        if cls == BOOK_HEADER_CLASS:
            continue

        # Skip bookmark classes (p13, p49) — already captured in metadata
        if skip_bookmarks:
            if cls == CHAPTER_BOOKMARK_CLASS:
                continue
            if cls == SUBCHAPTER_BOOKMARK_CLASS:
                continue

        # Skip p27 parent chapter heading if requested
        if skip_p27 and cls == PARENT_HEADING_CLASS:
            continue

        # Skip the first p31 that matches the sub-chapter title (duplicate heading)
        if (
            not first_p31_skipped
            and skip_first_p31_matching is not None
            and cls == SUBCHAPTER_TITLE_CLASS
        ):
            p31_text = _normalize_for_match(decode_text(p))
            target = _normalize_for_match(skip_first_p31_matching)
            if p31_text == target:
                first_p31_skipped = True
                continue

        # Check for dash lines — skip
        text = decode_text(p)
        if is_dash_line(text):
            continue

        # Sanskrit/Bengali verse quotes (p24, p66, p121, p134, p157, p163)
        # — accumulate into centered block
        if cls in VERSE_QUOTE_CLASSES:
            html = decode_html_content(p)
            if html:
                # Strip <i> tags for uniform formatting (we'll wrap in <i> at flush)
                stripped = re.sub(r"</?i>", "", html)
                verse_accumulator.append(stripped)
            continue

        # Non-verse paragraph — flush any accumulated verse first
        flush_verse()

        # Front matter classes
        if is_front_matter and cls in FRONT_MATTER_CLASSES:
            html = decode_html_content(p)
            if html:
                result.append(html)
            continue

        # Body/dialogue classes
        if cls in BODY_CLASSES:
            html = decode_html_content(p)
            if html and not is_dash_line(text):
                # Detect Sanskrit verses disguised as body paragraphs:
                # they start with <i> and contain <br> (multi-line italic = verse)
                stripped_html = html.strip()
                if stripped_html.startswith("<i>") and "<br" in stripped_html:
                    inner = re.sub(r"</?i>", "", stripped_html)
                    result.append(
                        f'<div style="text-align:center"><i>{inner}</i></div>'
                    )
                else:
                    result.append(html)
            continue

        # Sub-chapter title heading (p31) — include if not the duplicate
        if cls == SUBCHAPTER_TITLE_CLASS:
            html = decode_html_content(p)
            if html:
                result.append(html)
            continue

        # Front matter classes (if not in front matter mode, still include p155, etc.)
        if cls in FRONT_MATTER_CLASSES:
            html = decode_html_content(p)
            if html:
                stripped_html = html.strip()
                if stripped_html.startswith("<i>") and "<br" in stripped_html:
                    inner = re.sub(r"</?i>", "", stripped_html)
                    result.append(
                        f'<div style="text-align:center"><i>{inner}</i></div>'
                    )
                else:
                    result.append(html)
            continue

        # Any other class — try to include if it has content
        html = decode_html_content(p)
        if html and not is_dash_line(text):
            # Detect multi-line italic = verse
            stripped_html = html.strip()
            if stripped_html.startswith("<i>") and "<br" in stripped_html:
                inner = re.sub(r"</?i>", "", stripped_html)
                result.append(f'<div style="text-align:center"><i>{inner}</i></div>')
            else:
                result.append(html)

    # Flush any trailing verse
    flush_verse()

    return result


# ---------------------------------------------------------------------------
# Main extraction
# ---------------------------------------------------------------------------


def extract_ds(soup: BeautifulSoup) -> dict:
    """Extract the entire Dialectic Spiritualism from parsed HTML.

    Returns dict with:
        "intro_segments": list of intro segment dicts
        "sub_chapters": list of sub-chapter dicts
        "parent_chapters": list of parent chapter info for intro.json
    """
    all_paragraphs = list(soup.find_all("p"))

    # --- Step 1: Find DS boundaries ---
    ds_start, ds_end = find_ds_boundaries(all_paragraphs)
    print(f"  DS content: paragraphs {ds_start} to {ds_end}")
    print(f"  Total paragraphs in range: {ds_end - ds_start}")

    # --- Step 2: Find main chapter bookmarks (p13) ---
    chapter_bookmarks = find_chapter_bookmarks(all_paragraphs, ds_start, ds_end)
    print(f"  Found {len(chapter_bookmarks)} chapter bookmarks (p13)")
    for pos, ch_num, raw_title in chapter_bookmarks:
        clean_title = CLEAN_PARENT_TITLES.get(ch_num, raw_title)
        print(f"    Ch {ch_num}: {clean_title} (paragraph {pos})")

    # --- Step 3: Find all sub-chapter bookmarks (p49 with DS: prefix) ---
    all_sub_bookmarks = find_sub_bookmarks(all_paragraphs, ds_start, ds_end)
    print(f"  Found {len(all_sub_bookmarks)} sub-chapter bookmarks (p49 DS:)")
    for pos, title in all_sub_bookmarks:
        print(f"    paragraph {pos}: {title}")

    # --- Step 4: Classify sub-bookmarks as intro segments or philosopher sub-chapters ---
    # Known intro segments
    INTRO_SEGMENTS_MAP = {
        "Forward": {"slug": "foreword", "title": "Foreword", "order": 1},
        "Preface": {"slug": "preface", "title": "Preface", "order": 2},
    }

    intro_bookmark_list = []  # (pos, slug, title, order)
    philosopher_bookmarks = []  # (pos, title)

    for pos, title in all_sub_bookmarks:
        if title in INTRO_SEGMENTS_MAP:
            seg = INTRO_SEGMENTS_MAP[title]
            intro_bookmark_list.append((pos, seg["slug"], seg["title"], seg["order"]))
        else:
            philosopher_bookmarks.append((pos, title))

    print(f"\n  Intro bookmarks: {len(intro_bookmark_list)}")
    for pos, slug, title, order in intro_bookmark_list:
        print(f"    [{order}] {slug}: {title} (paragraph {pos})")

    print(f"  Philosopher bookmarks: {len(philosopher_bookmarks)}")

    # --- Step 5: Determine parent chapter for each philosopher ---
    # Build chapter boundary lookup: ch_num → (start_pos, end_pos)
    ch_bm_lookup = {}
    ch_nums_sorted = sorted(set(ch_num for _, ch_num, _ in chapter_bookmarks))

    for i, (pos, ch_num, raw_title) in enumerate(chapter_bookmarks):
        ch_bm_lookup[ch_num] = pos

    ch_end_lookup = {}
    for idx, ch_num in enumerate(ch_nums_sorted):
        if idx + 1 < len(ch_nums_sorted):
            next_ch = ch_nums_sorted[idx + 1]
            ch_end_lookup[ch_num] = ch_bm_lookup[next_ch]
        else:
            ch_end_lookup[ch_num] = ds_end

    # Assign each philosopher to its parent chapter based on position
    def find_parent_chapter(pos: int) -> int:
        """Find which parent chapter a position falls under."""
        result_ch = None
        for ch_num in ch_nums_sorted:
            ch_start = ch_bm_lookup[ch_num]
            if pos >= ch_start:
                result_ch = ch_num
            else:
                break
        return result_ch

    philosophers_by_parent = defaultdict(list)
    for pos, title in philosopher_bookmarks:
        parent = find_parent_chapter(pos)
        if parent is not None:
            philosophers_by_parent[parent].append((pos, title))
        else:
            print(f"  ⚠ Could not find parent chapter for: {title} at paragraph {pos}")

    # --- Step 6: Extract front pages ---
    intro_segments = []

    # Front pages: from DS header to first intro bookmark (DS:Forward)
    first_intro_pos = (
        intro_bookmark_list[0][0]
        if intro_bookmark_list
        else (chapter_bookmarks[0][0] if chapter_bookmarks else ds_end)
    )

    front_pages_paragraphs = extract_paragraphs(
        all_paragraphs,
        ds_start,
        first_intro_pos,
        is_front_matter=True,
    )
    intro_segments.append(
        {
            "slug": "front-pages",
            "title": "Front Pages",
            "type": "intro-segment",
            "book": "ds",
            "order": 0,
            "paragraphs": front_pages_paragraphs,
        }
    )

    # --- Step 7: Extract intro segments (Foreword, Preface) ---
    # Build boundary list for intro segments
    intro_boundaries = []
    for pos, slug, title, order in intro_bookmark_list:
        intro_boundaries.append((pos, slug, title, order))

    # Sort by position
    intro_boundaries.sort(key=lambda x: x[0])

    # First chapter boundary
    first_chapter_pos = chapter_bookmarks[0][0] if chapter_bookmarks else ds_end

    for bi in range(len(intro_boundaries)):
        b_pos, b_slug, b_title, b_order = intro_boundaries[bi]

        content_start = b_pos + 1  # skip the bookmark tag itself
        if bi + 1 < len(intro_boundaries):
            content_end = intro_boundaries[bi + 1][0]
        else:
            content_end = first_chapter_pos

        paragraphs = extract_paragraphs(
            all_paragraphs,
            content_start,
            content_end,
            skip_p27=(b_slug == "preface"),  # skip "Preface" p27 heading
        )

        intro_segments.append(
            {
                "slug": b_slug,
                "title": b_title,
                "type": "intro-segment",
                "book": "ds",
                "order": b_order,
                "paragraphs": paragraphs,
            }
        )

    # --- Step 8: Extract sub-chapters (philosophers) ---
    sub_chapters = []
    parent_chapters = []

    for ch_num in ch_nums_sorted:
        parent_title = CLEAN_PARENT_TITLES.get(ch_num, f"Chapter {ch_num}")
        parent_end = ch_end_lookup[ch_num]

        philosophers = philosophers_by_parent.get(ch_num, [])
        # Sort by position
        philosophers.sort(key=lambda x: x[0])

        sub_entries = []  # for intro.json
        sub_counter = 0

        for pi, (phil_pos, phil_title) in enumerate(philosophers):
            sub_counter += 1
            slug = f"{ch_num}-{sub_counter}"

            # Decode the title with ScaGoudy
            decoded_title = decode_scagoudy(phil_title)

            # Content range: from bookmark to next philosopher (or parent chapter end)
            content_start = phil_pos  # extract_paragraphs will skip bookmark internally
            if pi + 1 < len(philosophers):
                content_end = philosophers[pi + 1][0]
            else:
                content_end = parent_end

            # For the first philosopher in a chapter, also skip p27 chapter heading
            skip_p27_flag = True  # Always skip p27 headings

            paragraphs = extract_paragraphs(
                all_paragraphs,
                content_start,
                content_end,
                skip_p27=skip_p27_flag,
                skip_first_p31_matching=decoded_title,
            )

            sub_chapters.append(
                {
                    "book": "ds",
                    "chapter": slug,
                    "parentChapter": ch_num,
                    "subChapter": sub_counter,
                    "title": decoded_title,
                    "parentTitle": parent_title,
                    "type": "chapter",
                    "paragraphs": paragraphs,
                }
            )
            sub_entries.append({"slug": slug, "title": decoded_title})

        parent_chapters.append(
            {
                "number": ch_num,
                "title": parent_title,
                "subChapters": sub_entries,
            }
        )

        print(
            f"\n  Parent Ch {ch_num}: {parent_title} → {len(sub_entries)} sub-chapters"
        )
        for entry in sub_entries:
            matching = [sc for sc in sub_chapters if sc["chapter"] == entry["slug"]]
            para_count = len(matching[0]["paragraphs"]) if matching else 0
            print(
                f"    {entry['slug']:>5}: {para_count:>3} paragraphs — {entry['title']}"
            )

    return {
        "intro_segments": intro_segments,
        "sub_chapters": sub_chapters,
        "parent_chapters": parent_chapters,
    }


# ---------------------------------------------------------------------------
# Output helpers
# ---------------------------------------------------------------------------


def write_json(data, filepath):
    """Write data as formatted JSON to filepath, creating directories as needed."""
    os.makedirs(os.path.dirname(filepath), exist_ok=True)
    with open(filepath, "w", encoding="utf-8") as f:
        json.dump(data, f, ensure_ascii=False, indent=2)


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------


def main():
    base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    html_path = os.path.join(base_dir, "public", "html", "other.html")
    output_base = os.path.join(base_dir, "lib", "jsons", "books", "ds")

    if not os.path.exists(html_path):
        print(f"ERROR: {html_path} not found!")
        sys.exit(1)

    print(f"Reading {html_path}...")
    with open(html_path, "r", encoding="utf-8") as f:
        soup = BeautifulSoup(f, "html.parser")

    print("Extracting Dialectic Spiritualism (1985)...\n")
    result = extract_ds(soup)

    intro_segments = result["intro_segments"]
    sub_chapters = result["sub_chapters"]
    parent_chapters = result["parent_chapters"]

    # --- Write intro.json ---
    intro_meta = {
        "book": "ds",
        "title": "Dialectic Spiritualism",
        "subtitle": "A Vedic View of Western Philosophy",
        "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
        "year": 1985,
        "description": "Śrīla Prabhupāda's analysis of Western philosophers from Socrates to Sartre, presenting the Vedic perspective on Western philosophy.",
        "totalChapters": 13,
        "chapters": parent_chapters,
    }
    intro_path = os.path.join(output_base, "intro.json")
    write_json(intro_meta, intro_path)
    print(f"\n  Written: {intro_path}")

    # --- Write intro segments ---
    print(f"\n--- Intro Segments ---")
    for seg in intro_segments:
        filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
        write_json(seg, filepath)
        print(
            f"  {seg['order']:>2}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}"
        )

    # --- Write sub-chapter files ---
    print(f"\n--- Sub-Chapters ---")
    for sc in sub_chapters:
        slug = sc["chapter"]
        filepath = os.path.join(output_base, "chapters", f"{slug}.json")
        write_json(sc, filepath)
        print(
            f"  {slug:>5}: {len(sc['paragraphs']):>3} paragraphs — "
            f"[Ch {sc['parentChapter']}] {sc['title']}"
        )

    # --- Summary ---
    total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments)
    total_chapter_paras = sum(len(sc["paragraphs"]) for sc in sub_chapters)
    total_paras = total_intro_paras + total_chapter_paras

    print(f"\n{'=' * 60}")
    print(f"  EXTRACTION SUMMARY")
    print(f"{'=' * 60}")
    print(f"  Intro segments:     {len(intro_segments)}")
    print(f"  Parent chapters:    {len(parent_chapters)}")
    print(f"  Sub-chapters:       {len(sub_chapters)}")
    print(f"  Total paragraphs:   {total_paras}")
    print(f"    Intro:            {total_intro_paras}")
    print(f"    Chapters:         {total_chapter_paras}")
    if sub_chapters:
        print(f"  Avg paras/sub-ch:   {total_chapter_paras / len(sub_chapters):.1f}")
    print(f"  Output directory:   {output_base}/")

    # --- Per-parent breakdown ---
    print(f"\n--- Parent Chapter Breakdown ---")
    for pc in parent_chapters:
        ch_num = pc["number"]
        sub_count = len(pc["subChapters"])
        ch_paras = sum(
            len(sc["paragraphs"])
            for sc in sub_chapters
            if sc["parentChapter"] == ch_num
        )
        print(
            f"  Ch {ch_num:>2}: {sub_count:>2} sub-chapters, {ch_paras:>4} paragraphs — {pc['title']}"
        )

    # --- Samples ---
    print(f"\n--- Sample: Front Pages ---")
    if intro_segments:
        front = intro_segments[0]
        for i, p in enumerate(front["paragraphs"][:5]):
            preview = re.sub(r"<[^>]+>", "", p)[:100]
            print(f"  [{i}] {preview}...")

    if len(intro_segments) > 1:
        print(f"\n--- Sample: {intro_segments[1]['title']} ---")
        seg = intro_segments[1]
        for i, p in enumerate(seg["paragraphs"][:5]):
            preview = re.sub(r"<[^>]+>", "", p)[:100]
            print(f"  [{i}] {preview}...")

    if sub_chapters:
        first_sc = sub_chapters[0]
        print(f"\n--- Sample: {first_sc['chapter']} ({first_sc['title']}) ---")
        for i, p in enumerate(first_sc["paragraphs"][:5]):
            preview = re.sub(r"<[^>]+>", "", p)[:100]
            print(f"  [{i}] {preview}...")

    if len(sub_chapters) > 1:
        last_sc = sub_chapters[-1]
        print(f"\n--- Sample: {last_sc['chapter']} ({last_sc['title']}) ---")
        for i, p in enumerate(last_sc["paragraphs"][:5]):
            preview = re.sub(r"<[^>]+>", "", p)[:100]
            print(f"  [{i}] {preview}...")

    # --- Quality Checks ---
    print(f"\n--- Quality Checks ---")

    # Check all 13 parent chapters present
    expected_parents = set(range(1, 14))
    actual_parents = {pc["number"] for pc in parent_chapters}
    missing = expected_parents - actual_parents
    extra = actual_parents - expected_parents
    if missing:
        print(f"  ⚠ Missing parent chapters: {sorted(missing)}")
    if extra:
        print(f"  ⚠ Extra parent chapters: {sorted(extra)}")
    if not missing and not extra:
        print(f"  ✅ All 13 parent chapters present")

    # Check all 3 intro segments present
    expected_intros = {"front-pages", "foreword", "preface"}
    actual_intros = {s["slug"] for s in intro_segments}
    missing_intros = expected_intros - actual_intros
    extra_intros = actual_intros - expected_intros
    if missing_intros:
        print(f"  ⚠ Missing intro segments: {missing_intros}")
    if extra_intros:
        print(f"  ⚠ Extra intro segments: {extra_intros}")
    if not missing_intros and not extra_intros:
        print(f"  ✅ All 3 intro segments present")

    # Check 38 philosopher sub-chapters
    if len(sub_chapters) == 38:
        print(f"  ✅ All 38 philosopher sub-chapters extracted")
    else:
        print(f"  ⚠ Expected 38 sub-chapters, got {len(sub_chapters)}")

    # Check empty sub-chapters
    empty_scs = [sc["chapter"] for sc in sub_chapters if len(sc["paragraphs"]) == 0]
    if empty_scs:
        print(f"  ⚠ Empty sub-chapters (0 paragraphs): {empty_scs}")
    else:
        print(f"  ✅ No empty sub-chapters")

    # Check empty intro segments
    empty_intros = [s["slug"] for s in intro_segments if len(s["paragraphs"]) == 0]
    if empty_intros:
        print(f"  ⚠ Empty intro segments: {empty_intros}")
    else:
        print(f"  ✅ No empty intro segments")

    # Check for duplicate headings in first 3 paragraphs of each sub-chapter
    print(f"\n--- Duplicate Heading Check ---")
    dup_count = 0
    for sc in sub_chapters:
        title_norm = _normalize_for_match(sc["title"])
        for p in sc["paragraphs"][:3]:
            plain = re.sub(r"<[^>]+>", "", p).strip()
            plain_norm = _normalize_for_match(plain)
            if plain_norm == title_norm and len(plain) < 150:
                print(f"  ⚠ Duplicate heading in {sc['chapter']}: '{plain[:60]}'")
                dup_count += 1
                break
    if dup_count == 0:
        print(f"  ✅ No duplicate headings in first 3 paragraphs")
    else:
        print(f"  ⚠ {dup_count} sub-chapters have duplicate headings")

    # Check for Roman numeral parent headings leaking into content
    print(f"\n--- Parent Heading Leak Check ---")
    roman_pattern = re.compile(
        r"^(I{1,3}|IV|V|VI{0,3}|VII|VIII|IX|X|XI{0,3}|XII|XIII)\s*[A-Z]"
    )
    leak_count = 0
    for sc in sub_chapters:
        for p in sc["paragraphs"][:2]:
            plain = re.sub(r"<[^>]+>", "", p).strip()
            if roman_pattern.match(plain):
                print(f"  ⚠ Roman numeral heading in {sc['chapter']}: '{plain[:60]}'")
                leak_count += 1
                break
    if leak_count == 0:
        print(f"  ✅ No Roman numeral parent headings in sub-chapter content")

    # Check ScaGoudy decoding
    print(f"\n--- ScaGoudy Decoding Check ---")
    all_text = ""
    for sc in sub_chapters:
        all_text += " ".join(sc["paragraphs"][:20])
    for seg in intro_segments:
        all_text += " ".join(seg["paragraphs"][:10])

    if "Kṛṣṇa" in all_text:
        print(f"  ✅ 'Kṛṣṇa' found (ScaGoudy properly decoded)")
    else:
        print(f"  ⚠ 'Kṛṣṇa' not found — ScaGoudy decoding may have failed")

    if "Bhagavad-gītā" in all_text or "gītā" in all_text or "Gītā" in all_text:
        print(f"  ✅ 'gītā' found (ScaGoudy properly decoded)")
    else:
        print(f"  ⚠ 'Gītā' not found — ScaGoudy decoding may have failed")

    if "Prabhupāda" in all_text:
        print(f"  ✅ 'Prabhupāda' found (ScaGoudy properly decoded)")

    # Check for un-decoded ScaGoudy remnants
    scagoudy_remnants = 0
    for sc in sub_chapters:
        for p in sc["paragraphs"]:
            if "Kåñëa" in p or "Prabhupäda" in p:
                scagoudy_remnants += 1
    for seg in intro_segments:
        for p in seg["paragraphs"]:
            if "Kåñëa" in p or "Prabhupäda" in p:
                scagoudy_remnants += 1
    if scagoudy_remnants == 0:
        print(f"  ✅ No ScaGoudy remnants detected")
    else:
        print(f"  ⚠ {scagoudy_remnants} paragraphs with ScaGoudy remnants")

    # Check for centered verse blocks
    verse_block_count = 0
    for sc in sub_chapters:
        for p in sc["paragraphs"]:
            if '<div style="text-align:center"><i>' in p:
                verse_block_count += 1
    if verse_block_count > 0:
        print(f"  ✅ {verse_block_count} centered Sanskrit verse blocks found")
    else:
        print(f"  ⚠ No centered verse blocks found")

    # Check for residual span tags
    span_count = 0
    for sc in sub_chapters:
        for p in sc["paragraphs"]:
            if 'class="' in p and "<span" in p:
                span_count += 1
    for seg in intro_segments:
        for p in seg["paragraphs"]:
            if 'class="' in p and "<span" in p:
                span_count += 1
    if span_count == 0:
        print(f"  ✅ No residual span class artifacts found")
    else:
        print(f"  ⚠ {span_count} paragraphs still contain span class artifacts")

    # Check for dash-line paragraphs
    dash_count = 0
    for sc in sub_chapters:
        for p in sc["paragraphs"]:
            plain = re.sub(r"<[^>]+>", "", p)
            if is_dash_line(plain):
                dash_count += 1
    for seg in intro_segments:
        for p in seg["paragraphs"]:
            plain = re.sub(r"<[^>]+>", "", p)
            if is_dash_line(plain):
                dash_count += 1
    if dash_count == 0:
        print(f"  ✅ No dashed-line paragraphs in output")
    else:
        print(f"  ⚠ {dash_count} dashed-line paragraphs found in output")

    # Show a verse block sample
    print(f"\n--- Sample: Verse Block ---")
    for sc in sub_chapters:
        for p in sc["paragraphs"]:
            if '<div style="text-align:center"><i>' in p:
                preview = re.sub(r"<[^>]+>", "", p)[:120]
                print(f"  {preview}...")
                break
        else:
            continue
        break

    # Sub-chapter size check
    print(f"\n--- Sub-Chapter Size Check ---")
    very_small = []
    very_large = []
    for sc in sub_chapters:
        count = len(sc["paragraphs"])
        if count < 2:
            very_small.append((sc["chapter"], count, sc["title"]))
        elif count > 500:
            very_large.append((sc["chapter"], count, sc["title"]))
    if very_small:
        print(f"  ⚠ Very small sub-chapters (< 2 paragraphs):")
        for slug, count, title in very_small:
            print(f"    {slug}: {count} paragraphs — {title}")
    else:
        print(f"  ✅ All sub-chapters have ≥ 2 paragraphs")
    if very_large:
        print(f"  ⚠ Very large sub-chapters (> 500 paragraphs):")
        for slug, count, title in very_large:
            print(f"    {slug}: {count} paragraphs — {title}")
    else:
        print(f"  ✅ All sub-chapters have ≤ 500 paragraphs")

    # Sub-chapter title decoding check
    print(f"\n--- Sub-Chapter Title Decoding Check ---")
    bad_titles = 0
    for sc in sub_chapters:
        title = sc["title"]
        if "å" in title or "ë" in title or "ñ" in title or "ä" in title:
            print(f"  ⚠ {sc['chapter']} title has un-decoded ScaGoudy: {title}")
            bad_titles += 1
    if bad_titles == 0:
        print(f"  ✅ All {len(sub_chapters)} sub-chapter titles properly decoded")

    # Verify slug numbering continuity within each parent
    print(f"\n--- Slug Numbering Check ---")
    for pc in parent_chapters:
        ch_num = pc["number"]
        slugs = [s["slug"] for s in pc["subChapters"]]
        expected_slugs = [f"{ch_num}-{i + 1}" for i in range(len(slugs))]
        if slugs == expected_slugs:
            print(
                f"  ✅ Ch {ch_num:>2}: {len(slugs)} sub-chapters, slugs {slugs[0]}..{slugs[-1]}"
            )
        else:
            print(
                f"  ⚠ Ch {ch_num:>2}: slug mismatch! expected {expected_slugs}, got {slugs}"
            )

    # List all philosopher names
    print(f"\n--- All Philosophers ---")
    for sc in sub_chapters:
        print(f"  {sc['chapter']:>5}: {sc['title']}")

    print(f"\nDone! ✨")


if __name__ == "__main__":
    main()