spsays-training/scripts/the_beginning_extract.py

#!/usr/bin/env python3
"""Extract "The Beginning" — The 1966 New York Journal from others-before-1977.html.

Each diary day entry becomes its own chapter file.
Chapters are grouped hierarchically:
  - Introduction (standalone)
  - Chapter 1: The Upper West Side (summary + day entries grouped by month)
  - Chapter 2: The Bowery (summary + day entries grouped by month)
  - Chapter 3: 26 Second Avenue (summary + day entries grouped by month + Notice)
  - Memoranda and Accounts (standalone)
  - Epilogue (standalone)
  - Excerpts (each excerpt as its own chapter)

Output:
    lib/jsons/journals/the-beginning/intro.json
    lib/jsons/journals/the-beginning/chapters/introduction.json
    lib/jsons/journals/the-beginning/chapters/upper-west-side-summary.json
    lib/jsons/journals/the-beginning/chapters/jan-19.json
    ...
    lib/jsons/journals/the-beginning/chapters/memoranda.json
    lib/jsons/journals/the-beginning/chapters/epilogue.json
    lib/jsons/journals/the-beginning/chapters/excerpt-*.json

Usage:
    python3 scripts/the_beginning_extract.py
"""

import json
import os
import re
import sys

from bs4 import BeautifulSoup, Tag

# ---------------------------------------------------------------------------
# ScaGoudy → IAST Unicode mapping
# ---------------------------------------------------------------------------
SCAGOUDY_TO_IAST = {
    "ä": "ā",
    "Ä": "Ā",
    "å": "ṛ",
    "Å": "Ṛ",
    "ñ": "ṣ",
    "Ñ": "Ṣ",
    "ö": "ṭ",
    "Ö": "Ṭ",
    "ë": "ṇ",
    "Ë": "Ṇ",
    "ò": "ḍ",
    "Ò": "Ḍ",
    "ç": "ś",
    "Ç": "Ś",
    "ì": "ṅ",
    "Ì": "Ṅ",
    "ï": "ñ",
    "Ï": "Ñ",
    "ü": "ū",
    "Ü": "Ū",
    "é": "ī",
    "É": "Ī",
    "à": "ṁ",
    "ù": "ḥ",
    "è": "ṝ",
    "ÿ": "ḷ",
}


def decode_scagoudy(text: str) -> str:
    for src, dst in SCAGOUDY_TO_IAST.items():
        text = text.replace(src, dst)
    return text


def get_css_class(tag: Tag) -> str:
    classes = tag.get("class", [])
    return classes[0] if classes else ""


def inner_html(tag: Tag) -> str:
    """Get inner HTML of a tag, decode ScaGoudy, clean up spans."""
    html = "".join(str(child) for child in tag.children)
    html = decode_scagoudy(html)
    # <span class="s3">text</span> → <i>text</i>
    html = re.sub(
        r'<span[^>]*class="s3"[^>]*>(.*?)</span>', r"<i>\1</i>", html, flags=re.DOTALL
    )
    # <span class="s5">...</span> → keep inner text
    html = re.sub(
        r'<span[^>]*class="s5"[^>]*>(.*?)</span>', r"\1", html, flags=re.DOTALL
    )
    # <span class="s7">...</span> → keep inner text
    html = re.sub(
        r'<span[^>]*class="s7"[^>]*>(.*?)</span>', r"\1", html, flags=re.DOTALL
    )
    # <span class="s13|s23|s25|s2x">...</span> → keep inner text
    html = re.sub(
        r'<span[^>]*class="s\d+"[^>]*>(.*?)</span>', r"\1", html, flags=re.DOTALL
    )
    # Remove Apple-tab-span
    html = re.sub(r'<span class="Apple-tab-span">\t</span>', " ", html)
    # Remove any remaining spans but keep content
    html = re.sub(r"<span[^>]*>(.*?)</span>", r"\1", html, flags=re.DOTALL)
    # Clean up tab characters
    html = html.replace("\t", " ")
    # Collapse multiple spaces
    html = re.sub(r"[ ]+", " ", html)
    html = html.strip()
    # Remove leading <br> tags
    html = re.sub(r"^(<br\s*/?>[\s]*)+", "", html)
    return html


def write_json(data: dict, path: str):
    os.makedirs(os.path.dirname(path), exist_ok=True)
    with open(path, "w", encoding="utf-8") as f:
        json.dump(data, f, indent=2, ensure_ascii=False)
        f.write("\n")


# ---------------------------------------------------------------------------
# Month helpers
# ---------------------------------------------------------------------------
MONTH_MAP = {
    "january": "jan",
    "february": "feb",
    "march": "mar",
    "april": "apr",
    "may": "may",
    "june": "jun",
    "july": "jul",
    "august": "aug",
    "september": "sep",
    "october": "oct",
    "november": "nov",
    "december": "dec",
}

MONTH_FULL = {
    "jan": "January 1966",
    "feb": "February 1966",
    "mar": "March 1966",
    "apr": "April 1966",
    "may": "May 1966",
    "jun": "June 1966",
    "jul": "July 1966",
    "aug": "August 1966",
    "sep": "September 1966",
    "oct": "October 1966",
    "nov": "November 1966",
}

DAY_NAMES = {
    "monday",
    "tuesday",
    "wednesday",
    "thursday",
    "friday",
    "saturday",
    "sunday",
}


def parse_day_month(text: str):
    """Parse 'Wednesday 19 January' → ('Wednesday', 19, 'jan')
    Also handles 'Notice' and other non-date entries.
    Returns (day_name, date_num, month_abbr) or (None, None, None).
    """
    text = text.strip()
    # Try "DayName Number MonthName" pattern
    m = re.match(r"([A-Za-z]+)\s+(\d+)\s+([A-Za-z]+)", text)
    if m:
        day = m.group(1).capitalize()
        num = int(m.group(2))
        month = m.group(3).lower()
        if day.lower() in DAY_NAMES and month in MONTH_MAP:
            return (day, num, MONTH_MAP[month])

    return (None, None, None)


def slugify_excerpt(title: str) -> str:
    """Create a URL-friendly slug from an excerpt title."""
    # Remove HTML tags
    title = re.sub(r"<[^>]+>", "", title)
    title = decode_scagoudy(title)
    # Take first 60 chars
    title = title[:80].lower().strip()
    # Replace non-alnum with hyphens
    title = re.sub(r"[^a-z0-9]+", "-", title)
    title = title.strip("-")
    # Truncate to reasonable length
    if len(title) > 60:
        title = title[:60].rsplit("-", 1)[0]
    return f"excerpt-{title}"


# ---------------------------------------------------------------------------
# Section boundaries detector
# ---------------------------------------------------------------------------


def find_section_boundaries(all_tags):
    """Find the start/end tag indices for each major section."""
    sections = {}
    beg_start = None
    beg_end = None

    for i, tag in enumerate(all_tags):
        cls = get_css_class(tag)
        text = tag.get_text().strip()

        # Find "The Beginning" start
        if cls == "p170" and "The Beginning" in text:
            beg_start = i

        if beg_start is not None and i > beg_start:
            # Find section markers by p4 bookmarks
            if cls == "p4" and text.startswith("BEG"):
                if "Introduction" in text:
                    sections["introduction_start"] = i
                elif "BEG 1" in text:
                    sections["ch1_start"] = i
                elif "BEG 2" in text:
                    sections["ch2_start"] = i
                elif "BEG 3" in text:
                    sections["ch3_start"] = i
                elif "BEG 4" in text:
                    sections["ch4_start"] = i
                elif "BEG 5" in text:
                    sections["ch5_start"] = i
                elif "BEG 6" in text:
                    sections["ch6_start"] = i

            # Detect end of The Beginning (next non-BEG p4 or end of file)
            if (
                cls == "p4"
                and not text.startswith("BEG")
                and beg_start
                and "ch6_start" in sections
                and i > sections["ch6_start"] + 5
            ):
                beg_end = i
                break

    # If no explicit end found, use end of conversation content
    if beg_end is None:
        # Search for the last p74 conversation line
        for i in range(len(all_tags) - 1, sections.get("ch6_start", 0), -1):
            cls = get_css_class(all_tags[i])
            if cls in ("p74", "p6", "p77"):
                beg_end = i + 1
                break

    sections["beg_start"] = beg_start
    sections["beg_end"] = beg_end

    return sections


# ---------------------------------------------------------------------------
# Extract prose section (Introduction, Epilogue, etc.)
# ---------------------------------------------------------------------------


def extract_prose_section(tags, start_idx, end_idx, skip_classes=None):
    """Extract paragraphs from a prose section."""
    if skip_classes is None:
        skip_classes = {"p4", "p11", "p3", "p5", "p39", "p24", "p153"}

    paragraphs = []
    for i in range(start_idx, end_idx):
        tag = tags[i]
        cls = get_css_class(tag)

        if cls in skip_classes:
            continue

        html = inner_html(tag)
        if html and html != "<br>":
            paragraphs.append(html)

    return paragraphs


# ---------------------------------------------------------------------------
# Extract day entries from a chapter section
# ---------------------------------------------------------------------------


def extract_day_entries(tags, start_idx, end_idx):
    """Extract day entries from a chapter section.
    Returns (summary_paragraphs, day_entries_list).
    """
    summary_paragraphs = []
    day_entries = []
    current_entry = None
    in_summary = True  # Start in summary mode until first p39 day entry

    skip_classes = {"p4", "p11", "p3", "p5", "p153"}

    def finalize_entry():
        nonlocal current_entry
        if current_entry and current_entry["paragraphs"]:
            day_entries.append(current_entry)
        current_entry = None

    for i in range(start_idx, end_idx):
        tag = tags[i]
        cls = get_css_class(tag)
        text = tag.get_text().strip()

        if cls in skip_classes:
            continue

        # p39 = day bookmark
        if cls == "p39":
            finalize_entry()
            in_summary = False

            # Check if it's a special entry like "Notice" or "Memoranda"
            if text.lower() == "notice":
                current_entry = {
                    "day_name": None,
                    "date": None,
                    "month": None,
                    "slug": "notice",
                    "title": "Notice",
                    "paragraphs": [],
                }
                continue
            elif text.lower() == "memoranda":
                current_entry = {
                    "day_name": None,
                    "date": None,
                    "month": None,
                    "slug": "memoranda",
                    "title": "Memoranda",
                    "paragraphs": [],
                }
                continue

            day_name, date_num, month_abbr = parse_day_month(text)
            if day_name and date_num and month_abbr:
                slug = f"{month_abbr}-{date_num}"
                title = f"{day_name} {date_num}"
                current_entry = {
                    "day_name": day_name,
                    "date": date_num,
                    "month": month_abbr,
                    "slug": slug,
                    "title": title,
                    "paragraphs": [],
                }
            else:
                # Can't parse — might be an excerpt title or special entry
                # Treat as a titled entry
                slug_text = re.sub(r"[^a-z0-9]+", "-", text.lower().strip())[:50].strip(
                    "-"
                )
                current_entry = {
                    "day_name": None,
                    "date": None,
                    "month": None,
                    "slug": slug_text or "unknown",
                    "title": text,
                    "paragraphs": [],
                }
            continue

        # p8 = bold day header (display version of p39)
        if cls == "p8":
            # This is the bold display version — skip if we already have the entry from p39
            if current_entry is not None:
                continue
            # If we're in summary mode, this might be the chapter date range or summary header
            if in_summary:
                html = inner_html(tag)
                if html:
                    summary_paragraphs.append(html)
                continue
            continue

        # p24 = empty line (skip)
        if cls == "p24":
            continue

        # p29 = bold subheading (like "NOTICE")
        if cls == "p29":
            if current_entry is not None:
                html = inner_html(tag)
                if html:
                    current_entry["paragraphs"].append(html)
            continue

        # Collect body paragraphs
        html = inner_html(tag)
        if not html or html == "<br>":
            continue

        if in_summary:
            summary_paragraphs.append(html)
        elif current_entry is not None:
            current_entry["paragraphs"].append(html)

    finalize_entry()
    return summary_paragraphs, day_entries


# ---------------------------------------------------------------------------
# Extract excerpts
# ---------------------------------------------------------------------------


def extract_excerpts(tags, start_idx, end_idx):
    """Extract individual excerpts from the Excerpts section.
    Each excerpt starts with a p39 bookmark.
    Returns a list of { slug, title, paragraphs }.
    """
    excerpts = []
    current_excerpt = None
    in_preamble = True  # Before first p39 excerpt

    preamble_paragraphs = []
    skip_classes = {"p4", "p11", "p3", "p5", "p153"}
    seen_slugs = set()

    def finalize_excerpt():
        nonlocal current_excerpt
        if current_excerpt and current_excerpt["paragraphs"]:
            excerpts.append(current_excerpt)
        current_excerpt = None

    for i in range(start_idx, end_idx):
        tag = tags[i]
        cls = get_css_class(tag)
        text = tag.get_text().strip()

        if cls in skip_classes:
            continue

        # p39 = excerpt title bookmark
        if cls == "p39":
            finalize_excerpt()
            in_preamble = False

            title = decode_scagoudy(text)
            slug = slugify_excerpt(title)

            # Ensure unique slug
            if slug in seen_slugs:
                suffix = 2
                while f"{slug}-{suffix}" in seen_slugs:
                    suffix += 1
                slug = f"{slug}-{suffix}"
            seen_slugs.add(slug)

            current_excerpt = {
                "slug": slug,
                "title": title,
                "paragraphs": [],
            }
            continue

        # p8 = bold excerpt header (display version) — skip if we have the excerpt
        if cls == "p8":
            if current_excerpt is not None:
                # Skip the bold header — we already have the title from p39
                continue
            if in_preamble:
                html = inner_html(tag)
                if html:
                    preamble_paragraphs.append(html)
            continue

        if cls == "p24":
            continue

        html = inner_html(tag)
        if not html or html == "<br>":
            continue

        if in_preamble:
            preamble_paragraphs.append(html)
        elif current_excerpt is not None:
            # For conversation lines (p74), format with bold speaker
            # The inner_html already preserves <b> tags
            current_excerpt["paragraphs"].append(html)

    finalize_excerpt()

    return preamble_paragraphs, excerpts


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------


def main():
    base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html")
    output_dir = os.path.join(base_dir, "lib", "jsons", "journals", "the-beginning")
    chapters_dir = os.path.join(output_dir, "chapters")

    if not os.path.exists(html_path):
        print(f"ERROR: {html_path} not found!")
        sys.exit(1)

    print(f"Reading {html_path}...")
    with open(html_path, "r", encoding="utf-8") as f:
        content = f.read()

    soup = BeautifulSoup(content, "html.parser")
    all_tags = soup.find_all("p")
    print(f"  Parsed {len(all_tags)} <p> tags total")

    # Find section boundaries
    sections = find_section_boundaries(all_tags)
    print(
        f"  The Beginning: tags[{sections['beg_start']}] → tags[{sections['beg_end']}]"
    )
    for key, val in sorted(sections.items()):
        print(f"    {key}: {val}")

    # ---------------------------------------------------------------------------
    # Extract Introduction
    # ---------------------------------------------------------------------------
    intro_start = sections["introduction_start"]
    intro_end = sections["ch1_start"]
    intro_paragraphs = extract_prose_section(all_tags, intro_start, intro_end)
    print(f"\n  Introduction: {len(intro_paragraphs)} paragraphs")

    # ---------------------------------------------------------------------------
    # Extract Chapter 1: The Upper West Side
    # ---------------------------------------------------------------------------
    ch1_start = sections["ch1_start"]
    ch1_end = sections["ch2_start"]
    ch1_summary, ch1_entries = extract_day_entries(all_tags, ch1_start, ch1_end)
    print(
        f"  Ch 1 (Upper West Side): {len(ch1_summary)} summary paras, {len(ch1_entries)} day entries"
    )

    # ---------------------------------------------------------------------------
    # Extract Chapter 2: The Bowery
    # ---------------------------------------------------------------------------
    ch2_start = sections["ch2_start"]
    ch2_end = sections["ch3_start"]
    ch2_summary, ch2_entries = extract_day_entries(all_tags, ch2_start, ch2_end)
    print(
        f"  Ch 2 (The Bowery): {len(ch2_summary)} summary paras, {len(ch2_entries)} day entries"
    )

    # ---------------------------------------------------------------------------
    # Extract Chapter 3: 26 Second Avenue
    # ---------------------------------------------------------------------------
    ch3_start = sections["ch3_start"]
    ch3_end = sections["ch4_start"]
    ch3_summary, ch3_entries = extract_day_entries(all_tags, ch3_start, ch3_end)
    print(
        f"  Ch 3 (26 Second Ave): {len(ch3_summary)} summary paras, {len(ch3_entries)} day entries"
    )

    # ---------------------------------------------------------------------------
    # Extract Memoranda and Accounts
    # ---------------------------------------------------------------------------
    ch4_start = sections["ch4_start"]
    ch4_end = sections["ch5_start"]
    memo_paragraphs = extract_prose_section(all_tags, ch4_start, ch4_end)
    print(f"  Memoranda: {len(memo_paragraphs)} paragraphs")

    # ---------------------------------------------------------------------------
    # Extract Epilogue
    # ---------------------------------------------------------------------------
    ch5_start = sections["ch5_start"]
    ch5_end = sections["ch6_start"]
    epilogue_paragraphs = extract_prose_section(
        all_tags,
        ch5_start,
        ch5_end,
        skip_classes={"p4", "p11", "p3", "p5", "p153", "p24"},
    )
    print(f"  Epilogue: {len(epilogue_paragraphs)} paragraphs")

    # ---------------------------------------------------------------------------
    # Extract Excerpts
    # ---------------------------------------------------------------------------
    ch6_start = sections["ch6_start"]
    ch6_end = sections["beg_end"]
    excerpt_preamble, excerpt_list = extract_excerpts(all_tags, ch6_start, ch6_end)
    print(
        f"  Excerpts: {len(excerpt_preamble)} preamble paras, {len(excerpt_list)} excerpts"
    )
    for ex in excerpt_list:
        print(
            f"    - {ex['slug']}: {ex['title'][:60]}... ({len(ex['paragraphs'])} paras)"
        )

    # ---------------------------------------------------------------------------
    # Group day entries by month within each chapter
    # ---------------------------------------------------------------------------

    def group_by_month(entries):
        """Group day entries by month, preserving order."""
        groups = []  # List of (month_abbr, month_full, entries_list)
        current_month = None
        current_group = None

        for entry in entries:
            m = entry.get("month")
            if m and m != current_month:
                current_month = m
                current_group = {
                    "month": m,
                    "month_full": MONTH_FULL.get(m, m),
                    "entries": [],
                }
                groups.append(current_group)
            if current_group is not None:
                current_group["entries"].append(entry)
            else:
                # Entry without month (like "Notice") — create special group
                current_group = {
                    "month": None,
                    "month_full": "Other",
                    "entries": [entry],
                }
                groups.append(current_group)
                current_group = None
                current_month = None

        return groups

    ch1_months = group_by_month(ch1_entries)
    ch2_months = group_by_month(ch2_entries)
    ch3_months = group_by_month(ch3_entries)

    # ---------------------------------------------------------------------------
    # Ensure unique slugs across all entries
    # ---------------------------------------------------------------------------
    all_slugs = set()

    def ensure_unique_slug(slug):
        if slug in all_slugs:
            suffix = 2
            while f"{slug}-{suffix}" in all_slugs:
                suffix += 1
            slug = f"{slug}-{suffix}"
        all_slugs.add(slug)
        return slug

    # Reserve standalone slugs
    for s in [
        "introduction",
        "upper-west-side-summary",
        "bowery-summary",
        "second-avenue-summary",
        "memoranda",
        "epilogue",
        "excerpts-preamble",
    ]:
        all_slugs.add(s)

    # Ensure unique slugs for day entries
    for entries in [ch1_entries, ch2_entries, ch3_entries]:
        for entry in entries:
            entry["slug"] = ensure_unique_slug(entry["slug"])

    # Ensure unique slugs for excerpts
    for ex in excerpt_list:
        ex["slug"] = ensure_unique_slug(ex["slug"])

    # ---------------------------------------------------------------------------
    # Build intro.json with hierarchical chapters
    # ---------------------------------------------------------------------------

    intro_chapters = []

    # Introduction (standalone)
    intro_chapters.append({"slug": "introduction", "title": "Introduction"})

    # Chapter 1: The Upper West Side
    ch1_sub = [{"slug": "upper-west-side-summary", "title": "Summary"}]
    for mg in ch1_months:
        ch1_sub.append(
            {
                "title": mg["month_full"],
                "subChapters": [
                    {"slug": e["slug"], "title": e["title"]} for e in mg["entries"]
                ],
            }
        )
    intro_chapters.append(
        {
            "title": "The Upper West Side (Jan–Mar 1966)",
            "subChapters": ch1_sub,
        }
    )

    # Chapter 2: The Bowery
    ch2_sub = [{"slug": "bowery-summary", "title": "Summary"}]
    for mg in ch2_months:
        ch2_sub.append(
            {
                "title": mg["month_full"],
                "subChapters": [
                    {"slug": e["slug"], "title": e["title"]} for e in mg["entries"]
                ],
            }
        )
    intro_chapters.append(
        {
            "title": "The Bowery (Apr–Jun 1966)",
            "subChapters": ch2_sub,
        }
    )

    # Chapter 3: 26 Second Avenue
    ch3_sub = [{"slug": "second-avenue-summary", "title": "Summary"}]
    for mg in ch3_months:
        if mg["month"] is not None:
            ch3_sub.append(
                {
                    "title": mg["month_full"],
                    "subChapters": [
                        {"slug": e["slug"], "title": e["title"]} for e in mg["entries"]
                    ],
                }
            )
        else:
            # Non-month entries like "Notice"
            for e in mg["entries"]:
                ch3_sub.append({"slug": e["slug"], "title": e["title"]})
    intro_chapters.append(
        {
            "title": "26 Second Avenue (Jul–Nov 1966)",
            "subChapters": ch3_sub,
        }
    )

    # Memoranda
    intro_chapters.append({"slug": "memoranda", "title": "Memoranda and Accounts"})

    # Epilogue
    intro_chapters.append({"slug": "epilogue", "title": "Epilogue"})

    # Excerpts — group under a parent heading
    excerpt_sub = []
    if excerpt_preamble:
        excerpt_sub.append({"slug": "excerpts-preamble", "title": "Preamble"})
    for ex in excerpt_list:
        # Shorten title for sidebar
        short_title = ex["title"]
        if len(short_title) > 80:
            short_title = short_title[:77] + "..."
        excerpt_sub.append({"slug": ex["slug"], "title": short_title})
    intro_chapters.append(
        {
            "title": "Excerpts",
            "subChapters": excerpt_sub,
        }
    )

    # Count total chapters
    total_chapters = 0
    for ch in intro_chapters:
        if "slug" in ch:
            total_chapters += 1
        if "subChapters" in ch:
            for sc in ch["subChapters"]:
                if "slug" in sc:
                    total_chapters += 1
                if "subChapters" in sc:
                    total_chapters += len(sc["subChapters"])

    intro_json = {
        "book": "the-beginning",
        "title": "The Beginning — The 1966 New York Journal",
        "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
        "year": 1966,
        "description": "Śrīla Prabhupāda's personal journal documenting the founding of the Hare Kṛṣṇa movement in New York City, from January to November 1966.",
        "totalChapters": total_chapters,
        "chapters": intro_chapters,
    }

    # ---------------------------------------------------------------------------
    # Write all JSON files
    # ---------------------------------------------------------------------------

    # Clean output directory
    if os.path.exists(chapters_dir):
        for f in os.listdir(chapters_dir):
            os.remove(os.path.join(chapters_dir, f))

    # intro.json
    write_json(intro_json, os.path.join(output_dir, "intro.json"))
    print(f"\n  Written: intro.json ({total_chapters} total chapters)")

    chapter_num = 1
    files_written = 0

    # Introduction
    write_json(
        {
            "book": "the-beginning",
            "chapter": chapter_num,
            "title": "Introduction",
            "type": "introduction",
            "slug": "introduction",
            "paragraphs": intro_paragraphs,
        },
        os.path.join(chapters_dir, "introduction.json"),
    )
    print(f"  Written: introduction.json ({len(intro_paragraphs)} paras)")
    chapter_num += 1
    files_written += 1

    # Chapter 1 summary
    write_json(
        {
            "book": "the-beginning",
            "chapter": chapter_num,
            "title": "The Upper West Side — Summary",
            "type": "summary",
            "slug": "upper-west-side-summary",
            "paragraphs": ch1_summary,
        },
        os.path.join(chapters_dir, "upper-west-side-summary.json"),
    )
    print(f"  Written: upper-west-side-summary.json ({len(ch1_summary)} paras)")
    chapter_num += 1
    files_written += 1

    # Chapter 1 day entries
    for entry in ch1_entries:
        write_json(
            {
                "book": "the-beginning",
                "chapter": chapter_num,
                "title": entry["title"],
                "type": "chapter",
                "slug": entry["slug"],
                "paragraphs": entry["paragraphs"],
            },
            os.path.join(chapters_dir, f"{entry['slug']}.json"),
        )
        chapter_num += 1
        files_written += 1

    print(f"  Written: {len(ch1_entries)} Chapter 1 day entry files")

    # Chapter 2 summary
    write_json(
        {
            "book": "the-beginning",
            "chapter": chapter_num,
            "title": "The Bowery — Summary",
            "type": "summary",
            "slug": "bowery-summary",
            "paragraphs": ch2_summary,
        },
        os.path.join(chapters_dir, "bowery-summary.json"),
    )
    print(f"  Written: bowery-summary.json ({len(ch2_summary)} paras)")
    chapter_num += 1
    files_written += 1

    # Chapter 2 day entries
    for entry in ch2_entries:
        write_json(
            {
                "book": "the-beginning",
                "chapter": chapter_num,
                "title": entry["title"],
                "type": "chapter",
                "slug": entry["slug"],
                "paragraphs": entry["paragraphs"],
            },
            os.path.join(chapters_dir, f"{entry['slug']}.json"),
        )
        chapter_num += 1
        files_written += 1

    print(f"  Written: {len(ch2_entries)} Chapter 2 day entry files")

    # Chapter 3 summary
    write_json(
        {
            "book": "the-beginning",
            "chapter": chapter_num,
            "title": "26 Second Avenue — Summary",
            "type": "summary",
            "slug": "second-avenue-summary",
            "paragraphs": ch3_summary,
        },
        os.path.join(chapters_dir, "second-avenue-summary.json"),
    )
    print(f"  Written: second-avenue-summary.json ({len(ch3_summary)} paras)")
    chapter_num += 1
    files_written += 1

    # Chapter 3 day entries
    for entry in ch3_entries:
        write_json(
            {
                "book": "the-beginning",
                "chapter": chapter_num,
                "title": entry["title"],
                "type": "chapter",
                "slug": entry["slug"],
                "paragraphs": entry["paragraphs"],
            },
            os.path.join(chapters_dir, f"{entry['slug']}.json"),
        )
        chapter_num += 1
        files_written += 1

    print(f"  Written: {len(ch3_entries)} Chapter 3 day entry files")

    # Memoranda
    write_json(
        {
            "book": "the-beginning",
            "chapter": chapter_num,
            "title": "Memoranda and Accounts",
            "type": "chapter",
            "slug": "memoranda",
            "paragraphs": memo_paragraphs,
        },
        os.path.join(chapters_dir, "memoranda.json"),
    )
    print(f"  Written: memoranda.json ({len(memo_paragraphs)} paras)")
    chapter_num += 1
    files_written += 1

    # Epilogue
    write_json(
        {
            "book": "the-beginning",
            "chapter": chapter_num,
            "title": "Epilogue",
            "type": "chapter",
            "slug": "epilogue",
            "paragraphs": epilogue_paragraphs,
        },
        os.path.join(chapters_dir, "epilogue.json"),
    )
    print(f"  Written: epilogue.json ({len(epilogue_paragraphs)} paras)")
    chapter_num += 1
    files_written += 1

    # Excerpts preamble
    if excerpt_preamble:
        write_json(
            {
                "book": "the-beginning",
                "chapter": chapter_num,
                "title": "Excerpts — Preamble",
                "type": "chapter",
                "slug": "excerpts-preamble",
                "paragraphs": excerpt_preamble,
            },
            os.path.join(chapters_dir, "excerpts-preamble.json"),
        )
        print(f"  Written: excerpts-preamble.json ({len(excerpt_preamble)} paras)")
        chapter_num += 1
        files_written += 1

    # Excerpt entries
    for ex in excerpt_list:
        write_json(
            {
                "book": "the-beginning",
                "chapter": chapter_num,
                "title": ex["title"],
                "type": "excerpt",
                "slug": ex["slug"],
                "paragraphs": ex["paragraphs"],
            },
            os.path.join(chapters_dir, f"{ex['slug']}.json"),
        )
        chapter_num += 1
        files_written += 1

    print(f"  Written: {len(excerpt_list)} excerpt files")

    # ---------------------------------------------------------------------------
    # Summary
    # ---------------------------------------------------------------------------
    print(f"\n{'=' * 60}")
    print(f"  EXTRACTION SUMMARY")
    print(f"{'=' * 60}")
    print(f"  Introduction: {len(intro_paragraphs)} paragraphs")
    print(
        f"  Ch 1 (Upper West Side): {len(ch1_summary)} summary + {len(ch1_entries)} days"
    )
    for mg in ch1_months:
        print(f"    {mg['month_full']}: {len(mg['entries'])} entries")
    print(f"  Ch 2 (The Bowery): {len(ch2_summary)} summary + {len(ch2_entries)} days")
    for mg in ch2_months:
        print(f"    {mg['month_full']}: {len(mg['entries'])} entries")
    print(
        f"  Ch 3 (26 Second Ave): {len(ch3_summary)} summary + {len(ch3_entries)} days"
    )
    for mg in ch3_months:
        mname = mg["month_full"] if mg["month"] else "Other"
        print(f"    {mname}: {len(mg['entries'])} entries")
    print(f"  Memoranda: {len(memo_paragraphs)} paragraphs")
    print(f"  Epilogue: {len(epilogue_paragraphs)} paragraphs")
    print(f"  Excerpts: {len(excerpt_list)} excerpts")
    print(f"  Total chapter files: {files_written}")
    print(f"  Total chapters (for intro.json): {total_chapters}")
    print(f"{'=' * 60}")
    print(f"\nDone! ✨")


if __name__ == "__main__":
    main()