tags total") # Find section boundaries sections = find_section_boundaries(all_tags) print( f" The Beginning: tags[{sections['beg_start']}] → tags[{sections['beg_end']}]" ) for key, val in sorted(sections.items()): print(f" {key}: {val}") # --------------------------------------------------------------------------- # Extract Introduction # --------------------------------------------------------------------------- intro_start = sections["introduction_start"] intro_end = sections["ch1_start"] intro_paragraphs = extract_prose_section(all_tags, intro_start, intro_end) print(f"\n Introduction: {len(intro_paragraphs)} paragraphs") # --------------------------------------------------------------------------- # Extract Chapter 1: The Upper West Side # --------------------------------------------------------------------------- ch1_start = sections["ch1_start"] ch1_end = sections["ch2_start"] ch1_summary, ch1_entries = extract_day_entries(all_tags, ch1_start, ch1_end) print( f" Ch 1 (Upper West Side): {len(ch1_summary)} summary paras, {len(ch1_entries)} day entries" ) # --------------------------------------------------------------------------- # Extract Chapter 2: The Bowery # --------------------------------------------------------------------------- ch2_start = sections["ch2_start"] ch2_end = sections["ch3_start"] ch2_summary, ch2_entries = extract_day_entries(all_tags, ch2_start, ch2_end) print( f" Ch 2 (The Bowery): {len(ch2_summary)} summary paras, {len(ch2_entries)} day entries" ) # --------------------------------------------------------------------------- # Extract Chapter 3: 26 Second Avenue # --------------------------------------------------------------------------- ch3_start = sections["ch3_start"] ch3_end = sections["ch4_start"] ch3_summary, ch3_entries = extract_day_entries(all_tags, ch3_start, ch3_end) print( f" Ch 3 (26 Second Ave): {len(ch3_summary)} summary paras, {len(ch3_entries)} day entries" ) # --------------------------------------------------------------------------- # Extract Memoranda and Accounts # --------------------------------------------------------------------------- ch4_start = sections["ch4_start"] ch4_end = sections["ch5_start"] memo_paragraphs = extract_prose_section(all_tags, ch4_start, ch4_end) print(f" Memoranda: {len(memo_paragraphs)} paragraphs") # --------------------------------------------------------------------------- # Extract Epilogue # --------------------------------------------------------------------------- ch5_start = sections["ch5_start"] ch5_end = sections["ch6_start"] epilogue_paragraphs = extract_prose_section( all_tags, ch5_start, ch5_end, skip_classes={"p4", "p11", "p3", "p5", "p153", "p24"}, ) print(f" Epilogue: {len(epilogue_paragraphs)} paragraphs") # --------------------------------------------------------------------------- # Extract Excerpts # --------------------------------------------------------------------------- ch6_start = sections["ch6_start"] ch6_end = sections["beg_end"] excerpt_preamble, excerpt_list = extract_excerpts(all_tags, ch6_start, ch6_end) print( f" Excerpts: {len(excerpt_preamble)} preamble paras, {len(excerpt_list)} excerpts" ) for ex in excerpt_list: print( f" - {ex['slug']}: {ex['title'][:60]}... ({len(ex['paragraphs'])} paras)" ) # --------------------------------------------------------------------------- # Group day entries by month within each chapter # --------------------------------------------------------------------------- def group_by_month(entries): """Group day entries by month, preserving order.""" groups = [] # List of (month_abbr, month_full, entries_list) current_month = None current_group = None for entry in entries: m = entry.get("month") if m and m != current_month: current_month = m current_group = { "month": m, "month_full": MONTH_FULL.get(m, m), "entries": [], } groups.append(current_group) if current_group is not None: current_group["entries"].append(entry) else: # Entry without month (like "Notice") — create special group current_group = { "month": None, "month_full": "Other", "entries": [entry], } groups.append(current_group) current_group = None current_month = None return groups ch1_months = group_by_month(ch1_entries) ch2_months = group_by_month(ch2_entries) ch3_months = group_by_month(ch3_entries) # --------------------------------------------------------------------------- # Ensure unique slugs across all entries # --------------------------------------------------------------------------- all_slugs = set() def ensure_unique_slug(slug): if slug in all_slugs: suffix = 2 while f"{slug}-{suffix}" in all_slugs: suffix += 1 slug = f"{slug}-{suffix}" all_slugs.add(slug) return slug # Reserve standalone slugs for s in [ "introduction", "upper-west-side-summary", "bowery-summary", "second-avenue-summary", "memoranda", "epilogue", "excerpts-preamble", ]: all_slugs.add(s) # Ensure unique slugs for day entries for entries in [ch1_entries, ch2_entries, ch3_entries]: for entry in entries: entry["slug"] = ensure_unique_slug(entry["slug"]) # Ensure unique slugs for excerpts for ex in excerpt_list: ex["slug"] = ensure_unique_slug(ex["slug"]) # --------------------------------------------------------------------------- # Build intro.json with hierarchical chapters # --------------------------------------------------------------------------- intro_chapters = [] # Introduction (standalone) intro_chapters.append({"slug": "introduction", "title": "Introduction"}) # Chapter 1: The Upper West Side ch1_sub = [{"slug": "upper-west-side-summary", "title": "Summary"}] for mg in ch1_months: ch1_sub.append( { "title": mg["month_full"], "subChapters": [ {"slug": e["slug"], "title": e["title"]} for e in mg["entries"] ], } ) intro_chapters.append( { "title": "The Upper West Side (Jan–Mar 1966)", "subChapters": ch1_sub, } ) # Chapter 2: The Bowery ch2_sub = [{"slug": "bowery-summary", "title": "Summary"}] for mg in ch2_months: ch2_sub.append( { "title": mg["month_full"], "subChapters": [ {"slug": e["slug"], "title": e["title"]} for e in mg["entries"] ], } ) intro_chapters.append( { "title": "The Bowery (Apr–Jun 1966)", "subChapters": ch2_sub, } ) # Chapter 3: 26 Second Avenue ch3_sub = [{"slug": "second-avenue-summary", "title": "Summary"}] for mg in ch3_months: if mg["month"] is not None: ch3_sub.append( { "title": mg["month_full"], "subChapters": [ {"slug": e["slug"], "title": e["title"]} for e in mg["entries"] ], } ) else: # Non-month entries like "Notice" for e in mg["entries"]: ch3_sub.append({"slug": e["slug"], "title": e["title"]}) intro_chapters.append( { "title": "26 Second Avenue (Jul–Nov 1966)", "subChapters": ch3_sub, } ) # Memoranda intro_chapters.append({"slug": "memoranda", "title": "Memoranda and Accounts"}) # Epilogue intro_chapters.append({"slug": "epilogue", "title": "Epilogue"}) # Excerpts — group under a parent heading excerpt_sub = [] if excerpt_preamble: excerpt_sub.append({"slug": "excerpts-preamble", "title": "Preamble"}) for ex in excerpt_list: # Shorten title for sidebar short_title = ex["title"] if len(short_title) > 80: short_title = short_title[:77] + "..." excerpt_sub.append({"slug": ex["slug"], "title": short_title}) intro_chapters.append( { "title": "Excerpts", "subChapters": excerpt_sub, } ) # Count total chapters total_chapters = 0 for ch in intro_chapters: if "slug" in ch: total_chapters += 1 if "subChapters" in ch: for sc in ch["subChapters"]: if "slug" in sc: total_chapters += 1 if "subChapters" in sc: total_chapters += len(sc["subChapters"]) intro_json = { "book": "the-beginning", "title": "The Beginning — The 1966 New York Journal", "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda", "year": 1966, "description": "Śrīla Prabhupāda's personal journal documenting the founding of the Hare Kṛṣṇa movement in New York City, from January to November 1966.", "totalChapters": total_chapters, "chapters": intro_chapters, } # --------------------------------------------------------------------------- # Write all JSON files # --------------------------------------------------------------------------- # Clean output directory if os.path.exists(chapters_dir): for f in os.listdir(chapters_dir): os.remove(os.path.join(chapters_dir, f)) # intro.json write_json(intro_json, os.path.join(output_dir, "intro.json")) print(f"\n Written: intro.json ({total_chapters} total chapters)") chapter_num = 1 files_written = 0 # Introduction write_json( { "book": "the-beginning", "chapter": chapter_num, "title": "Introduction", "type": "introduction", "slug": "introduction", "paragraphs": intro_paragraphs, }, os.path.join(chapters_dir, "introduction.json"), ) print(f" Written: introduction.json ({len(intro_paragraphs)} paras)") chapter_num += 1 files_written += 1 # Chapter 1 summary write_json( { "book": "the-beginning", "chapter": chapter_num, "title": "The Upper West Side — Summary", "type": "summary", "slug": "upper-west-side-summary", "paragraphs": ch1_summary, }, os.path.join(chapters_dir, "upper-west-side-summary.json"), ) print(f" Written: upper-west-side-summary.json ({len(ch1_summary)} paras)") chapter_num += 1 files_written += 1 # Chapter 1 day entries for entry in ch1_entries: write_json( { "book": "the-beginning", "chapter": chapter_num, "title": entry["title"], "type": "chapter", "slug": entry["slug"], "paragraphs": entry["paragraphs"], }, os.path.join(chapters_dir, f"{entry['slug']}.json"), ) chapter_num += 1 files_written += 1 print(f" Written: {len(ch1_entries)} Chapter 1 day entry files") # Chapter 2 summary write_json( { "book": "the-beginning", "chapter": chapter_num, "title": "The Bowery — Summary", "type": "summary", "slug": "bowery-summary", "paragraphs": ch2_summary, }, os.path.join(chapters_dir, "bowery-summary.json"), ) print(f" Written: bowery-summary.json ({len(ch2_summary)} paras)") chapter_num += 1 files_written += 1 # Chapter 2 day entries for entry in ch2_entries: write_json( { "book": "the-beginning", "chapter": chapter_num, "title": entry["title"], "type": "chapter", "slug": entry["slug"], "paragraphs": entry["paragraphs"], }, os.path.join(chapters_dir, f"{entry['slug']}.json"), ) chapter_num += 1 files_written += 1 print(f" Written: {len(ch2_entries)} Chapter 2 day entry files") # Chapter 3 summary write_json( { "book": "the-beginning", "chapter": chapter_num, "title": "26 Second Avenue — Summary", "type": "summary", "slug": "second-avenue-summary", "paragraphs": ch3_summary, }, os.path.join(chapters_dir, "second-avenue-summary.json"), ) print(f" Written: second-avenue-summary.json ({len(ch3_summary)} paras)") chapter_num += 1 files_written += 1 # Chapter 3 day entries for entry in ch3_entries: write_json( { "book": "the-beginning", "chapter": chapter_num, "title": entry["title"], "type": "chapter", "slug": entry["slug"], "paragraphs": entry["paragraphs"], }, os.path.join(chapters_dir, f"{entry['slug']}.json"), ) chapter_num += 1 files_written += 1 print(f" Written: {len(ch3_entries)} Chapter 3 day entry files") # Memoranda write_json( { "book": "the-beginning", "chapter": chapter_num, "title": "Memoranda and Accounts", "type": "chapter", "slug": "memoranda", "paragraphs": memo_paragraphs, }, os.path.join(chapters_dir, "memoranda.json"), ) print(f" Written: memoranda.json ({len(memo_paragraphs)} paras)") chapter_num += 1 files_written += 1 # Epilogue write_json( { "book": "the-beginning", "chapter": chapter_num, "title": "Epilogue", "type": "chapter", "slug": "epilogue", "paragraphs": epilogue_paragraphs, }, os.path.join(chapters_dir, "epilogue.json"), ) print(f" Written: epilogue.json ({len(epilogue_paragraphs)} paras)") chapter_num += 1 files_written += 1 # Excerpts preamble if excerpt_preamble: write_json( { "book": "the-beginning", "chapter": chapter_num, "title": "Excerpts — Preamble", "type": "chapter", "slug": "excerpts-preamble", "paragraphs": excerpt_preamble, }, os.path.join(chapters_dir, "excerpts-preamble.json"), ) print(f" Written: excerpts-preamble.json ({len(excerpt_preamble)} paras)") chapter_num += 1 files_written += 1 # Excerpt entries for ex in excerpt_list: write_json( { "book": "the-beginning", "chapter": chapter_num, "title": ex["title"], "type": "excerpt", "slug": ex["slug"], "paragraphs": ex["paragraphs"], }, os.path.join(chapters_dir, f"{ex['slug']}.json"), ) chapter_num += 1 files_written += 1 print(f" Written: {len(excerpt_list)} excerpt files") # --------------------------------------------------------------------------- # Summary # --------------------------------------------------------------------------- print(f"\n{'=' * 60}") print(f" EXTRACTION SUMMARY") print(f"{'=' * 60}") print(f" Introduction: {len(intro_paragraphs)} paragraphs") print( f" Ch 1 (Upper West Side): {len(ch1_summary)} summary + {len(ch1_entries)} days" ) for mg in ch1_months: print(f" {mg['month_full']}: {len(mg['entries'])} entries") print(f" Ch 2 (The Bowery): {len(ch2_summary)} summary + {len(ch2_entries)} days") for mg in ch2_months: print(f" {mg['month_full']}: {len(mg['entries'])} entries") print( f" Ch 3 (26 Second Ave): {len(ch3_summary)} summary + {len(ch3_entries)} days" ) for mg in ch3_months: mname = mg["month_full"] if mg["month"] else "Other" print(f" {mname}: {len(mg['entries'])} entries") print(f" Memoranda: {len(memo_paragraphs)} paragraphs") print(f" Epilogue: {len(epilogue_paragraphs)} paragraphs") print(f" Excerpts: {len(excerpt_list)} excerpts") print(f" Total chapter files: {files_written}") print(f" Total chapters (for intro.json): {total_chapters}") print(f"{'=' * 60}") print(f"\nDone! ✨") if __name__ == "__main__": main()