tags total") # Find the Jaladuta Diary section boundaries start_idx = None end_idx = None for i, tag in enumerate(all_tags): text = tag.get_text().strip() cls = get_css_class(tag) if cls == "p35" and "Jaladuta Diary" in text: start_idx = i if start_idx is not None and i > start_idx: # End at "Prayers Written by Śrīla Prabhupāda aboard the Jaladuta" or the next p4 bookmark after JD if cls == "p4" and "JD 8" in text: end_idx = i break if start_idx is None: print("ERROR: Could not find Jaladuta Diary start") sys.exit(1) if end_idx is None: # Fallback: use the memoranda end for i, tag in enumerate(all_tags): if ( i > start_idx and "MEMORANDA" in tag.get_text() and get_css_class(tag) == "p110" ): end_idx = i + 1 break print(f" Section: tags[{start_idx}] → tags[{end_idx}]") section_tags = all_tags[start_idx:end_idx] # --------------------------------------------------------------------------- # Phase 1: Extract Preface and Introduction # --------------------------------------------------------------------------- preface_paragraphs = [] intro_paragraphs = [] current_section = None # "preface" | "introduction" | None # Also track where months/diary entries begin diary_start_idx = None for i, tag in enumerate(section_tags): cls = get_css_class(tag) text = tag.get_text().strip() # Detect Preface header if cls == "p3" and text == "Preface": current_section = "preface" continue if cls == "p153": # bookmark continue # Detect Introduction header if cls in ("p11", "p157") and "Introduction" in text: if cls == "p157": current_section = "introduction" continue elif cls == "p11": current_section = "introduction" continue # Detect month start (diary entries begin) if cls == "p11" and re.match(r"JD \d+:", text): diary_start_idx = i current_section = None break if cls == "p29": # Bold month header like "APRIL 1965" — this is diary territory diary_start_idx = i current_section = None break # Collect preface/intro paragraphs if current_section == "preface" and cls in ("p154", "p155", "p156"): html = inner_html(tag) if html: preface_paragraphs.append(html) elif current_section == "introduction" and cls in ("p154", "p155", "p156"): html = inner_html(tag) if html: intro_paragraphs.append(html) print(f"\n Preface: {len(preface_paragraphs)} paragraphs") print(f" Introduction: {len(intro_paragraphs)} paragraphs") # --------------------------------------------------------------------------- # Phase 2: Extract diary entries grouped by month # --------------------------------------------------------------------------- # Walk through remaining tags, tracking current month current_month = None # "apr", "may", etc. current_month_full = None # "April 1965", etc. # Section headers that appear mid-month section_headers = {} # Maps tag index to header text # All day entries: list of { month, day, date, slug, title, paragraphs } day_entries = [] current_entry = None MONTH_MAP = { "APRIL": ("apr", "April 1965"), "MAY": ("may", "May 1965"), "JUNE": ("jun", "June 1965"), "JULY": ("jul", "July 1965"), "AUGUST": ("aug", "August 1965"), "SEPTEMBER": ("sep", "September 1965"), "OCTOBER": ("oct", "October 1965"), "NOVEMBER": ("nov", "November 1965"), } def finalize_entry(): nonlocal current_entry if current_entry and current_entry["paragraphs"]: day_entries.append(current_entry) current_entry = None diary_tags = section_tags[diary_start_idx:] if diary_start_idx else [] for i, tag in enumerate(diary_tags): cls = get_css_class(tag) text = tag.get_text().strip() # Skip bookmarks if cls in ("p11", "p4", "p39", "p153"): continue # Detect month headers (p29 bold) if cls == "p29": html = inner_html(tag) # Check if this is a month header like "APRIL 1965" or "MAY 1965" for month_key, (abbr, full) in MONTH_MAP.items(): if month_key in text.upper(): finalize_entry() current_month = abbr current_month_full = full break else: # Could be a section header like "Verse 1." — skip if not month # Or it could be a section header within a month pass continue # Detect section headers (p68 class — "The Journey", "America") if cls == "p68": finalize_entry() html = inner_html(tag) # Create a section-header entry if html and current_month: section_entry = { "month": current_month, "month_full": current_month_full, "day": None, "date": None, "slug": None, "title": None, "paragraphs": [f"{re.sub('<[^>]+>', ' ', html).strip()}"], "is_section_header": True, } # Extract title text plain = re.sub("<[^>]+>", " ", html) plain = re.sub(r"\s+", " ", plain).strip() section_entry["title"] = plain day_entries.append(section_entry) continue # Detect diary entries (p110 class) if cls == "p110": finalize_entry() html = inner_html(tag) if not html: continue # Parse the bold header b_tag = tag.find("b") if b_tag: bold_text = b_tag.get_text().strip() day_name, date_num = parse_day_header(bold_text) else: day_name, date_num = (None, None) if day_name == "MEMORANDA": current_entry = { "month": current_month, "month_full": current_month_full, "day": "MEMORANDA", "date": None, "slug": "memoranda", "title": "Memoranda", "paragraphs": [html], "is_section_header": False, } elif day_name and date_num and current_month: slug = f"{current_month}-{date_num}" title = f"{day_name.capitalize()} {date_num}" current_entry = { "month": current_month, "month_full": current_month_full, "day": day_name, "date": date_num, "slug": slug, "title": title, "paragraphs": [html], "is_section_header": False, } elif day_name and current_month: # Day without date number (rare — like "SATURDAY" alone on Aug 14) # Try to infer from context or just skip slug = f"{current_month}-{day_name.lower()}" title = day_name.capitalize() current_entry = { "month": current_month, "month_full": current_month_full, "day": day_name, "date": None, "slug": slug, "title": title, "paragraphs": [html], "is_section_header": False, } else: # Can't parse — treat as continuation or standalone current_entry = { "month": current_month, "month_full": current_month_full, "day": None, "date": None, "slug": None, "title": text[:50], "paragraphs": [html], "is_section_header": False, } continue # Detect p12 paragraphs (continuation text in some entries like Sep 16) if cls == "p12" and current_entry: html = inner_html(tag) if html: current_entry["paragraphs"].append(html) continue # p155 (source citations within diary section) if cls == "p155" and current_entry: html = inner_html(tag) if html: current_entry["paragraphs"].append(html) continue finalize_entry() print(f"\n Total day entries extracted: {len(day_entries)}") # Count by month month_counts = {} for entry in day_entries: m = entry["month"] month_counts[m] = month_counts.get(m, 0) + 1 for m in MONTH_ORDER: if m in month_counts: print(f" {m}: {month_counts[m]} entries") # --------------------------------------------------------------------------- # Phase 3: Fix up entries — handle dateless entries, dedup slugs # --------------------------------------------------------------------------- # Some entries have no date — try to fix known cases # Aug "SATURDAY" without number is Aug 14 (follows Aug 13 FRIDAY) for i, entry in enumerate(day_entries): if entry.get("is_section_header"): continue if entry["date"] is None and entry["day"] and entry["day"] != "MEMORANDA": # Try to infer from previous entry if i > 0: prev = day_entries[i - 1] if prev["date"] and prev["month"] == entry["month"]: inferred = prev["date"] + 1 entry["date"] = inferred entry["slug"] = f"{entry['month']}-{inferred}" entry["title"] = f"{entry['day'].capitalize()} {inferred}" print( f" Inferred date: {entry['slug']} (from previous {prev['slug']})" ) # Ensure unique slugs seen_slugs = set() for entry in day_entries: if entry.get("is_section_header"): continue slug = entry["slug"] if slug and slug in seen_slugs: # Append suffix suffix = 2 while f"{slug}-{suffix}" in seen_slugs: suffix += 1 entry["slug"] = f"{slug}-{suffix}" print(f" Deduped slug: {slug} → {entry['slug']}") if slug: seen_slugs.add(entry["slug"]) # Merge section headers into the next entry's paragraphs (prepend) merged_entries = [] pending_header = None for entry in day_entries: if entry.get("is_section_header"): pending_header = entry["paragraphs"][0] continue if pending_header: entry["paragraphs"] = [pending_header] + entry["paragraphs"] pending_header = None merged_entries.append(entry) day_entries = merged_entries # --------------------------------------------------------------------------- # Phase 4: Build intro.json with hierarchical month groups # --------------------------------------------------------------------------- # Assign chapter numbers: preface=1, introduction=2, then day entries sequentially chapter_num = 1 all_chapters = [] # Preface preface_chapter = { "slug": "preface", "title": "Preface", "number": chapter_num, } all_chapters.append(preface_chapter) chapter_num += 1 # Introduction intro_chapter = { "slug": "introduction", "title": "Introduction", "number": chapter_num, } all_chapters.append(intro_chapter) chapter_num += 1 # Group day entries by month months_data = [] # List of { month, title, subChapters: [...] } current_month_group = None for entry in day_entries: month = entry["month"] month_full = entry["month_full"] if current_month_group is None or current_month_group["month"] != month: current_month_group = { "month": month, "title": month_full, "subChapters": [], } months_data.append(current_month_group) entry["number"] = chapter_num current_month_group["subChapters"].append( { "slug": entry["slug"], "title": entry["title"], } ) chapter_num += 1 # Build the intro.json chapters array # Front matter chapters (preface, introduction) are top-level # Month groups use the hierarchical sub-chapter pattern intro_chapters = [] # Add preface and introduction as simple chapters intro_chapters.append({"slug": "preface", "title": "Preface"}) intro_chapters.append({"slug": "introduction", "title": "Introduction"}) # Add month groups for mg in months_data: intro_chapters.append( { "title": mg["title"], "subChapters": mg["subChapters"], } ) intro_json = { "book": "jaladuta-diary", "title": "The Jaladuta Diary, 1965 — Journey to the U.S.A.", "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda", "year": 1965, "description": "Śrīla Prabhupāda's personal diary kept during his historic voyage from Calcutta to New York aboard the cargo ship Jaladuta in 1965.", "totalChapters": chapter_num - 1, "chapters": intro_chapters, } # --------------------------------------------------------------------------- # Phase 5: Write JSON files # --------------------------------------------------------------------------- # Clean output directory if os.path.exists(chapters_dir): for f in os.listdir(chapters_dir): os.remove(os.path.join(chapters_dir, f)) # Write intro.json write_json(intro_json, os.path.join(output_dir, "intro.json")) print(f"\n Written: intro.json ({len(intro_chapters)} top-level entries)") # Write preface write_json( { "book": "jaladuta-diary", "chapter": 1, "title": "Preface", "type": "preface", "slug": "preface", "paragraphs": preface_paragraphs, }, os.path.join(chapters_dir, "preface.json"), ) print(f" Written: preface.json ({len(preface_paragraphs)} paragraphs)") # Write introduction write_json( { "book": "jaladuta-diary", "chapter": 2, "title": "Introduction", "type": "introduction", "slug": "introduction", "paragraphs": intro_paragraphs, }, os.path.join(chapters_dir, "introduction.json"), ) print(f" Written: introduction.json ({len(intro_paragraphs)} paragraphs)") # Write day entries ch_num = 3 for entry in day_entries: slug = entry["slug"] if not slug: print(f" ⚠ Skipping entry with no slug: {entry.get('title', '?')}") continue chapter_data = { "book": "jaladuta-diary", "chapter": ch_num, "title": entry["title"], "type": "chapter", "slug": slug, "paragraphs": entry["paragraphs"], } write_json(chapter_data, os.path.join(chapters_dir, f"{slug}.json")) ch_num += 1 total_files = 2 + len([e for e in day_entries if e.get("slug")]) print(f"\n Total chapter files written: {total_files}") # --------------------------------------------------------------------------- # Summary # --------------------------------------------------------------------------- print(f"\n{'=' * 60}") print(f" EXTRACTION SUMMARY") print(f"{'=' * 60}") print(f" Preface: {len(preface_paragraphs)} paragraphs") print(f" Introduction: {len(intro_paragraphs)} paragraphs") print(f" Day entries: {len(day_entries)}") for mg in months_data: print(f" {mg['title']}: {len(mg['subChapters'])} entries") for sc in mg["subChapters"]: print(f" - {sc['slug']}: {sc['title']}") print(f" Total chapters: {ch_num - 1}") print(f"{'=' * 60}") print(f"\nDone! ✨") if __name__ == "__main__": main()