+ {letter.paragraphs.map((paragraph, i) => (
+
+ ))}
+
+ + {letter.date} + {letter.location ? ` · ${letter.location}` : ""} +
++ {resultCount === 0 + ? "No results found" + : `${resultCount} of ${totalCount} ${currentView === "year" ? "years" : currentView === "recipient" ? "recipients" : "locations"}`} +
+ )} + + {/* Year view */} + {currentView === "year" && ( ++ {yg.year} +
++ {yg.count} {yg.count === 1 ? "letter" : "letters"} +
++ {rg.name} +
+{lg.name}
++ {lg.count} {lg.count === 1 ? "letter" : "letters"} +
++ {letters.length} {letters.length === 1 ? "letter" : "letters"} +
++ To: {letter.recipientDisplay} +
++ {letter.date || "Undated"} +
++ {totalCount.toLocaleString()} letters by Śrīla Prabhupāda (1947–1977) +
++ {letters.length} {letters.length === 1 ? "letter" : "letters"} +
++ To: {letter.recipientDisplay} +
++ {letter.date || "Undated"} + {letter.location ? ` · ${letter.location}` : ""} +
++ {letters.length} {letters.length === 1 ? "letter" : "letters"} +
++ To: {letter.recipientDisplay} +
++ {letter.date} + {letter.location ? ` · ${letter.location}` : ""} +
+{meta.author}
+{meta.description}
+ )} + + {meta.chapters.map((ch: MagazineChapterEntry, idx: number) => { + if ("slug" in ch && !("subChapters" in ch)) { + return ( +{ch.title}
+ ++ {sc.title} +
+ ++ Back to Godhead and other periodicals by Śrīla Prabhupāda +
++ The original Back to Godhead magazine founded by Śrīla + Prabhupāda in 1944 — 29 issues across 3 volumes with 89 + articles on Kṛṣṇa consciousness, Vedic philosophy, and + spiritual life. +
++ Śrīla Prabhupāda's personal correspondence — over 6,800 + letters to disciples, Godbrothers, government officials, and + well-wishers. Browse by year, recipient, or location. +
+ tags total")
+
+ # ---------------------------------------------------------------------------
+ # Phase 1: Find all issue boundaries
+ # ---------------------------------------------------------------------------
+ issue_boundaries = [] # List of (tag_index, issue_num, volume, display_title, date_str, year, slug)
+
+ for i, tag in enumerate(all_tags):
+ cls = get_css_class(tag)
+ if cls == "p2":
+ text = tag.get_text().strip()
+ if "Back to Godhead" in text:
+ parsed = parse_issue_info(text)
+ if parsed:
+ issue_num, volume, display_title, date_str, year, slug = parsed
+ issue_boundaries.append(
+ (i, issue_num, volume, display_title, date_str, year, slug)
+ )
+
+ print(f"\n Found {len(issue_boundaries)} issues")
+ for ib in issue_boundaries:
+ tag_idx, inum, vol, title, ds, yr, slug = ib
+ print(f" tag[{tag_idx}]: #{inum} (Vol {vol}) {title} → {slug}")
+
+ # ---------------------------------------------------------------------------
+ # Phase 2: Extract articles and paragraphs for each issue
+ # ---------------------------------------------------------------------------
+
+ # Known skip classes (bookmarks, empty lines)
+ SKIP_CLASSES = {"p2", "p11", "p28", "p24"}
+
+ issues_data = []
+ skipped_ads = 0
+
+ # Maximum paragraphs per issue — prevents mega-issues from absorbing
+ # unmarked content that belongs to missing intermediate issues.
+ MAX_PARAGRAPHS_PER_ISSUE = 500
+
+ for idx, (
+ start_tag_idx,
+ issue_num,
+ vol,
+ display_title,
+ date_str,
+ year,
+ slug,
+ ) in enumerate(issue_boundaries):
+ # Determine end boundary
+ if idx + 1 < len(issue_boundaries):
+ end_tag_idx = issue_boundaries[idx + 1][0]
+ else:
+ end_tag_idx = len(all_tags)
+
+ issue_tags = all_tags[start_tag_idx:end_tag_idx]
+
+ paragraphs = []
+ articles = []
+ in_advertisement = False
+
+ for tag in issue_tags:
+ cls = get_css_class(tag)
+ text = tag.get_text().strip()
+
+ # Skip bookmark tags and empty lines
+ if cls in SKIP_CLASSES:
+ continue
+ if not text:
+ continue
+
+ # p8 = Article bookmark
+ if cls == "p8":
+ article_title = decode_scagoudy(text).strip()
+
+ # Check if this is an advertisement
+ if re.match(r"^Advertisements?", article_title, re.IGNORECASE):
+ in_advertisement = True
+ skipped_ads += 1
+ continue
+ else:
+ in_advertisement = False
+ articles.append({"title": article_title})
+ continue
+
+ # p122 and other classes can also be advertisement markers
+ if re.match(r"^Advertisements?\s*$", text, re.IGNORECASE):
+ in_advertisement = True
+ skipped_ads += 1
+ continue
+
+ # p9 = Article display title
+ if cls == "p9":
+ if in_advertisement:
+ continue
+ # Also check the p9 display title itself for advertisement
+ plain_title = decode_scagoudy(text).strip()
+ if re.match(r"^Advertisements?\s*$", plain_title, re.IGNORECASE):
+ in_advertisement = True
+ skipped_ads += 1
+ continue
+ html = inner_html(tag)
+ if html:
+ # Make it a bold article heading if not already
+ if not html.startswith(""):
+ html = f"{html}"
+ paragraphs.append(html)
+ continue
+
+ # Skip content within advertisement sections
+ if in_advertisement:
+ # Check if next p8 will reset it — for now just skip
+ continue
+
+ # p15 = Verse block — center align
+ if cls == "p15":
+ html = inner_html(tag)
+ if html:
+ paragraphs.append(f' tags total")
+
+ # ---------------------------------------------------------------------------
+ # Phase 1: Find all issue boundaries
+ # ---------------------------------------------------------------------------
+ # Each issue starts with a p2 bookmark tag
+ issue_boundaries = [] # List of (tag_index, volume, part, date_str, slug)
+
+ for i, tag in enumerate(all_tags):
+ cls = get_css_class(tag)
+ if cls == "p2":
+ text = tag.get_text().strip()
+ parsed = parse_volume_part(text)
+ if parsed:
+ vol, part, date_str, slug = parsed
+ issue_boundaries.append((i, vol, part, date_str, slug))
+
+ print(f"\n Found {len(issue_boundaries)} issues")
+ for ib in issue_boundaries:
+ print(f" tag[{ib[0]}]: Vol {ib[1]}, Part {ib[2]}, {ib[3]} → {ib[4]}")
+
+ # ---------------------------------------------------------------------------
+ # Phase 2: Extract articles and paragraphs for each issue
+ # ---------------------------------------------------------------------------
+
+ # Classes to skip entirely
+ SKIP_CLASSES = {"p2", "p11", "p24", "p28"}
+
+ # Classes that are header/masthead material (include as paragraphs)
+ MASTHEAD_CLASSES = {"p3", "p4", "p5", "p6", "p7", "p32", "p33", "p34", "p40", "p47"}
+
+ # Classes that are body text
+ BODY_CLASSES = {
+ "p10",
+ "p12",
+ "p13",
+ "p14",
+ "p15",
+ "p16",
+ "p17",
+ "p35",
+ "p36",
+ "p37",
+ "p38",
+ "p39",
+ "p41",
+ "p42",
+ "p43",
+ "p44",
+ "p45",
+ "p46",
+ "p48",
+ "p49",
+ "p50",
+ "p51",
+ "p52",
+ "p53",
+ "p54",
+ "p55",
+ "p56",
+ "p57",
+ "p58",
+ "p59",
+ "p60",
+ "p61",
+ "p62",
+ "p63",
+ "p64",
+ "p65",
+ "p66",
+ "p67",
+ "p68",
+ "p69",
+ "p70",
+ "p71",
+ "p72",
+ "p73",
+ "p74",
+ "p75",
+ "p76",
+ "p77",
+ "p78",
+ "p79",
+ "p80",
+ "p81",
+ "p82",
+ "p83",
+ "p84",
+ "p85",
+ "p86",
+ "p87",
+ "p88",
+ "p89",
+ "p90",
+ "p91",
+ "p92",
+ "p93",
+ "p94",
+ "p95",
+ "p96",
+ "p97",
+ "p98",
+ "p99",
+ }
+
+ issues_data = []
+
+ for idx, (start_tag_idx, vol, part, date_str, slug) in enumerate(issue_boundaries):
+ # Determine end boundary
+ if idx + 1 < len(issue_boundaries):
+ end_tag_idx = issue_boundaries[idx + 1][0]
+ else:
+ end_tag_idx = len(all_tags)
+
+ issue_tags = all_tags[start_tag_idx:end_tag_idx]
+
+ paragraphs = []
+ articles = [] # List of { title, slug }
+
+ for tag in issue_tags:
+ cls = get_css_class(tag)
+ text = tag.get_text().strip()
+
+ # Skip bookmark tags and empty lines
+ if cls in SKIP_CLASSES:
+ continue
+ if not text and cls not in ("p15",):
+ continue
+
+ # p8 = Article bookmark — extract title for metadata
+ if cls == "p8":
+ # Extract article title from bookmark text (remove BTGPYxx: prefix)
+ article_text = text
+ m = re.match(r"BTGPY\d+[a-z]?:\s*(.*)", article_text)
+ if m:
+ article_title = m.group(1).strip()
+ else:
+ article_title = article_text
+ article_title = decode_scagoudy(article_title)
+ articles.append({"title": article_title})
+ continue
+
+ # p9 = Article display title — render as bold heading
+ if cls == "p9":
+ html = inner_html(tag)
+ if html:
+ # Make it a bold article heading
+ if not html.startswith(""):
+ html = f"{html}"
+ paragraphs.append(html)
+ continue
+
+ # p15 = Verse block — center align
+ if cls == "p15":
+ html = format_verse(tag)
+ if html:
+ paragraphs.append(html)
+ continue
+
+ # p3 = Volume title — skip (it's the display title of the issue header)
+ if cls == "p3":
+ continue
+
+ # Masthead classes — include but as regular paragraphs
+ if cls in MASTHEAD_CLASSES:
+ html = inner_html(tag)
+ if html and html != " tags total")
+
+ # ---------------------------------------------------------------------------
+ # Phase 1: Find all letter boundaries
+ # ---------------------------------------------------------------------------
+ letter_starts = []
+ for i, tag in enumerate(all_tags):
+ cls = get_css_class(tag)
+ text = tag.get_text().strip()
+ if cls == "p7" and text.startswith("Letter to:"):
+ letter_starts.append(i)
+
+ print(f"\n Found {len(letter_starts)} letters")
+
+ # ---------------------------------------------------------------------------
+ # Phase 2: Extract each letter
+ # ---------------------------------------------------------------------------
+
+ # Classes that are just empty spacers
+ SKIP_CLASSES = {"p2", "p15", "p23", "p24", "p26"}
+
+ # p12 = centered (verse-like)
+ CENTERED_CLASSES = {"p12", "p18", "p19", "p20", "p21", "p27"}
+
+ # p13, p34, p38 = signature/closing blocks (colored)
+ SIGNATURE_CLASSES = {"p13", "p34", "p38"}
+
+ letters = []
+ slug_counts = Counter()
+
+ for li, start_idx in enumerate(letter_starts):
+ # Determine end boundary
+ end_idx = (
+ letter_starts[li + 1] if li + 1 < len(letter_starts) else len(all_tags)
+ )
+
+ # Parse header
+ header_tag = all_tags[start_idx]
+ recipient, location, date_str = parse_header(header_tag)
+
+ # Parse date into components
+ year, month, day = parse_date(date_str) if date_str else (None, None, None)
+
+ # Find date reference code (first p8 after header that matches YY-MM-DD)
+ date_ref = None
+ salutation = None
+ paragraphs = []
+ first_p8_seen = False
+
+ for i in range(start_idx + 1, end_idx):
+ tag = all_tags[i]
+ cls = get_css_class(tag)
+ text = tag.get_text().strip()
+
+ if cls in SKIP_CLASSES:
+ continue
+ if not text:
+ continue
+
+ # p8 = reference code or body text
+ if cls == "p8":
+ if not first_p8_seen:
+ # First p8 after header — check if it's a date reference
+ if re.match(r"\d{2}-\d{2}", text):
+ date_ref = text.strip()
+ first_p8_seen = True
+
+ # Try to extract year/month/day from ref if not already parsed
+ if not year and date_ref:
+ ref_parts = date_ref.split("-")
+ if len(ref_parts) >= 3:
+ try:
+ yr = int(ref_parts[0])
+ mo = int(ref_parts[1])
+ dy = int(ref_parts[2])
+ year = 1900 + yr if yr < 100 else yr
+ month = mo if 1 <= mo <= 12 else None
+ day = dy if 1 <= dy <= 31 else None
+ except ValueError:
+ pass
+ continue
+ else:
+ first_p8_seen = True
+ # Not a date ref — treat as body text
+ html = inner_html(tag)
+ if html:
+ paragraphs.append(html)
+ continue
+ else:
+ # Subsequent p8 tags — body/closing text
+ html = inner_html(tag)
+ if html:
+ paragraphs.append(html)
+ continue
+
+ # p9 = recipient address block
+ if cls == "p9":
+ html = inner_html(tag)
+ if html:
+ paragraphs.append(html)
+ continue
+
+ # p10 = salutation
+ if cls == "p10":
+ salutation = decode_scagoudy(text)
+ html = inner_html(tag)
+ if html:
+ paragraphs.append(html)
+ continue
+
+ # Centered classes (verse blocks)
+ if cls in CENTERED_CLASSES:
+ html = inner_html(tag)
+ if html:
+ # Check if it has line breaks (verse-like)
+ if "{html}')
+ continue
+
+ # All other classes — extract as body text
+ html = inner_html(tag)
+ if html and html != "
":
+ # Check if this looks like a verse (entirely italic with
breaks)
+ stripped = html.strip()
+ if (
+ stripped.startswith("")
+ and "{stripped}'
+
+ paragraphs.append(html)
+
+ # Post-process: remove any ad-like paragraphs that slipped through.
+ # Two-pass approach:
+ # 1. Detect ad section start markers and skip until next real article
+ # 2. Filter individual ad-like paragraphs by content patterns
+
+ # --- Pass 1: Section-level ad detection ---
+ ad_section_patterns = [
+ r"^Advertisements?\s*$",
+ r"^CIRCULATION AND SUBSCRIPTIONS",
+ r"^Order Form",
+ r"^SPECIAL OFFER",
+ r"^Subscriptions?:",
+ r"^WORLDWIDE IMPORTS",
+ r"^THE\s+ORIENT\s+SHOP",
+ r"^PROTECT YOUR CHECKS",
+ ]
+ ad_section_re = re.compile("|".join(ad_section_patterns), re.IGNORECASE)
+
+ cleaned_pass1 = []
+ skip_until_next_bold = False
+ for p in paragraphs:
+ plain = re.sub(r"<[^>]+>", "", p).strip()
+ # Detect ad section start
+ if ad_section_re.search(plain):
+ skip_until_next_bold = True
+ continue
+ if skip_until_next_bold:
+ # A bold article heading that's NOT an ad resets the skip
+ if p.startswith("") and len(plain) < 200:
+ if ad_section_re.search(plain):
+ continue
+ else:
+ skip_until_next_bold = False
+ cleaned_pass1.append(p)
+ continue
+ # If it looks like real content (long paragraph), stop skipping
+ if len(plain) > 150:
+ skip_until_next_bold = False
+ cleaned_pass1.append(p)
+ continue
+ # Short paragraphs in ad zone — skip
+ continue
+ cleaned_pass1.append(p)
+
+ # --- Pass 2: Individual paragraph ad detection ---
+ # These patterns catch standalone ad fragments that weren't part of
+ # a marked advertisement section.
+ ad_para_keywords = [
+ "subscription",
+ "per copy",
+ "enclosed is",
+ "please send me",
+ "order form",
+ "postage paid",
+ "money order",
+ "check or money order",
+ "zip code",
+ "state_____",
+ "address_____",
+ "city_____",
+ "available from iskcon",
+ "waterpipes",
+ "djellabas",
+ "panchos",
+ "orient shop",
+ "worldwide imports",
+ "sullivan st",
+ "bleecker",
+ "check protector",
+ "safeguards your checks",
+ "satisfaction guaranteed",
+ "import-export",
+ "davis & manswell",
+ "davis & manswell",
+ "bedford ave",
+ "brooklyn, n.y.",
+ "immediate delivery",
+ "free gift guide",
+ "write for free",
+ "free catalog",
+ "free catalogue",
+ "protect your checks",
+ "name of purchaser",
+ "such dresses at $",
+ "and gowns yet $",
+ "price per copy",
+ "one-year subscription",
+ "year subscription costs",
+ "back issues available",
+ "a subscription to",
+ "send check or money",
+ "your name & address",
+ "your name & address",
+ "674-9511",
+ ]
+ # Also detect short paragraphs that look like addresses or phone numbers
+ address_re = re.compile(
+ r"^\d+\s+\w+\s+(st|street|ave|avenue|blvd|road|rd|lane|ln|dr|drive)\b",
+ re.IGNORECASE,
+ )
+ phone_re = re.compile(r"^\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}\s*$")
+ zipcode_re = re.compile(
+ r"(N\.?Y\.?\s+\d{5}|CA\s+\d{5}|Brooklyn|Manhattan)",
+ re.IGNORECASE,
+ )
+
+ cleaned = []
+ for p in cleaned_pass1:
+ plain = re.sub(r"<[^>]+>", "", p).strip()
+ plain_lower = plain.lower()
+
+ # Skip if matches any ad keyword
+ if any(kw in plain_lower for kw in ad_para_keywords):
+ continue
+
+ # Skip short paragraphs that look like addresses
+ if len(plain) < 80 and address_re.search(plain):
+ continue
+
+ # Skip standalone phone numbers
+ if phone_re.match(plain):
+ continue
+
+ # Skip short paragraphs with zip codes (likely address fragments)
+ if len(plain) < 60 and zipcode_re.search(plain):
+ continue
+
+ # Skip price-only paragraphs like "$1 plus 25c"
+ if re.match(
+ r"^\$?\d+[\.\d]*\s*(plus|postage|each|per)", plain, re.IGNORECASE
+ ):
+ continue
+
+ cleaned.append(p)
+ paragraphs = cleaned
+
+ # Cap paragraphs to prevent mega-issues from absorbing unmarked
+ # intermediate issues (e.g., issues 18-29 have no p2 markers and
+ # would otherwise be absorbed into issue 17).
+ if len(paragraphs) > MAX_PARAGRAPHS_PER_ISSUE:
+ print(
+ f" ⚠ {slug}: Capped from {len(paragraphs)} to {MAX_PARAGRAPHS_PER_ISSUE} paragraphs "
+ f"(likely absorbed unmarked intermediate issues)"
+ )
+ paragraphs = paragraphs[:MAX_PARAGRAPHS_PER_ISSUE]
+
+ issue_data = {
+ "book": "btg-1978",
+ "volume": vol,
+ "part": issue_num,
+ "title": display_title,
+ "slug": slug,
+ "year": year,
+ "articles": articles,
+ "paragraphs": paragraphs,
+ }
+ issues_data.append(issue_data)
+
+ print(
+ f" {slug}: {display_title} — "
+ f"{len(articles)} articles, {len(paragraphs)} paragraphs"
+ )
+
+ print(f"\n Skipped {skipped_ads} advertisement sections")
+
+ # ---------------------------------------------------------------------------
+ # Phase 3: Build intro.json with hierarchical groups
+ # ---------------------------------------------------------------------------
+
+ # Group issues by era/volume
+ # Early issues (1-17) have Vol 1/Vol 2 designations
+ # Later issues (30+) just have issue numbers
+ group_early = [] # Issues 1-17 (1966-1968)
+ group_later = [] # Issues 30+ (1969-1973)
+
+ for issue in issues_data:
+ if issue["part"] <= 17:
+ group_early.append(issue)
+ else:
+ group_later.append(issue)
+
+ chapters = []
+
+ if group_early:
+ sub = []
+ for issue in group_early:
+ sub.append({"slug": issue["slug"], "title": issue["title"]})
+ chapters.append(
+ {
+ "title": "Issues #1–17 (1966–1968)",
+ "subChapters": sub,
+ }
+ )
+
+ if group_later:
+ sub = []
+ for issue in group_later:
+ sub.append({"slug": issue["slug"], "title": issue["title"]})
+ chapters.append(
+ {
+ "title": "Issues #30–54 (1969–1973)",
+ "subChapters": sub,
+ }
+ )
+
+ total_issues = len(issues_data)
+ total_articles = sum(len(i["articles"]) for i in issues_data)
+
+ # Determine year range
+ years = [i["year"] for i in issues_data if i["year"]]
+ min_year = min(years) if years else None
+ max_year = max(years) if years else None
+
+ intro_json = {
+ "book": "btg-1978",
+ "title": "Back to Godhead (1966–1973)",
+ "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
+ "year": min_year,
+ "description": (
+ f"Back to Godhead magazine issues from the Western era ({min_year}–{max_year}). "
+ f"{total_issues} issues containing {total_articles} articles — lectures, essays, "
+ f"poems, and philosophical discussions on Kṛṣṇa consciousness."
+ ),
+ "totalIssues": total_issues,
+ "totalArticles": total_articles,
+ "flag": "Pre 1978",
+ "chapters": chapters,
+ }
+
+ # ---------------------------------------------------------------------------
+ # Phase 4: Write JSON files
+ # ---------------------------------------------------------------------------
+
+ # Clean output directory
+ if os.path.exists(issues_dir):
+ for f in os.listdir(issues_dir):
+ os.remove(os.path.join(issues_dir, f))
+
+ # Write intro.json
+ write_json(intro_json, os.path.join(output_dir, "intro.json"))
+ print(f"\n Written: intro.json ({total_issues} issues, {total_articles} articles)")
+
+ # Write issue files
+ order = 100
+ for issue in issues_data:
+ issue_json = {
+ "book": "btg-1978",
+ "volume": issue["volume"],
+ "part": issue["part"],
+ "title": issue["title"],
+ "slug": issue["slug"],
+ "order": order,
+ "articles": issue["articles"],
+ "paragraphs": issue["paragraphs"],
+ }
+ filepath = os.path.join(issues_dir, f"{issue['slug']}.json")
+ write_json(issue_json, filepath)
+ order += 1
+
+ print(f" Written: {len(issues_data)} issue files")
+
+ # ---------------------------------------------------------------------------
+ # Phase 5: Quality checks
+ # ---------------------------------------------------------------------------
+ print(f"\n{'=' * 60}")
+ print(f" QUALITY CHECKS")
+ print(f"{'=' * 60}")
+
+ # Check for ScaGoudy remnants
+ sca_chars = set(SCAGOUDY_TO_IAST.keys())
+ sca_remnants = 0
+ for issue in issues_data:
+ for p in issue["paragraphs"]:
+ if any(c in p for c in ("å", "ë", "ö")):
+ sca_remnants += 1
+ break
+ if sca_remnants == 0:
+ print(f" ✅ No ScaGoudy remnants detected")
+ else:
+ print(f" ⚠ {sca_remnants} issues may have ScaGoudy remnants")
+
+ # Check for empty issues
+ empty_issues = [i for i in issues_data if len(i["paragraphs"]) == 0]
+ if empty_issues:
+ print(
+ f" ⚠ {len(empty_issues)} empty issues: {[i['slug'] for i in empty_issues]}"
+ )
+ else:
+ print(f" ✅ No empty issues")
+
+ # Check centered verses
+ verse_count = 0
+ for issue in issues_data:
+ for p in issue["paragraphs"]:
+ if '' in p:
+ verse_count += 1
+ print(f" ℹ️ {verse_count} center-aligned verse paragraphs")
+
+ # Check for remaining advertisement content
+ ad_check = 0
+ ad_keywords = ["advertisement", "subscribe", "subscription form"]
+ for issue in issues_data:
+ for p in issue["paragraphs"]:
+ lower_p = p.lower()
+ if any(kw in lower_p for kw in ad_keywords):
+ # Only flag if the paragraph is primarily an ad
+ if "advertisement" in lower_p and len(p) < 200:
+ ad_check += 1
+ if ad_check == 0:
+ print(f" ✅ No advertisement content detected in output")
+ else:
+ print(f" ⚠ {ad_check} paragraphs may contain advertisement content")
+
+ # ---------------------------------------------------------------------------
+ # Summary
+ # ---------------------------------------------------------------------------
+ print(f"\n{'=' * 60}")
+ print(f" EXTRACTION SUMMARY")
+ print(f"{'=' * 60}")
+
+ if group_early:
+ early_arts = sum(len(i["articles"]) for i in group_early)
+ early_paras = sum(len(i["paragraphs"]) for i in group_early)
+ print(
+ f" Issues #1–17 (1966–1968): {len(group_early)} issues, "
+ f"{early_arts} articles, {early_paras} paragraphs"
+ )
+ for issue in group_early:
+ print(
+ f" {issue['slug']}: {issue['title']} "
+ f"({len(issue['articles'])} articles, {len(issue['paragraphs'])} paras)"
+ )
+
+ if group_later:
+ later_arts = sum(len(i["articles"]) for i in group_later)
+ later_paras = sum(len(i["paragraphs"]) for i in group_later)
+ print(
+ f" Issues #30–54 (1969–1973): {len(group_later)} issues, "
+ f"{later_arts} articles, {later_paras} paragraphs"
+ )
+ for issue in group_later:
+ print(
+ f" {issue['slug']}: {issue['title']} "
+ f"({len(issue['articles'])} articles, {len(issue['paragraphs'])} paras)"
+ )
+
+ print(f"\n Total: {total_issues} issues, {total_articles} articles")
+ print(f" Skipped: {skipped_ads} advertisement sections")
+ print(f"{'=' * 60}")
+ print(f"\nDone! ✨")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/btg_extract.py b/scripts/btg_extract.py
new file mode 100644
index 00000000..2b3a706c
--- /dev/null
+++ b/scripts/btg_extract.py
@@ -0,0 +1,546 @@
+#!/usr/bin/env python3
+"""Extract Back to Godhead magazine from btg-1960.html.
+
+Each magazine issue becomes its own JSON file.
+Issues are grouped by volume in the intro.json metadata.
+
+Structure:
+ - p2 = Volume/Part bookmark (e.g., "BTGPY1: Volume 1, Part 1, 1944")
+ - p3 = Volume/Part display title
+ - p8 = Article bookmark within an issue (e.g., "BTGPY1a: Message of His Divine Grace")
+ - p9 = Article display title (bold)
+ - p12, p35 = Body paragraphs
+ - p14, p17, p88 = Continuation notes / misc text
+ - p15 = Verse blocks (italic, with ScaGoudy encoding)
+ - p33, p40, p47, p54 = Magazine mastheads/headers
+
+Output:
+ lib/jsons/magazines/btg/intro.json
+ lib/jsons/magazines/btg/issues/v1-p1.json
+ lib/jsons/magazines/btg/issues/v1-p2.json
+ ...
+ lib/jsons/magazines/btg/issues/v4-p1.json
+
+Usage:
+ python3 scripts/btg_extract.py
+"""
+
+import json
+import os
+import re
+import sys
+
+from bs4 import BeautifulSoup, Tag
+
+# ---------------------------------------------------------------------------
+# ScaGoudy → IAST Unicode mapping
+# ---------------------------------------------------------------------------
+SCAGOUDY_TO_IAST = {
+ "ä": "ā",
+ "Ä": "Ā",
+ "å": "ṛ",
+ "Å": "Ṛ",
+ "ñ": "ṣ",
+ "Ñ": "Ṣ",
+ "ö": "ṭ",
+ "Ö": "Ṭ",
+ "ë": "ṇ",
+ "Ë": "Ṇ",
+ "ò": "ḍ",
+ "Ò": "Ḍ",
+ "ç": "ś",
+ "Ç": "Ś",
+ "ì": "ṅ",
+ "Ì": "Ṅ",
+ "ï": "ñ",
+ "Ï": "Ñ",
+ "ü": "ū",
+ "Ü": "Ū",
+ "é": "ī",
+ "É": "Ī",
+ "à": "ṁ",
+ "ù": "ḥ",
+ "è": "ṝ",
+ "ÿ": "ḷ",
+}
+
+
+def decode_scagoudy(text: str) -> str:
+ """Replace ScaGoudy-encoded characters with proper IAST Unicode."""
+ for src, dst in SCAGOUDY_TO_IAST.items():
+ text = text.replace(src, dst)
+ return text
+
+
+def get_css_class(tag: Tag) -> str:
+ classes = tag.get("class", [])
+ return classes[0] if classes else ""
+
+
+def inner_html(tag: Tag) -> str:
+ """Get inner HTML of a tag, decode ScaGoudy, clean up spans."""
+ html = "".join(str(child) for child in tag.children)
+ html = decode_scagoudy(html)
+ # text → keep inner text (s1 is a ref style)
+ html = re.sub(
+ r'
)
+ html = re.sub(r"[ ]+", " ", html)
+ html = html.strip()
+ # Remove leading
tags
+ html = re.sub(r"^(
tags
+ html = re.sub(r"(
breaks
+ # (some verses use p14 or other classes but are entirely in tags)
+ return False
+
+
+def format_verse(tag: Tag) -> str:
+ """Format a verse paragraph as a center-aligned div."""
+ html = inner_html(tag)
+ if not html:
+ return ""
+ # Wrap in center-aligned div
+ return f'{html}'
+
+
+def write_json(data: dict, path: str):
+ os.makedirs(os.path.dirname(path), exist_ok=True)
+ with open(path, "w", encoding="utf-8") as f:
+ json.dump(data, f, indent=2, ensure_ascii=False)
+ f.write("\n")
+
+
+# ---------------------------------------------------------------------------
+# Parse volume/part info from p2 bookmark text
+# ---------------------------------------------------------------------------
+
+
+def parse_volume_part(text: str):
+ """Parse 'BTGPY1: Volume 1, Part 1, 1944' → (volume, part, date_str, slug).
+ Returns (volume_num, part_num, date_string, slug) or None.
+ """
+ # Remove the BTGPY prefix
+ text = text.strip()
+ # Match "Volume X, Part Y, ..."
+ m = re.search(r"Volume\s+(\d+),?\s*Part\s+(\d+),?\s*(.*)", text, re.IGNORECASE)
+ if m:
+ vol = int(m.group(1))
+ part = int(m.group(2))
+ date_str = m.group(3).strip().rstrip(" ")
+ slug = f"v{vol}-p{part}"
+ return vol, part, date_str, slug
+ return None
+
+
+# ---------------------------------------------------------------------------
+# Main extraction
+# ---------------------------------------------------------------------------
+
+
+def main():
+ base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+ html_path = os.path.join(base_dir, "public", "html", "btg-1960.html")
+ output_dir = os.path.join(base_dir, "lib", "jsons", "magazines", "btg")
+ issues_dir = os.path.join(output_dir, "issues")
+
+ if not os.path.exists(html_path):
+ print(f"ERROR: {html_path} not found!")
+ sys.exit(1)
+
+ print(f"Reading {html_path}...")
+ with open(html_path, "r", encoding="utf-8") as f:
+ content = f.read()
+
+ soup = BeautifulSoup(content, "html.parser")
+ all_tags = soup.find_all("p")
+ print(f" Parsed {len(all_tags)}
":
+ paragraphs.append(html)
+ continue
+
+ # Body text classes
+ html = inner_html(tag)
+ if html and html != "
":
+ # Check if this looks like a verse (entirely italic with
breaks)
+ stripped = html.strip()
+ if (
+ stripped.startswith("")
+ and "
+ plain = re.sub(r"", "", stripped)
+ lines = [
+ l.strip() for l in re.split(r"{stripped}'
+
+ paragraphs.append(html)
+
+ # Build display title
+ if date_str:
+ display_title = f"Volume {vol}, Part {part}, {date_str}"
+ else:
+ display_title = f"Volume {vol}, Part {part}"
+
+ issue_data = {
+ "book": "btg",
+ "volume": vol,
+ "part": part,
+ "title": display_title,
+ "slug": slug,
+ "articles": articles,
+ "paragraphs": paragraphs,
+ }
+ issues_data.append(issue_data)
+
+ print(
+ f" {slug}: {display_title} — "
+ f"{len(articles)} articles, {len(paragraphs)} paragraphs"
+ )
+
+ # ---------------------------------------------------------------------------
+ # Phase 3: Build intro.json with hierarchical volume groups
+ # ---------------------------------------------------------------------------
+
+ # Group issues by volume
+ volume_groups = {}
+ for issue in issues_data:
+ vol = issue["volume"]
+ if vol not in volume_groups:
+ volume_groups[vol] = []
+ volume_groups[vol].append(issue)
+
+ # Build chapters array for intro.json
+ chapters = []
+ for vol_num in sorted(volume_groups.keys()):
+ vol_issues = volume_groups[vol_num]
+ sub_chapters = []
+ for issue in vol_issues:
+ sub_chapters.append(
+ {
+ "slug": issue["slug"],
+ "title": issue["title"],
+ }
+ )
+ chapters.append(
+ {
+ "title": f"Volume {vol_num}",
+ "subChapters": sub_chapters,
+ }
+ )
+
+ total_issues = len(issues_data)
+ total_articles = sum(len(i["articles"]) for i in issues_data)
+
+ intro_json = {
+ "book": "btg",
+ "title": "Back to Godhead",
+ "author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
+ "year": 1944,
+ "description": f"The original Back to Godhead magazine founded in 1944. {total_issues} issues across {len(volume_groups)} volumes, containing {total_articles} articles on Kṛṣṇa consciousness, Vedic philosophy, and spiritual life.",
+ "totalIssues": total_issues,
+ "totalArticles": total_articles,
+ "chapters": chapters,
+ }
+
+ # ---------------------------------------------------------------------------
+ # Phase 4: Write JSON files
+ # ---------------------------------------------------------------------------
+
+ # Clean output directory
+ if os.path.exists(issues_dir):
+ for f in os.listdir(issues_dir):
+ os.remove(os.path.join(issues_dir, f))
+
+ # Write intro.json
+ write_json(intro_json, os.path.join(output_dir, "intro.json"))
+ print(f"\n Written: intro.json ({total_issues} issues, {total_articles} articles)")
+
+ # Write issue files
+ order = 100
+ for issue in issues_data:
+ issue_json = {
+ "book": "btg",
+ "volume": issue["volume"],
+ "part": issue["part"],
+ "title": issue["title"],
+ "slug": issue["slug"],
+ "order": order,
+ "articles": issue["articles"],
+ "paragraphs": issue["paragraphs"],
+ }
+ filepath = os.path.join(issues_dir, f"{issue['slug']}.json")
+ write_json(issue_json, filepath)
+ order += 1
+
+ print(f" Written: {len(issues_data)} issue files")
+
+ # ---------------------------------------------------------------------------
+ # Phase 5: Quality checks
+ # ---------------------------------------------------------------------------
+ print(f"\n{'=' * 60}")
+ print(f" QUALITY CHECKS")
+ print(f"{'=' * 60}")
+
+ # Check for ScaGoudy remnants
+ sca_chars = set(SCAGOUDY_TO_IAST.keys())
+ sca_remnants = 0
+ for issue in issues_data:
+ for p in issue["paragraphs"]:
+ if any(c in p for c in sca_chars if c not in ("é", "ü", "à")):
+ # Skip common French/German chars that might be legitimate
+ # Focus on clearly ScaGoudy-only chars
+ if "å" in p or "ë" in p or "ö" in p and "Ö" not in p:
+ sca_remnants += 1
+ break
+ if sca_remnants == 0:
+ print(f" ✅ No ScaGoudy remnants detected")
+ else:
+ print(f" ⚠ {sca_remnants} issues may have ScaGoudy remnants")
+
+ # Check for empty issues
+ empty_issues = [i for i in issues_data if len(i["paragraphs"]) == 0]
+ if empty_issues:
+ print(
+ f" ⚠ {len(empty_issues)} empty issues: {[i['slug'] for i in empty_issues]}"
+ )
+ else:
+ print(f" ✅ No empty issues")
+
+ # Check centered verses
+ verse_count = 0
+ for issue in issues_data:
+ for p in issue["paragraphs"]:
+ if '' in p:
+ verse_count += 1
+ print(f" ℹ️ {verse_count} center-aligned verse paragraphs")
+
+ # ---------------------------------------------------------------------------
+ # Summary
+ # ---------------------------------------------------------------------------
+ print(f"\n{'=' * 60}")
+ print(f" EXTRACTION SUMMARY")
+ print(f"{'=' * 60}")
+ for vol_num in sorted(volume_groups.keys()):
+ vol_issues = volume_groups[vol_num]
+ vol_articles = sum(len(i["articles"]) for i in vol_issues)
+ vol_paras = sum(len(i["paragraphs"]) for i in vol_issues)
+ print(
+ f" Volume {vol_num}: {len(vol_issues)} issues, {vol_articles} articles, {vol_paras} paragraphs"
+ )
+ for issue in vol_issues:
+ print(
+ f" {issue['slug']}: {issue['title']} "
+ f"({len(issue['articles'])} articles, {len(issue['paragraphs'])} paras)"
+ )
+ print(f"\n Total: {total_issues} issues, {total_articles} articles")
+ print(f"{'=' * 60}")
+ print(f"\nDone! ✨")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/letters_extract.py b/scripts/letters_extract.py
new file mode 100644
index 00000000..6190366e
--- /dev/null
+++ b/scripts/letters_extract.py
@@ -0,0 +1,689 @@
+#!/usr/bin/env python3
+"""Extract letters from letters.html.
+
+Each letter starts with a p7 tag containing "Letter to: '
+ paragraphs.append(html)
+ continue
+
+ # Signature/closing blocks
+ if cls in SIGNATURE_CLASSES:
+ html = inner_html(tag)
+ if html:
+ paragraphs.append(html)
+ continue
+
+ # All other body text classes (p11, p14, p16, p22, p28, p29, etc.)
+ html = inner_html(tag)
+ if html and html != "
tags
+ html = re.sub(r"^(
tags
+ html = re.sub(r"(
":
+ # Check for verse-like content
+ stripped = html.strip()
+ if (
+ stripped.startswith("")
+ and "{stripped}'
+
+ paragraphs.append(html)
+
+ # Clean recipient name — strip trailing punctuation
+ if recipient:
+ recipient = recipient.rstrip(":,.; ").strip()
+ # Also clean " :" (space before colon)
+ recipient = re.sub(r"\s+:$", "", recipient).strip()
+
+ # Split multi-recipient strings into individual names
+ # Patterns: "Mukunda, Janaki" or "Mukunda and Janaki"
+ recipient_display = recipient or "Unknown"
+ if recipient:
+ # Split by comma, semicolon, " and ", " & "
+ parts = re.split(r"\s*[,;]\s*|\s+and\s+|\s+&\s+", recipient)
+ recipients_list = []
+ for p in parts:
+ name = p.strip().rstrip(":,.; ").strip()
+ # Clean " :" pattern
+ name = re.sub(r"\s+:$", "", name).strip()
+ if name:
+ recipients_list.append(name)
+ if not recipients_list:
+ recipients_list = ["Unknown"]
+ else:
+ recipients_list = ["Unknown"]
+
+ # Build slug
+ slug = make_slug(recipient_display, date_ref, year, month, day, li)
+
+ # Ensure unique slug
+ if slug in slug_counts:
+ slug_counts[slug] += 1
+ slug = f"{slug}-{slug_counts[slug]}"
+ else:
+ slug_counts[slug] = 1
+
+ # Normalize location
+ if location:
+ # Clean up common variants
+ location = location.rstrip(",.")
+ # Normalize "Vrndavana" variants
+ if location.lower() in ("vrndavana", "vrndävana"):
+ location = "Vrindaban"
+
+ letter = {
+ "slug": slug,
+ "recipients": recipients_list,
+ "recipientDisplay": recipient_display,
+ "date": date_str,
+ "dateRef": date_ref,
+ "year": year,
+ "month": month,
+ "day": day,
+ "location": location,
+ "salutation": salutation,
+ "paragraphs": paragraphs,
+ }
+ letters.append(letter)
+
+ print(f"\n Extracted {len(letters)} letters")
+
+ # ---------------------------------------------------------------------------
+ # Phase 3: Statistics
+ # ---------------------------------------------------------------------------
+
+ # Year distribution
+ year_counts = Counter()
+ for letter in letters:
+ if letter["year"]:
+ year_counts[letter["year"]] += 1
+ print(f"\n Year distribution:")
+ for yr in sorted(year_counts.keys()):
+ if 1940 <= yr <= 1980:
+ print(f" {yr}: {year_counts[yr]} letters")
+
+ # Location distribution (top 20)
+ loc_counts = Counter()
+ for letter in letters:
+ if letter["location"]:
+ loc_counts[letter["location"]] += 1
+ print(f"\n Top 20 locations:")
+ for loc, count in loc_counts.most_common(20):
+ print(f" {loc}: {count}")
+
+ # Recipient distribution (by individual name, not group)
+ recip_counts = Counter()
+ for letter in letters:
+ for r in letter["recipients"]:
+ recip_counts[r] += 1
+ print(f"\n Top 20 recipients (individual):")
+ for recip, count in recip_counts.most_common(20):
+ print(f" {recip}: {count}")
+
+ # Letters with missing data
+ no_year = sum(1 for l in letters if not l["year"])
+ no_location = sum(1 for l in letters if not l["location"])
+ no_date = sum(1 for l in letters if not l["date"])
+ no_recipient = sum(1 for l in letters if l["recipients"] == ["Unknown"])
+ print(f"\n Missing data:")
+ print(f" No year: {no_year}")
+ print(f" No location: {no_location}")
+ print(f" No date: {no_date}")
+ print(f" No recipient (Unknown): {no_recipient}")
+
+ # Unique counts
+ all_individual_recipients = set()
+ for l in letters:
+ for r in l["recipients"]:
+ all_individual_recipients.add(r)
+ unique_recipients = len(all_individual_recipients)
+ unique_locations = len(set(l["location"] for l in letters if l["location"]))
+ unique_years = len(set(l["year"] for l in letters if l["year"]))
+ print(f"\n Unique counts:")
+ print(f" Recipients: {unique_recipients}")
+ print(f" Locations: {unique_locations}")
+ print(f" Years: {unique_years}")
+
+ # ---------------------------------------------------------------------------
+ # Phase 4: Write output
+ # ---------------------------------------------------------------------------
+
+ # Write individual letter files grouped by year for manageable sizes
+ years_with_letters = sorted(
+ set(l["year"] for l in letters if l["year"] and 1940 <= l["year"] <= 1980)
+ )
+
+ # Write a summary/index file
+ index_data = {
+ "totalLetters": len(letters),
+ "uniqueRecipients": unique_recipients,
+ "uniqueLocations": unique_locations,
+ "years": [],
+ "topRecipients": [
+ {"name": r, "count": c} for r, c in recip_counts.most_common(50)
+ ],
+ "topLocations": [
+ {"name": l, "count": c} for l, c in loc_counts.most_common(50)
+ ],
+ }
+
+ for yr in years_with_letters:
+ yr_letters = [l for l in letters if l["year"] == yr]
+ index_data["years"].append(
+ {
+ "year": yr,
+ "count": len(yr_letters),
+ }
+ )
+
+ # Also count letters without a valid year
+ undated = [
+ l for l in letters if not l["year"] or l["year"] < 1940 or l["year"] > 1980
+ ]
+ if undated:
+ index_data["years"].append(
+ {
+ "year": 0,
+ "count": len(undated),
+ }
+ )
+
+ write_json(index_data, os.path.join(output_dir, "index.json"))
+ print(f"\n Written: index.json")
+
+ # Write letters grouped by year (one JSON file per year)
+ for yr in years_with_letters:
+ yr_letters = [l for l in letters if l["year"] == yr]
+ yr_letters.sort(key=lambda l: (l["month"] or 0, l["day"] or 0))
+ write_json(yr_letters, os.path.join(output_dir, f"{yr}.json"))
+
+ # Write undated letters
+ if undated:
+ write_json(undated, os.path.join(output_dir, "undated.json"))
+
+ print(f" Written: {len(years_with_letters)} year files + undated.json")
+
+ # ---------------------------------------------------------------------------
+ # Phase 5: Quality checks
+ # ---------------------------------------------------------------------------
+ print(f"\n{'=' * 60}")
+ print(f" QUALITY CHECKS")
+ print(f"{'=' * 60}")
+
+ # Check for ScaGoudy remnants
+ sca_chars = {"å", "ë", "ö"}
+ sca_remnants = 0
+ for letter in letters[:100]: # Check first 100
+ for p in letter["paragraphs"]:
+ if any(c in p for c in sca_chars):
+ sca_remnants += 1
+ break
+ if sca_remnants == 0:
+ print(f" ✅ No ScaGoudy remnants detected (checked first 100)")
+ else:
+ print(f" ⚠ {sca_remnants} letters may have ScaGoudy remnants")
+
+ # Check empty letters
+ empty = [l for l in letters if len(l["paragraphs"]) == 0]
+ if empty:
+ print(f" ⚠ {len(empty)} empty letters (no paragraphs)")
+ else:
+ print(f" ✅ No empty letters")
+
+ # Check duplicate slugs
+ slug_list = [l["slug"] for l in letters]
+ dupes = len(slug_list) - len(set(slug_list))
+ if dupes:
+ print(f" ⚠ {dupes} duplicate slugs")
+ else:
+ print(f" ✅ No duplicate slugs")
+
+ # Verse count
+ verse_count = 0
+ for letter in letters:
+ for p in letter["paragraphs"]:
+ if '' in p:
+ verse_count += 1
+ print(f" ℹ️ {verse_count} center-aligned verse paragraphs")
+
+ # ---------------------------------------------------------------------------
+ # Summary
+ # ---------------------------------------------------------------------------
+ print(f"\n{'=' * 60}")
+ print(f" EXTRACTION SUMMARY")
+ print(f"{'=' * 60}")
+ print(f" Total letters: {len(letters)}")
+ print(f" Unique recipients: {unique_recipients}")
+ print(f" Unique locations: {unique_locations}")
+ print(f" Year range: {min(years_with_letters)}–{max(years_with_letters)}")
+ print(f" Total paragraphs: {sum(len(l['paragraphs']) for l in letters)}")
+ print(f"{'=' * 60}")
+ print(f"\nDone! ✨")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/seed-letters.ts b/scripts/seed-letters.ts
new file mode 100644
index 00000000..90d1b604
--- /dev/null
+++ b/scripts/seed-letters.ts
@@ -0,0 +1,158 @@
+/**
+ * seed-letters.ts — Push letters into MongoDB.
+ *
+ * Collection written:
+ * - letters → 6,866 individual letter documents
+ *
+ * JSON sources:
+ * - lib/jsons/letters/1947.json through 1977.json
+ * - lib/jsons/letters/undated.json
+ *
+ * Usage:
+ * npx tsx scripts/seed-letters.ts
+ */
+
+import dotenv from "dotenv";
+import fs from "fs";
+import path from "path";
+import { MongoClient } from "mongodb";
+
+dotenv.config({ path: ".env.local" });
+dotenv.config({ path: ".env" });
+
+const DATABASE_URL = process.env.DATABASE_URL;
+if (!DATABASE_URL) {
+ console.error("❌ DATABASE_URL is not set");
+ process.exit(1);
+}
+
+const BASE_DIR = path.resolve(__dirname, "..");
+const LETTERS_DIR = path.join(BASE_DIR, "lib", "jsons", "letters");
+
+function readJson