feat: letters added
This commit is contained in:
gosangam
@@ -0,0 +1,717 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract Back to Godhead magazine (1966-1973) from btg-1978.html.
|
||||
|
||||
Each magazine issue becomes its own JSON file.
|
||||
Issues are grouped by volume/era in the intro.json metadata.
|
||||
Advertisement sections are excluded.
|
||||
|
||||
Structure:
|
||||
- p2 = Issue bookmark (bold, e.g., "Back to Godhead Magazine. Vol. 1, No. 01, October 23, 1966")
|
||||
- p8 = Article bookmark within an issue
|
||||
- p9 = Article display title (bold)
|
||||
- p7, p5, p12, p4, p3, etc. = Body text
|
||||
- Articles titled "Advertisement" are skipped entirely
|
||||
|
||||
Output:
|
||||
lib/jsons/magazines/btg-1978/intro.json
|
||||
lib/jsons/magazines/btg-1978/issues/no-01.json
|
||||
lib/jsons/magazines/btg-1978/issues/no-02.json
|
||||
...
|
||||
lib/jsons/magazines/btg-1978/issues/no-54.json
|
||||
|
||||
Usage:
|
||||
python3 scripts/btg_1978_extract.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ScaGoudy → IAST Unicode mapping
|
||||
# ---------------------------------------------------------------------------
|
||||
SCAGOUDY_TO_IAST = {
|
||||
"ä": "ā",
|
||||
"Ä": "Ā",
|
||||
"å": "ṛ",
|
||||
"Å": "Ṛ",
|
||||
"ñ": "ṣ",
|
||||
"Ñ": "Ṣ",
|
||||
"ö": "ṭ",
|
||||
"Ö": "Ṭ",
|
||||
"ë": "ṇ",
|
||||
"Ë": "Ṇ",
|
||||
"ò": "ḍ",
|
||||
"Ò": "Ḍ",
|
||||
"ç": "ś",
|
||||
"Ç": "Ś",
|
||||
"ì": "ṅ",
|
||||
"Ì": "Ṅ",
|
||||
"ï": "ñ",
|
||||
"Ï": "Ñ",
|
||||
"ü": "ū",
|
||||
"Ü": "Ū",
|
||||
"é": "ī",
|
||||
"É": "Ī",
|
||||
"à": "ṁ",
|
||||
"ù": "ḥ",
|
||||
"è": "ṝ",
|
||||
"ÿ": "ḷ",
|
||||
}
|
||||
|
||||
|
||||
def decode_scagoudy(text: str) -> str:
|
||||
"""Replace ScaGoudy-encoded characters with proper IAST Unicode."""
|
||||
for src, dst in SCAGOUDY_TO_IAST.items():
|
||||
text = text.replace(src, dst)
|
||||
return text
|
||||
|
||||
|
||||
def get_css_class(tag: Tag) -> str:
|
||||
classes = tag.get("class", [])
|
||||
return classes[0] if classes else ""
|
||||
|
||||
|
||||
def inner_html(tag: Tag) -> str:
|
||||
"""Get inner HTML of a tag, decode ScaGoudy, clean up spans."""
|
||||
html = "".join(str(child) for child in tag.children)
|
||||
html = decode_scagoudy(html)
|
||||
|
||||
# <span class="s2">text</span> → <i>text</i> (italic marker in btg-1978)
|
||||
html = re.sub(
|
||||
r'<span[^>]*class="s2"[^>]*>(.*?)</span>', r"<i>\1</i>", html, flags=re.DOTALL
|
||||
)
|
||||
|
||||
# Remove all other span tags but keep inner content
|
||||
html = re.sub(r"<span[^>]*>(.*?)</span>", r"\1", html, flags=re.DOTALL)
|
||||
|
||||
# Remove Apple-tab-span and tab characters
|
||||
html = html.replace("\t", " ")
|
||||
|
||||
# Collapse multiple spaces (but not around <br>)
|
||||
html = re.sub(r"[ ]+", " ", html)
|
||||
html = html.strip()
|
||||
|
||||
# Remove leading <br> tags
|
||||
html = re.sub(r"^(<br\s*/?>[\s]*)+", "", html)
|
||||
# Remove trailing <br> tags
|
||||
html = re.sub(r"(<br\s*/?>[\s]*)+$", "", html)
|
||||
|
||||
return html
|
||||
|
||||
|
||||
def write_json(data: dict, path: str):
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
f.write("\n")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Parse issue info from p2 text
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def parse_issue_info(text: str):
|
||||
"""Parse issue info from the p2 bookmark text.
|
||||
|
||||
Handles various formats:
|
||||
"Back to Godhead Magazine. Vol. 1, No. 01, October 23, 1966"
|
||||
"Back to Godhead Magazine. Vol. 1, No. 06, January 20, 1967"
|
||||
"Back to Godhead Magazine. [VOL II No 1.] No. 16, 1968"
|
||||
"Back to Godhead Magazine VOL II. No 2. [#17], 1968"
|
||||
"Back to Godhead Magazine no. 30, 1969"
|
||||
"Back to Godhead Magazine No. 36, 1970"
|
||||
|
||||
Returns (issue_number, volume, display_title, date_str, slug) or None.
|
||||
"""
|
||||
text = text.strip()
|
||||
# Remove "Back to Godhead Magazine" prefix variations
|
||||
text_clean = re.sub(
|
||||
r"^Back to Godhead Magazine\.?\s*", "", text, flags=re.IGNORECASE
|
||||
).strip()
|
||||
|
||||
issue_num = None
|
||||
volume = None
|
||||
date_str = ""
|
||||
|
||||
# Try to extract issue number from various patterns
|
||||
|
||||
# Pattern 1: "Vol. X, No. YY, date" or "Vol. X No. YY, date"
|
||||
m = re.search(
|
||||
r"Vol\.?\s*(\d+)\.?\s*,?\s*No\.?\s*(\d+)\s*,?\s*(.*)", text_clean, re.IGNORECASE
|
||||
)
|
||||
if m:
|
||||
volume = int(m.group(1))
|
||||
issue_num = int(m.group(2))
|
||||
date_str = m.group(3).strip().rstrip(",. ")
|
||||
|
||||
# Pattern 2: "[VOL II No 1.] No. 16, 1968" — use the absolute number
|
||||
if issue_num is None:
|
||||
m = re.search(r"No\.?\s*(\d+)\s*,?\s*(\d{4})", text_clean, re.IGNORECASE)
|
||||
if m:
|
||||
issue_num = int(m.group(1))
|
||||
date_str = m.group(2).strip()
|
||||
|
||||
# Pattern 3: "VOL II. No 2. [#17], 1968"
|
||||
if issue_num is None:
|
||||
m = re.search(r"#(\d+)", text_clean)
|
||||
if m:
|
||||
issue_num = int(m.group(1))
|
||||
|
||||
# Pattern 4: "no. 30, 1969" or "no 31, 1969"
|
||||
if issue_num is None:
|
||||
m = re.search(r"no\.?\s*(\d+)\s*,?\s*(.*)", text_clean, re.IGNORECASE)
|
||||
if m:
|
||||
issue_num = int(m.group(1))
|
||||
date_str = m.group(2).strip().rstrip(",. ")
|
||||
|
||||
# Try to extract volume from text if not found yet
|
||||
if volume is None:
|
||||
m = re.search(r"VOL\.?\s*(?:II|2)", text_clean, re.IGNORECASE)
|
||||
if m:
|
||||
volume = 2
|
||||
else:
|
||||
m = re.search(r"VOL\.?\s*(?:I|1)", text_clean, re.IGNORECASE)
|
||||
if m:
|
||||
volume = 1
|
||||
|
||||
if issue_num is None:
|
||||
print(f" ⚠ Could not parse issue number from: {text}")
|
||||
return None
|
||||
|
||||
# Extract year from date_str if present
|
||||
year_match = re.search(r"(\d{4})", date_str)
|
||||
year = int(year_match.group(1)) if year_match else None
|
||||
|
||||
# Build display title
|
||||
if volume and issue_num <= 17:
|
||||
display_title = f"Vol. {volume}, No. {issue_num:02d}"
|
||||
if date_str:
|
||||
display_title += f", {date_str}"
|
||||
else:
|
||||
display_title = f"No. {issue_num}"
|
||||
if date_str:
|
||||
display_title += f", {date_str}"
|
||||
|
||||
slug = f"no-{issue_num:02d}"
|
||||
|
||||
return issue_num, volume, display_title, date_str, year, slug
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "btg-1978.html")
|
||||
output_dir = os.path.join(base_dir, "lib", "jsons", "magazines", "btg-1978")
|
||||
issues_dir = os.path.join(output_dir, "issues")
|
||||
|
||||
if not os.path.exists(html_path):
|
||||
print(f"ERROR: {html_path} not found!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Reading {html_path}...")
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
|
||||
soup = BeautifulSoup(content, "html.parser")
|
||||
all_tags = soup.find_all("p")
|
||||
print(f" Parsed {len(all_tags)} <p> tags total")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 1: Find all issue boundaries
|
||||
# ---------------------------------------------------------------------------
|
||||
issue_boundaries = [] # List of (tag_index, issue_num, volume, display_title, date_str, year, slug)
|
||||
|
||||
for i, tag in enumerate(all_tags):
|
||||
cls = get_css_class(tag)
|
||||
if cls == "p2":
|
||||
text = tag.get_text().strip()
|
||||
if "Back to Godhead" in text:
|
||||
parsed = parse_issue_info(text)
|
||||
if parsed:
|
||||
issue_num, volume, display_title, date_str, year, slug = parsed
|
||||
issue_boundaries.append(
|
||||
(i, issue_num, volume, display_title, date_str, year, slug)
|
||||
)
|
||||
|
||||
print(f"\n Found {len(issue_boundaries)} issues")
|
||||
for ib in issue_boundaries:
|
||||
tag_idx, inum, vol, title, ds, yr, slug = ib
|
||||
print(f" tag[{tag_idx}]: #{inum} (Vol {vol}) {title} → {slug}")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 2: Extract articles and paragraphs for each issue
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Known skip classes (bookmarks, empty lines)
|
||||
SKIP_CLASSES = {"p2", "p11", "p28", "p24"}
|
||||
|
||||
issues_data = []
|
||||
skipped_ads = 0
|
||||
|
||||
# Maximum paragraphs per issue — prevents mega-issues from absorbing
|
||||
# unmarked content that belongs to missing intermediate issues.
|
||||
MAX_PARAGRAPHS_PER_ISSUE = 500
|
||||
|
||||
for idx, (
|
||||
start_tag_idx,
|
||||
issue_num,
|
||||
vol,
|
||||
display_title,
|
||||
date_str,
|
||||
year,
|
||||
slug,
|
||||
) in enumerate(issue_boundaries):
|
||||
# Determine end boundary
|
||||
if idx + 1 < len(issue_boundaries):
|
||||
end_tag_idx = issue_boundaries[idx + 1][0]
|
||||
else:
|
||||
end_tag_idx = len(all_tags)
|
||||
|
||||
issue_tags = all_tags[start_tag_idx:end_tag_idx]
|
||||
|
||||
paragraphs = []
|
||||
articles = []
|
||||
in_advertisement = False
|
||||
|
||||
for tag in issue_tags:
|
||||
cls = get_css_class(tag)
|
||||
text = tag.get_text().strip()
|
||||
|
||||
# Skip bookmark tags and empty lines
|
||||
if cls in SKIP_CLASSES:
|
||||
continue
|
||||
if not text:
|
||||
continue
|
||||
|
||||
# p8 = Article bookmark
|
||||
if cls == "p8":
|
||||
article_title = decode_scagoudy(text).strip()
|
||||
|
||||
# Check if this is an advertisement
|
||||
if re.match(r"^Advertisements?", article_title, re.IGNORECASE):
|
||||
in_advertisement = True
|
||||
skipped_ads += 1
|
||||
continue
|
||||
else:
|
||||
in_advertisement = False
|
||||
articles.append({"title": article_title})
|
||||
continue
|
||||
|
||||
# p122 and other classes can also be advertisement markers
|
||||
if re.match(r"^Advertisements?\s*$", text, re.IGNORECASE):
|
||||
in_advertisement = True
|
||||
skipped_ads += 1
|
||||
continue
|
||||
|
||||
# p9 = Article display title
|
||||
if cls == "p9":
|
||||
if in_advertisement:
|
||||
continue
|
||||
# Also check the p9 display title itself for advertisement
|
||||
plain_title = decode_scagoudy(text).strip()
|
||||
if re.match(r"^Advertisements?\s*$", plain_title, re.IGNORECASE):
|
||||
in_advertisement = True
|
||||
skipped_ads += 1
|
||||
continue
|
||||
html = inner_html(tag)
|
||||
if html:
|
||||
# Make it a bold article heading if not already
|
||||
if not html.startswith("<b>"):
|
||||
html = f"<b>{html}</b>"
|
||||
paragraphs.append(html)
|
||||
continue
|
||||
|
||||
# Skip content within advertisement sections
|
||||
if in_advertisement:
|
||||
# Check if next p8 will reset it — for now just skip
|
||||
continue
|
||||
|
||||
# p15 = Verse block — center align
|
||||
if cls == "p15":
|
||||
html = inner_html(tag)
|
||||
if html:
|
||||
paragraphs.append(f'<div style="text-align:center">{html}</div>')
|
||||
continue
|
||||
|
||||
# All other classes — extract as body text
|
||||
html = inner_html(tag)
|
||||
if html and html != "<br>":
|
||||
# Check if this looks like a verse (entirely italic with <br> breaks)
|
||||
stripped = html.strip()
|
||||
if (
|
||||
stripped.startswith("<i>")
|
||||
and "<br" in stripped
|
||||
and '<div style="text-align:center">' not in stripped
|
||||
):
|
||||
plain = re.sub(r"</?[bi]>", "", stripped)
|
||||
lines = [
|
||||
l.strip() for l in re.split(r"<br\s*/?>", plain) if l.strip()
|
||||
]
|
||||
if len(lines) >= 2:
|
||||
html = f'<div style="text-align:center">{stripped}</div>'
|
||||
|
||||
paragraphs.append(html)
|
||||
|
||||
# Post-process: remove any ad-like paragraphs that slipped through.
|
||||
# Two-pass approach:
|
||||
# 1. Detect ad section start markers and skip until next real article
|
||||
# 2. Filter individual ad-like paragraphs by content patterns
|
||||
|
||||
# --- Pass 1: Section-level ad detection ---
|
||||
ad_section_patterns = [
|
||||
r"^Advertisements?\s*$",
|
||||
r"^CIRCULATION AND SUBSCRIPTIONS",
|
||||
r"^Order Form",
|
||||
r"^SPECIAL OFFER",
|
||||
r"^Subscriptions?:",
|
||||
r"^WORLDWIDE IMPORTS",
|
||||
r"^THE\s+ORIENT\s+SHOP",
|
||||
r"^PROTECT YOUR CHECKS",
|
||||
]
|
||||
ad_section_re = re.compile("|".join(ad_section_patterns), re.IGNORECASE)
|
||||
|
||||
cleaned_pass1 = []
|
||||
skip_until_next_bold = False
|
||||
for p in paragraphs:
|
||||
plain = re.sub(r"<[^>]+>", "", p).strip()
|
||||
# Detect ad section start
|
||||
if ad_section_re.search(plain):
|
||||
skip_until_next_bold = True
|
||||
continue
|
||||
if skip_until_next_bold:
|
||||
# A bold article heading that's NOT an ad resets the skip
|
||||
if p.startswith("<b>") and len(plain) < 200:
|
||||
if ad_section_re.search(plain):
|
||||
continue
|
||||
else:
|
||||
skip_until_next_bold = False
|
||||
cleaned_pass1.append(p)
|
||||
continue
|
||||
# If it looks like real content (long paragraph), stop skipping
|
||||
if len(plain) > 150:
|
||||
skip_until_next_bold = False
|
||||
cleaned_pass1.append(p)
|
||||
continue
|
||||
# Short paragraphs in ad zone — skip
|
||||
continue
|
||||
cleaned_pass1.append(p)
|
||||
|
||||
# --- Pass 2: Individual paragraph ad detection ---
|
||||
# These patterns catch standalone ad fragments that weren't part of
|
||||
# a marked advertisement section.
|
||||
ad_para_keywords = [
|
||||
"subscription",
|
||||
"per copy",
|
||||
"enclosed is",
|
||||
"please send me",
|
||||
"order form",
|
||||
"postage paid",
|
||||
"money order",
|
||||
"check or money order",
|
||||
"zip code",
|
||||
"state_____",
|
||||
"address_____",
|
||||
"city_____",
|
||||
"available from iskcon",
|
||||
"waterpipes",
|
||||
"djellabas",
|
||||
"panchos",
|
||||
"orient shop",
|
||||
"worldwide imports",
|
||||
"sullivan st",
|
||||
"bleecker",
|
||||
"check protector",
|
||||
"safeguards your checks",
|
||||
"satisfaction guaranteed",
|
||||
"import-export",
|
||||
"davis & manswell",
|
||||
"davis & manswell",
|
||||
"bedford ave",
|
||||
"brooklyn, n.y.",
|
||||
"immediate delivery",
|
||||
"free gift guide",
|
||||
"write for free",
|
||||
"free catalog",
|
||||
"free catalogue",
|
||||
"protect your checks",
|
||||
"name of purchaser",
|
||||
"such dresses at $",
|
||||
"and gowns yet $",
|
||||
"price per copy",
|
||||
"one-year subscription",
|
||||
"year subscription costs",
|
||||
"back issues available",
|
||||
"a subscription to",
|
||||
"send check or money",
|
||||
"your name & address",
|
||||
"your name & address",
|
||||
"674-9511",
|
||||
]
|
||||
# Also detect short paragraphs that look like addresses or phone numbers
|
||||
address_re = re.compile(
|
||||
r"^\d+\s+\w+\s+(st|street|ave|avenue|blvd|road|rd|lane|ln|dr|drive)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
phone_re = re.compile(r"^\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}\s*$")
|
||||
zipcode_re = re.compile(
|
||||
r"(N\.?Y\.?\s+\d{5}|CA\s+\d{5}|Brooklyn|Manhattan)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
cleaned = []
|
||||
for p in cleaned_pass1:
|
||||
plain = re.sub(r"<[^>]+>", "", p).strip()
|
||||
plain_lower = plain.lower()
|
||||
|
||||
# Skip if matches any ad keyword
|
||||
if any(kw in plain_lower for kw in ad_para_keywords):
|
||||
continue
|
||||
|
||||
# Skip short paragraphs that look like addresses
|
||||
if len(plain) < 80 and address_re.search(plain):
|
||||
continue
|
||||
|
||||
# Skip standalone phone numbers
|
||||
if phone_re.match(plain):
|
||||
continue
|
||||
|
||||
# Skip short paragraphs with zip codes (likely address fragments)
|
||||
if len(plain) < 60 and zipcode_re.search(plain):
|
||||
continue
|
||||
|
||||
# Skip price-only paragraphs like "$1 plus 25c"
|
||||
if re.match(
|
||||
r"^\$?\d+[\.\d]*\s*(plus|postage|each|per)", plain, re.IGNORECASE
|
||||
):
|
||||
continue
|
||||
|
||||
cleaned.append(p)
|
||||
paragraphs = cleaned
|
||||
|
||||
# Cap paragraphs to prevent mega-issues from absorbing unmarked
|
||||
# intermediate issues (e.g., issues 18-29 have no p2 markers and
|
||||
# would otherwise be absorbed into issue 17).
|
||||
if len(paragraphs) > MAX_PARAGRAPHS_PER_ISSUE:
|
||||
print(
|
||||
f" ⚠ {slug}: Capped from {len(paragraphs)} to {MAX_PARAGRAPHS_PER_ISSUE} paragraphs "
|
||||
f"(likely absorbed unmarked intermediate issues)"
|
||||
)
|
||||
paragraphs = paragraphs[:MAX_PARAGRAPHS_PER_ISSUE]
|
||||
|
||||
issue_data = {
|
||||
"book": "btg-1978",
|
||||
"volume": vol,
|
||||
"part": issue_num,
|
||||
"title": display_title,
|
||||
"slug": slug,
|
||||
"year": year,
|
||||
"articles": articles,
|
||||
"paragraphs": paragraphs,
|
||||
}
|
||||
issues_data.append(issue_data)
|
||||
|
||||
print(
|
||||
f" {slug}: {display_title} — "
|
||||
f"{len(articles)} articles, {len(paragraphs)} paragraphs"
|
||||
)
|
||||
|
||||
print(f"\n Skipped {skipped_ads} advertisement sections")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 3: Build intro.json with hierarchical groups
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Group issues by era/volume
|
||||
# Early issues (1-17) have Vol 1/Vol 2 designations
|
||||
# Later issues (30+) just have issue numbers
|
||||
group_early = [] # Issues 1-17 (1966-1968)
|
||||
group_later = [] # Issues 30+ (1969-1973)
|
||||
|
||||
for issue in issues_data:
|
||||
if issue["part"] <= 17:
|
||||
group_early.append(issue)
|
||||
else:
|
||||
group_later.append(issue)
|
||||
|
||||
chapters = []
|
||||
|
||||
if group_early:
|
||||
sub = []
|
||||
for issue in group_early:
|
||||
sub.append({"slug": issue["slug"], "title": issue["title"]})
|
||||
chapters.append(
|
||||
{
|
||||
"title": "Issues #1–17 (1966–1968)",
|
||||
"subChapters": sub,
|
||||
}
|
||||
)
|
||||
|
||||
if group_later:
|
||||
sub = []
|
||||
for issue in group_later:
|
||||
sub.append({"slug": issue["slug"], "title": issue["title"]})
|
||||
chapters.append(
|
||||
{
|
||||
"title": "Issues #30–54 (1969–1973)",
|
||||
"subChapters": sub,
|
||||
}
|
||||
)
|
||||
|
||||
total_issues = len(issues_data)
|
||||
total_articles = sum(len(i["articles"]) for i in issues_data)
|
||||
|
||||
# Determine year range
|
||||
years = [i["year"] for i in issues_data if i["year"]]
|
||||
min_year = min(years) if years else None
|
||||
max_year = max(years) if years else None
|
||||
|
||||
intro_json = {
|
||||
"book": "btg-1978",
|
||||
"title": "Back to Godhead (1966–1973)",
|
||||
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
||||
"year": min_year,
|
||||
"description": (
|
||||
f"Back to Godhead magazine issues from the Western era ({min_year}–{max_year}). "
|
||||
f"{total_issues} issues containing {total_articles} articles — lectures, essays, "
|
||||
f"poems, and philosophical discussions on Kṛṣṇa consciousness."
|
||||
),
|
||||
"totalIssues": total_issues,
|
||||
"totalArticles": total_articles,
|
||||
"flag": "Pre 1978",
|
||||
"chapters": chapters,
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 4: Write JSON files
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Clean output directory
|
||||
if os.path.exists(issues_dir):
|
||||
for f in os.listdir(issues_dir):
|
||||
os.remove(os.path.join(issues_dir, f))
|
||||
|
||||
# Write intro.json
|
||||
write_json(intro_json, os.path.join(output_dir, "intro.json"))
|
||||
print(f"\n Written: intro.json ({total_issues} issues, {total_articles} articles)")
|
||||
|
||||
# Write issue files
|
||||
order = 100
|
||||
for issue in issues_data:
|
||||
issue_json = {
|
||||
"book": "btg-1978",
|
||||
"volume": issue["volume"],
|
||||
"part": issue["part"],
|
||||
"title": issue["title"],
|
||||
"slug": issue["slug"],
|
||||
"order": order,
|
||||
"articles": issue["articles"],
|
||||
"paragraphs": issue["paragraphs"],
|
||||
}
|
||||
filepath = os.path.join(issues_dir, f"{issue['slug']}.json")
|
||||
write_json(issue_json, filepath)
|
||||
order += 1
|
||||
|
||||
print(f" Written: {len(issues_data)} issue files")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 5: Quality checks
|
||||
# ---------------------------------------------------------------------------
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f" QUALITY CHECKS")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
# Check for ScaGoudy remnants
|
||||
sca_chars = set(SCAGOUDY_TO_IAST.keys())
|
||||
sca_remnants = 0
|
||||
for issue in issues_data:
|
||||
for p in issue["paragraphs"]:
|
||||
if any(c in p for c in ("å", "ë", "ö")):
|
||||
sca_remnants += 1
|
||||
break
|
||||
if sca_remnants == 0:
|
||||
print(f" ✅ No ScaGoudy remnants detected")
|
||||
else:
|
||||
print(f" ⚠ {sca_remnants} issues may have ScaGoudy remnants")
|
||||
|
||||
# Check for empty issues
|
||||
empty_issues = [i for i in issues_data if len(i["paragraphs"]) == 0]
|
||||
if empty_issues:
|
||||
print(
|
||||
f" ⚠ {len(empty_issues)} empty issues: {[i['slug'] for i in empty_issues]}"
|
||||
)
|
||||
else:
|
||||
print(f" ✅ No empty issues")
|
||||
|
||||
# Check centered verses
|
||||
verse_count = 0
|
||||
for issue in issues_data:
|
||||
for p in issue["paragraphs"]:
|
||||
if '<div style="text-align:center">' in p:
|
||||
verse_count += 1
|
||||
print(f" ℹ️ {verse_count} center-aligned verse paragraphs")
|
||||
|
||||
# Check for remaining advertisement content
|
||||
ad_check = 0
|
||||
ad_keywords = ["advertisement", "subscribe", "subscription form"]
|
||||
for issue in issues_data:
|
||||
for p in issue["paragraphs"]:
|
||||
lower_p = p.lower()
|
||||
if any(kw in lower_p for kw in ad_keywords):
|
||||
# Only flag if the paragraph is primarily an ad
|
||||
if "advertisement" in lower_p and len(p) < 200:
|
||||
ad_check += 1
|
||||
if ad_check == 0:
|
||||
print(f" ✅ No advertisement content detected in output")
|
||||
else:
|
||||
print(f" ⚠ {ad_check} paragraphs may contain advertisement content")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Summary
|
||||
# ---------------------------------------------------------------------------
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f" EXTRACTION SUMMARY")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
if group_early:
|
||||
early_arts = sum(len(i["articles"]) for i in group_early)
|
||||
early_paras = sum(len(i["paragraphs"]) for i in group_early)
|
||||
print(
|
||||
f" Issues #1–17 (1966–1968): {len(group_early)} issues, "
|
||||
f"{early_arts} articles, {early_paras} paragraphs"
|
||||
)
|
||||
for issue in group_early:
|
||||
print(
|
||||
f" {issue['slug']}: {issue['title']} "
|
||||
f"({len(issue['articles'])} articles, {len(issue['paragraphs'])} paras)"
|
||||
)
|
||||
|
||||
if group_later:
|
||||
later_arts = sum(len(i["articles"]) for i in group_later)
|
||||
later_paras = sum(len(i["paragraphs"]) for i in group_later)
|
||||
print(
|
||||
f" Issues #30–54 (1969–1973): {len(group_later)} issues, "
|
||||
f"{later_arts} articles, {later_paras} paragraphs"
|
||||
)
|
||||
for issue in group_later:
|
||||
print(
|
||||
f" {issue['slug']}: {issue['title']} "
|
||||
f"({len(issue['articles'])} articles, {len(issue['paragraphs'])} paras)"
|
||||
)
|
||||
|
||||
print(f"\n Total: {total_issues} issues, {total_articles} articles")
|
||||
print(f" Skipped: {skipped_ads} advertisement sections")
|
||||
print(f"{'=' * 60}")
|
||||
print(f"\nDone! ✨")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,546 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract Back to Godhead magazine from btg-1960.html.
|
||||
|
||||
Each magazine issue becomes its own JSON file.
|
||||
Issues are grouped by volume in the intro.json metadata.
|
||||
|
||||
Structure:
|
||||
- p2 = Volume/Part bookmark (e.g., "BTGPY1: Volume 1, Part 1, 1944")
|
||||
- p3 = Volume/Part display title
|
||||
- p8 = Article bookmark within an issue (e.g., "BTGPY1a: Message of His Divine Grace")
|
||||
- p9 = Article display title (bold)
|
||||
- p12, p35 = Body paragraphs
|
||||
- p14, p17, p88 = Continuation notes / misc text
|
||||
- p15 = Verse blocks (italic, with ScaGoudy encoding)
|
||||
- p33, p40, p47, p54 = Magazine mastheads/headers
|
||||
|
||||
Output:
|
||||
lib/jsons/magazines/btg/intro.json
|
||||
lib/jsons/magazines/btg/issues/v1-p1.json
|
||||
lib/jsons/magazines/btg/issues/v1-p2.json
|
||||
...
|
||||
lib/jsons/magazines/btg/issues/v4-p1.json
|
||||
|
||||
Usage:
|
||||
python3 scripts/btg_extract.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ScaGoudy → IAST Unicode mapping
|
||||
# ---------------------------------------------------------------------------
|
||||
SCAGOUDY_TO_IAST = {
|
||||
"ä": "ā",
|
||||
"Ä": "Ā",
|
||||
"å": "ṛ",
|
||||
"Å": "Ṛ",
|
||||
"ñ": "ṣ",
|
||||
"Ñ": "Ṣ",
|
||||
"ö": "ṭ",
|
||||
"Ö": "Ṭ",
|
||||
"ë": "ṇ",
|
||||
"Ë": "Ṇ",
|
||||
"ò": "ḍ",
|
||||
"Ò": "Ḍ",
|
||||
"ç": "ś",
|
||||
"Ç": "Ś",
|
||||
"ì": "ṅ",
|
||||
"Ì": "Ṅ",
|
||||
"ï": "ñ",
|
||||
"Ï": "Ñ",
|
||||
"ü": "ū",
|
||||
"Ü": "Ū",
|
||||
"é": "ī",
|
||||
"É": "Ī",
|
||||
"à": "ṁ",
|
||||
"ù": "ḥ",
|
||||
"è": "ṝ",
|
||||
"ÿ": "ḷ",
|
||||
}
|
||||
|
||||
|
||||
def decode_scagoudy(text: str) -> str:
|
||||
"""Replace ScaGoudy-encoded characters with proper IAST Unicode."""
|
||||
for src, dst in SCAGOUDY_TO_IAST.items():
|
||||
text = text.replace(src, dst)
|
||||
return text
|
||||
|
||||
|
||||
def get_css_class(tag: Tag) -> str:
|
||||
classes = tag.get("class", [])
|
||||
return classes[0] if classes else ""
|
||||
|
||||
|
||||
def inner_html(tag: Tag) -> str:
|
||||
"""Get inner HTML of a tag, decode ScaGoudy, clean up spans."""
|
||||
html = "".join(str(child) for child in tag.children)
|
||||
html = decode_scagoudy(html)
|
||||
# <span class="s1">text</span> → keep inner text (s1 is a ref style)
|
||||
html = re.sub(
|
||||
r'<span[^>]*class="s1"[^>]*>(.*?)</span>', r"\1", html, flags=re.DOTALL
|
||||
)
|
||||
# <span class="s3">text</span> → keep inner text (superscript digit)
|
||||
html = re.sub(
|
||||
r'<span[^>]*class="s3"[^>]*>(.*?)</span>', r"\1", html, flags=re.DOTALL
|
||||
)
|
||||
# <span class="s4">text</span> → keep inner text
|
||||
html = re.sub(
|
||||
r'<span[^>]*class="s4"[^>]*>(.*?)</span>', r"\1", html, flags=re.DOTALL
|
||||
)
|
||||
# <span class="s5">text</span> → keep inner text
|
||||
html = re.sub(
|
||||
r'<span[^>]*class="s5"[^>]*>(.*?)</span>', r"\1", html, flags=re.DOTALL
|
||||
)
|
||||
# <span class="s7">text</span> → keep inner text
|
||||
html = re.sub(
|
||||
r'<span[^>]*class="s7"[^>]*>(.*?)</span>', r"\1", html, flags=re.DOTALL
|
||||
)
|
||||
# <span class="s8">text</span> → keep inner text
|
||||
html = re.sub(
|
||||
r'<span[^>]*class="s8"[^>]*>(.*?)</span>', r"\1", html, flags=re.DOTALL
|
||||
)
|
||||
# Remove any remaining spans but keep content
|
||||
html = re.sub(r"<span[^>]*>(.*?)</span>", r"\1", html, flags=re.DOTALL)
|
||||
# Remove Apple-tab-span
|
||||
html = html.replace("\t", " ")
|
||||
# Collapse multiple spaces (but not around <br>)
|
||||
html = re.sub(r"[ ]+", " ", html)
|
||||
html = html.strip()
|
||||
# Remove leading <br> tags
|
||||
html = re.sub(r"^(<br\s*/?>[\s]*)+", "", html)
|
||||
# Remove trailing <br> tags
|
||||
html = re.sub(r"(<br\s*/?>[\s]*)+$", "", html)
|
||||
return html
|
||||
|
||||
|
||||
def is_verse_paragraph(tag: Tag) -> bool:
|
||||
"""Check if a paragraph is a verse block (italic with line breaks)."""
|
||||
cls = get_css_class(tag)
|
||||
# p15 is commonly used for verse blocks
|
||||
if cls == "p15":
|
||||
return True
|
||||
# Also check for paragraphs that are entirely italic with <br> breaks
|
||||
# (some verses use p14 or other classes but are entirely in <i> tags)
|
||||
return False
|
||||
|
||||
|
||||
def format_verse(tag: Tag) -> str:
|
||||
"""Format a verse paragraph as a center-aligned div."""
|
||||
html = inner_html(tag)
|
||||
if not html:
|
||||
return ""
|
||||
# Wrap in center-aligned div
|
||||
return f'<div style="text-align:center">{html}</div>'
|
||||
|
||||
|
||||
def write_json(data: dict, path: str):
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
f.write("\n")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Parse volume/part info from p2 bookmark text
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def parse_volume_part(text: str):
|
||||
"""Parse 'BTGPY1: Volume 1, Part 1, 1944' → (volume, part, date_str, slug).
|
||||
Returns (volume_num, part_num, date_string, slug) or None.
|
||||
"""
|
||||
# Remove the BTGPY prefix
|
||||
text = text.strip()
|
||||
# Match "Volume X, Part Y, ..."
|
||||
m = re.search(r"Volume\s+(\d+),?\s*Part\s+(\d+),?\s*(.*)", text, re.IGNORECASE)
|
||||
if m:
|
||||
vol = int(m.group(1))
|
||||
part = int(m.group(2))
|
||||
date_str = m.group(3).strip().rstrip(" ")
|
||||
slug = f"v{vol}-p{part}"
|
||||
return vol, part, date_str, slug
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "btg-1960.html")
|
||||
output_dir = os.path.join(base_dir, "lib", "jsons", "magazines", "btg")
|
||||
issues_dir = os.path.join(output_dir, "issues")
|
||||
|
||||
if not os.path.exists(html_path):
|
||||
print(f"ERROR: {html_path} not found!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Reading {html_path}...")
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
|
||||
soup = BeautifulSoup(content, "html.parser")
|
||||
all_tags = soup.find_all("p")
|
||||
print(f" Parsed {len(all_tags)} <p> tags total")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 1: Find all issue boundaries
|
||||
# ---------------------------------------------------------------------------
|
||||
# Each issue starts with a p2 bookmark tag
|
||||
issue_boundaries = [] # List of (tag_index, volume, part, date_str, slug)
|
||||
|
||||
for i, tag in enumerate(all_tags):
|
||||
cls = get_css_class(tag)
|
||||
if cls == "p2":
|
||||
text = tag.get_text().strip()
|
||||
parsed = parse_volume_part(text)
|
||||
if parsed:
|
||||
vol, part, date_str, slug = parsed
|
||||
issue_boundaries.append((i, vol, part, date_str, slug))
|
||||
|
||||
print(f"\n Found {len(issue_boundaries)} issues")
|
||||
for ib in issue_boundaries:
|
||||
print(f" tag[{ib[0]}]: Vol {ib[1]}, Part {ib[2]}, {ib[3]} → {ib[4]}")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 2: Extract articles and paragraphs for each issue
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Classes to skip entirely
|
||||
SKIP_CLASSES = {"p2", "p11", "p24", "p28"}
|
||||
|
||||
# Classes that are header/masthead material (include as paragraphs)
|
||||
MASTHEAD_CLASSES = {"p3", "p4", "p5", "p6", "p7", "p32", "p33", "p34", "p40", "p47"}
|
||||
|
||||
# Classes that are body text
|
||||
BODY_CLASSES = {
|
||||
"p10",
|
||||
"p12",
|
||||
"p13",
|
||||
"p14",
|
||||
"p15",
|
||||
"p16",
|
||||
"p17",
|
||||
"p35",
|
||||
"p36",
|
||||
"p37",
|
||||
"p38",
|
||||
"p39",
|
||||
"p41",
|
||||
"p42",
|
||||
"p43",
|
||||
"p44",
|
||||
"p45",
|
||||
"p46",
|
||||
"p48",
|
||||
"p49",
|
||||
"p50",
|
||||
"p51",
|
||||
"p52",
|
||||
"p53",
|
||||
"p54",
|
||||
"p55",
|
||||
"p56",
|
||||
"p57",
|
||||
"p58",
|
||||
"p59",
|
||||
"p60",
|
||||
"p61",
|
||||
"p62",
|
||||
"p63",
|
||||
"p64",
|
||||
"p65",
|
||||
"p66",
|
||||
"p67",
|
||||
"p68",
|
||||
"p69",
|
||||
"p70",
|
||||
"p71",
|
||||
"p72",
|
||||
"p73",
|
||||
"p74",
|
||||
"p75",
|
||||
"p76",
|
||||
"p77",
|
||||
"p78",
|
||||
"p79",
|
||||
"p80",
|
||||
"p81",
|
||||
"p82",
|
||||
"p83",
|
||||
"p84",
|
||||
"p85",
|
||||
"p86",
|
||||
"p87",
|
||||
"p88",
|
||||
"p89",
|
||||
"p90",
|
||||
"p91",
|
||||
"p92",
|
||||
"p93",
|
||||
"p94",
|
||||
"p95",
|
||||
"p96",
|
||||
"p97",
|
||||
"p98",
|
||||
"p99",
|
||||
}
|
||||
|
||||
issues_data = []
|
||||
|
||||
for idx, (start_tag_idx, vol, part, date_str, slug) in enumerate(issue_boundaries):
|
||||
# Determine end boundary
|
||||
if idx + 1 < len(issue_boundaries):
|
||||
end_tag_idx = issue_boundaries[idx + 1][0]
|
||||
else:
|
||||
end_tag_idx = len(all_tags)
|
||||
|
||||
issue_tags = all_tags[start_tag_idx:end_tag_idx]
|
||||
|
||||
paragraphs = []
|
||||
articles = [] # List of { title, slug }
|
||||
|
||||
for tag in issue_tags:
|
||||
cls = get_css_class(tag)
|
||||
text = tag.get_text().strip()
|
||||
|
||||
# Skip bookmark tags and empty lines
|
||||
if cls in SKIP_CLASSES:
|
||||
continue
|
||||
if not text and cls not in ("p15",):
|
||||
continue
|
||||
|
||||
# p8 = Article bookmark — extract title for metadata
|
||||
if cls == "p8":
|
||||
# Extract article title from bookmark text (remove BTGPYxx: prefix)
|
||||
article_text = text
|
||||
m = re.match(r"BTGPY\d+[a-z]?:\s*(.*)", article_text)
|
||||
if m:
|
||||
article_title = m.group(1).strip()
|
||||
else:
|
||||
article_title = article_text
|
||||
article_title = decode_scagoudy(article_title)
|
||||
articles.append({"title": article_title})
|
||||
continue
|
||||
|
||||
# p9 = Article display title — render as bold heading
|
||||
if cls == "p9":
|
||||
html = inner_html(tag)
|
||||
if html:
|
||||
# Make it a bold article heading
|
||||
if not html.startswith("<b>"):
|
||||
html = f"<b>{html}</b>"
|
||||
paragraphs.append(html)
|
||||
continue
|
||||
|
||||
# p15 = Verse block — center align
|
||||
if cls == "p15":
|
||||
html = format_verse(tag)
|
||||
if html:
|
||||
paragraphs.append(html)
|
||||
continue
|
||||
|
||||
# p3 = Volume title — skip (it's the display title of the issue header)
|
||||
if cls == "p3":
|
||||
continue
|
||||
|
||||
# Masthead classes — include but as regular paragraphs
|
||||
if cls in MASTHEAD_CLASSES:
|
||||
html = inner_html(tag)
|
||||
if html and html != "<br>":
|
||||
paragraphs.append(html)
|
||||
continue
|
||||
|
||||
# Body text classes
|
||||
html = inner_html(tag)
|
||||
if html and html != "<br>":
|
||||
# Check if this looks like a verse (entirely italic with <br> breaks)
|
||||
stripped = html.strip()
|
||||
if (
|
||||
stripped.startswith("<i>")
|
||||
and "<br" in stripped
|
||||
and not '<div style="text-align:center">' in stripped
|
||||
):
|
||||
# Check if it's predominantly a verse (italic block with line breaks)
|
||||
# by seeing if removing the <i> tags leaves mostly text with <br>
|
||||
plain = re.sub(r"</?[bi]>", "", stripped)
|
||||
lines = [
|
||||
l.strip() for l in re.split(r"<br\s*/?>", plain) if l.strip()
|
||||
]
|
||||
if len(lines) >= 2:
|
||||
html = f'<div style="text-align:center">{stripped}</div>'
|
||||
|
||||
paragraphs.append(html)
|
||||
|
||||
# Build display title
|
||||
if date_str:
|
||||
display_title = f"Volume {vol}, Part {part}, {date_str}"
|
||||
else:
|
||||
display_title = f"Volume {vol}, Part {part}"
|
||||
|
||||
issue_data = {
|
||||
"book": "btg",
|
||||
"volume": vol,
|
||||
"part": part,
|
||||
"title": display_title,
|
||||
"slug": slug,
|
||||
"articles": articles,
|
||||
"paragraphs": paragraphs,
|
||||
}
|
||||
issues_data.append(issue_data)
|
||||
|
||||
print(
|
||||
f" {slug}: {display_title} — "
|
||||
f"{len(articles)} articles, {len(paragraphs)} paragraphs"
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 3: Build intro.json with hierarchical volume groups
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Group issues by volume
|
||||
volume_groups = {}
|
||||
for issue in issues_data:
|
||||
vol = issue["volume"]
|
||||
if vol not in volume_groups:
|
||||
volume_groups[vol] = []
|
||||
volume_groups[vol].append(issue)
|
||||
|
||||
# Build chapters array for intro.json
|
||||
chapters = []
|
||||
for vol_num in sorted(volume_groups.keys()):
|
||||
vol_issues = volume_groups[vol_num]
|
||||
sub_chapters = []
|
||||
for issue in vol_issues:
|
||||
sub_chapters.append(
|
||||
{
|
||||
"slug": issue["slug"],
|
||||
"title": issue["title"],
|
||||
}
|
||||
)
|
||||
chapters.append(
|
||||
{
|
||||
"title": f"Volume {vol_num}",
|
||||
"subChapters": sub_chapters,
|
||||
}
|
||||
)
|
||||
|
||||
total_issues = len(issues_data)
|
||||
total_articles = sum(len(i["articles"]) for i in issues_data)
|
||||
|
||||
intro_json = {
|
||||
"book": "btg",
|
||||
"title": "Back to Godhead",
|
||||
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
||||
"year": 1944,
|
||||
"description": f"The original Back to Godhead magazine founded in 1944. {total_issues} issues across {len(volume_groups)} volumes, containing {total_articles} articles on Kṛṣṇa consciousness, Vedic philosophy, and spiritual life.",
|
||||
"totalIssues": total_issues,
|
||||
"totalArticles": total_articles,
|
||||
"chapters": chapters,
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 4: Write JSON files
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Clean output directory
|
||||
if os.path.exists(issues_dir):
|
||||
for f in os.listdir(issues_dir):
|
||||
os.remove(os.path.join(issues_dir, f))
|
||||
|
||||
# Write intro.json
|
||||
write_json(intro_json, os.path.join(output_dir, "intro.json"))
|
||||
print(f"\n Written: intro.json ({total_issues} issues, {total_articles} articles)")
|
||||
|
||||
# Write issue files
|
||||
order = 100
|
||||
for issue in issues_data:
|
||||
issue_json = {
|
||||
"book": "btg",
|
||||
"volume": issue["volume"],
|
||||
"part": issue["part"],
|
||||
"title": issue["title"],
|
||||
"slug": issue["slug"],
|
||||
"order": order,
|
||||
"articles": issue["articles"],
|
||||
"paragraphs": issue["paragraphs"],
|
||||
}
|
||||
filepath = os.path.join(issues_dir, f"{issue['slug']}.json")
|
||||
write_json(issue_json, filepath)
|
||||
order += 1
|
||||
|
||||
print(f" Written: {len(issues_data)} issue files")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 5: Quality checks
|
||||
# ---------------------------------------------------------------------------
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f" QUALITY CHECKS")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
# Check for ScaGoudy remnants
|
||||
sca_chars = set(SCAGOUDY_TO_IAST.keys())
|
||||
sca_remnants = 0
|
||||
for issue in issues_data:
|
||||
for p in issue["paragraphs"]:
|
||||
if any(c in p for c in sca_chars if c not in ("é", "ü", "à")):
|
||||
# Skip common French/German chars that might be legitimate
|
||||
# Focus on clearly ScaGoudy-only chars
|
||||
if "å" in p or "ë" in p or "ö" in p and "Ö" not in p:
|
||||
sca_remnants += 1
|
||||
break
|
||||
if sca_remnants == 0:
|
||||
print(f" ✅ No ScaGoudy remnants detected")
|
||||
else:
|
||||
print(f" ⚠ {sca_remnants} issues may have ScaGoudy remnants")
|
||||
|
||||
# Check for empty issues
|
||||
empty_issues = [i for i in issues_data if len(i["paragraphs"]) == 0]
|
||||
if empty_issues:
|
||||
print(
|
||||
f" ⚠ {len(empty_issues)} empty issues: {[i['slug'] for i in empty_issues]}"
|
||||
)
|
||||
else:
|
||||
print(f" ✅ No empty issues")
|
||||
|
||||
# Check centered verses
|
||||
verse_count = 0
|
||||
for issue in issues_data:
|
||||
for p in issue["paragraphs"]:
|
||||
if '<div style="text-align:center">' in p:
|
||||
verse_count += 1
|
||||
print(f" ℹ️ {verse_count} center-aligned verse paragraphs")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Summary
|
||||
# ---------------------------------------------------------------------------
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f" EXTRACTION SUMMARY")
|
||||
print(f"{'=' * 60}")
|
||||
for vol_num in sorted(volume_groups.keys()):
|
||||
vol_issues = volume_groups[vol_num]
|
||||
vol_articles = sum(len(i["articles"]) for i in vol_issues)
|
||||
vol_paras = sum(len(i["paragraphs"]) for i in vol_issues)
|
||||
print(
|
||||
f" Volume {vol_num}: {len(vol_issues)} issues, {vol_articles} articles, {vol_paras} paragraphs"
|
||||
)
|
||||
for issue in vol_issues:
|
||||
print(
|
||||
f" {issue['slug']}: {issue['title']} "
|
||||
f"({len(issue['articles'])} articles, {len(issue['paragraphs'])} paras)"
|
||||
)
|
||||
print(f"\n Total: {total_issues} issues, {total_articles} articles")
|
||||
print(f"{'=' * 60}")
|
||||
print(f"\nDone! ✨")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,689 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract letters from letters.html.
|
||||
|
||||
Each letter starts with a p7 tag containing "Letter to: <recipient>".
|
||||
The header contains recipient, location, and date information.
|
||||
|
||||
Structure:
|
||||
- p7 = Letter header (recipient, dash, location, date)
|
||||
- p8 = Reference code (YY-MM-DD) and also postscript/closing lines
|
||||
- p9 = Recipient address block
|
||||
- p10 = Salutation ("Dear...")
|
||||
- p11 = Body paragraphs
|
||||
- p12 = Centered text (verses, centered quotes)
|
||||
- p13 = Closing/signature block (color: #408080)
|
||||
- p14 = Indented/numbered items
|
||||
- p16-p22, p28-p37 = Various body text styles
|
||||
|
||||
Output:
|
||||
lib/jsons/letters/letters.json (array of all letters with metadata)
|
||||
|
||||
Usage:
|
||||
python3 scripts/letters_extract.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ScaGoudy → IAST Unicode mapping
|
||||
# ---------------------------------------------------------------------------
|
||||
SCAGOUDY_TO_IAST = {
|
||||
"ä": "ā",
|
||||
"Ä": "Ā",
|
||||
"å": "ṛ",
|
||||
"Å": "Ṛ",
|
||||
"ñ": "ṣ",
|
||||
"Ñ": "Ṣ",
|
||||
"ö": "ṭ",
|
||||
"Ö": "Ṭ",
|
||||
"ë": "ṇ",
|
||||
"Ë": "Ṇ",
|
||||
"ò": "ḍ",
|
||||
"Ò": "Ḍ",
|
||||
"ç": "ś",
|
||||
"Ç": "Ś",
|
||||
"ì": "ṅ",
|
||||
"Ì": "Ṅ",
|
||||
"ï": "ñ",
|
||||
"Ï": "Ñ",
|
||||
"ü": "ū",
|
||||
"Ü": "Ū",
|
||||
"é": "ī",
|
||||
"É": "Ī",
|
||||
"à": "ṁ",
|
||||
"ù": "ḥ",
|
||||
"è": "ṝ",
|
||||
"ÿ": "ḷ",
|
||||
}
|
||||
|
||||
|
||||
def decode_scagoudy(text: str) -> str:
|
||||
"""Replace ScaGoudy-encoded characters with proper IAST Unicode."""
|
||||
for src, dst in SCAGOUDY_TO_IAST.items():
|
||||
text = text.replace(src, dst)
|
||||
return text
|
||||
|
||||
|
||||
def get_css_class(tag: Tag) -> str:
|
||||
classes = tag.get("class", [])
|
||||
return classes[0] if classes else ""
|
||||
|
||||
|
||||
def inner_html(tag: Tag) -> str:
|
||||
"""Get inner HTML of a tag, decode ScaGoudy, clean up spans."""
|
||||
html = "".join(str(child) for child in tag.children)
|
||||
html = decode_scagoudy(html)
|
||||
|
||||
# <span class="s1">text</span> → keep inner text (reference style)
|
||||
html = re.sub(
|
||||
r'<span[^>]*class="s\d+"[^>]*>(.*?)</span>', r"\1", html, flags=re.DOTALL
|
||||
)
|
||||
# Remove any remaining spans but keep content
|
||||
html = re.sub(r"<span[^>]*>(.*?)</span>", r"\1", html, flags=re.DOTALL)
|
||||
# Remove tab characters
|
||||
html = html.replace("\t", " ")
|
||||
# Collapse multiple spaces
|
||||
html = re.sub(r"[ ]+", " ", html)
|
||||
html = html.strip()
|
||||
# Remove leading <br> tags
|
||||
html = re.sub(r"^(<br\s*/?>[\s]*)+", "", html)
|
||||
# Remove trailing <br> tags
|
||||
html = re.sub(r"(<br\s*/?>[\s]*)+$", "", html)
|
||||
return html
|
||||
|
||||
|
||||
def write_json(data, path: str):
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
f.write("\n")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Month name → number mapping
|
||||
# ---------------------------------------------------------------------------
|
||||
MONTH_MAP = {
|
||||
"january": 1,
|
||||
"february": 2,
|
||||
"march": 3,
|
||||
"april": 4,
|
||||
"may": 5,
|
||||
"june": 6,
|
||||
"july": 7,
|
||||
"august": 8,
|
||||
"september": 9,
|
||||
"october": 10,
|
||||
"november": 11,
|
||||
"december": 12,
|
||||
"jan": 1,
|
||||
"feb": 2,
|
||||
"mar": 3,
|
||||
"apr": 4,
|
||||
"jun": 6,
|
||||
"jul": 7,
|
||||
"aug": 8,
|
||||
"sep": 9,
|
||||
"sept": 9,
|
||||
"oct": 10,
|
||||
"nov": 11,
|
||||
"dec": 12,
|
||||
}
|
||||
|
||||
|
||||
def parse_date(date_str: str):
|
||||
"""Parse a date string like '12 July, 1947' or 'May 1949'.
|
||||
|
||||
Returns (year, month, day) where month and day can be None.
|
||||
"""
|
||||
if not date_str:
|
||||
return None, None, None
|
||||
|
||||
date_str = date_str.strip().rstrip(".")
|
||||
|
||||
# Try "DD Month, YYYY" or "DD Month YYYY"
|
||||
m = re.match(r"(\d{1,2})\s+([A-Za-z]+),?\s*(\d{4})", date_str)
|
||||
if m:
|
||||
day = int(m.group(1))
|
||||
month_name = m.group(2).lower()
|
||||
year = int(m.group(3))
|
||||
month = MONTH_MAP.get(month_name)
|
||||
if year >= 1900 and year <= 2000:
|
||||
return year, month, day
|
||||
|
||||
# Try "Month DD, YYYY" or "Month DD YYYY"
|
||||
m = re.match(r"([A-Za-z]+)\s+(\d{1,2}),?\s*(\d{4})", date_str)
|
||||
if m:
|
||||
month_name = m.group(1).lower()
|
||||
day = int(m.group(2))
|
||||
year = int(m.group(3))
|
||||
month = MONTH_MAP.get(month_name)
|
||||
if year >= 1900 and year <= 2000:
|
||||
return year, month, day
|
||||
|
||||
# Try "Month, YYYY" or "Month YYYY"
|
||||
m = re.match(r"([A-Za-z]+),?\s*(\d{4})", date_str)
|
||||
if m:
|
||||
month_name = m.group(1).lower()
|
||||
year = int(m.group(2))
|
||||
month = MONTH_MAP.get(month_name)
|
||||
if month and year >= 1900 and year <= 2000:
|
||||
return year, month, None
|
||||
|
||||
# Try just "YYYY"
|
||||
m = re.match(r"(\d{4})$", date_str.strip())
|
||||
if m:
|
||||
year = int(m.group(1))
|
||||
if year >= 1900 and year <= 2000:
|
||||
return year, None, None
|
||||
|
||||
return None, None, None
|
||||
|
||||
|
||||
def parse_header(tag: Tag):
|
||||
"""Parse a p7 letter header tag.
|
||||
|
||||
Expected format:
|
||||
Letter to: <recipient>
|
||||
—
|
||||
<location>
|
||||
<date>
|
||||
|
||||
Or sometimes:
|
||||
Letter to: <recipient>
|
||||
<date>
|
||||
|
||||
Returns (recipient, location, date_str).
|
||||
"""
|
||||
text = tag.get_text().strip()
|
||||
lines = [l.strip() for l in text.split("\n") if l.strip()]
|
||||
|
||||
recipient = None
|
||||
location = None
|
||||
date_str = None
|
||||
|
||||
if not lines:
|
||||
return recipient, location, date_str
|
||||
|
||||
# First line: "Letter to: <recipient>"
|
||||
first = lines[0]
|
||||
m = re.match(r"Letter to:\s*(.*)", first, re.IGNORECASE)
|
||||
if m:
|
||||
recipient = m.group(1).strip().rstrip(",.")
|
||||
recipient = decode_scagoudy(recipient)
|
||||
|
||||
# Remove "—" lines
|
||||
content_lines = [l for l in lines[1:] if l != "—" and l != "-"]
|
||||
|
||||
if len(content_lines) >= 2:
|
||||
# Last line is usually the date
|
||||
potential_date = content_lines[-1]
|
||||
year, month, day = parse_date(potential_date)
|
||||
if year:
|
||||
date_str = potential_date
|
||||
# Everything before the date is location
|
||||
loc_lines = content_lines[:-1]
|
||||
if loc_lines:
|
||||
location = loc_lines[0].strip().rstrip(",.")
|
||||
location = decode_scagoudy(location)
|
||||
else:
|
||||
# Maybe the second-to-last is location and last is something else
|
||||
# Try to find a date anywhere
|
||||
for cl in reversed(content_lines):
|
||||
yr, mo, dy = parse_date(cl)
|
||||
if yr:
|
||||
date_str = cl
|
||||
idx = content_lines.index(cl)
|
||||
if idx > 0:
|
||||
location = content_lines[0].strip().rstrip(",.")
|
||||
location = decode_scagoudy(location)
|
||||
break
|
||||
elif len(content_lines) == 1:
|
||||
# Could be just a date, or just a location
|
||||
potential_date = content_lines[0]
|
||||
year, month, day = parse_date(potential_date)
|
||||
if year:
|
||||
date_str = potential_date
|
||||
else:
|
||||
location = potential_date.strip().rstrip(",.")
|
||||
location = decode_scagoudy(location)
|
||||
|
||||
return recipient, location, date_str
|
||||
|
||||
|
||||
def make_slug(recipient: str, date_ref: str, year, month, day, index: int) -> str:
|
||||
"""Create a unique slug for a letter."""
|
||||
parts = []
|
||||
|
||||
# Date part
|
||||
if date_ref and re.match(r"\d{2}-\d{2}", date_ref):
|
||||
parts.append(date_ref)
|
||||
elif year:
|
||||
yr = str(year)[2:] # Last 2 digits
|
||||
if month:
|
||||
if day:
|
||||
parts.append(f"{yr}-{month:02d}-{day:02d}")
|
||||
else:
|
||||
parts.append(f"{yr}-{month:02d}-00")
|
||||
else:
|
||||
parts.append(f"{yr}-00-00")
|
||||
else:
|
||||
parts.append(f"unknown-{index}")
|
||||
|
||||
# Recipient part
|
||||
if recipient:
|
||||
# Slugify recipient name
|
||||
name = recipient.lower()
|
||||
name = re.sub(r"[^a-z0-9]+", "-", name)
|
||||
name = name.strip("-")[:40]
|
||||
parts.append(name)
|
||||
|
||||
slug = "-".join(parts)
|
||||
return slug
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "letters.html")
|
||||
output_dir = os.path.join(base_dir, "lib", "jsons", "letters")
|
||||
|
||||
if not os.path.exists(html_path):
|
||||
print(f"ERROR: {html_path} not found!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Reading {html_path}...")
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
|
||||
soup = BeautifulSoup(content, "html.parser")
|
||||
all_tags = soup.find_all("p")
|
||||
print(f" Parsed {len(all_tags)} <p> tags total")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 1: Find all letter boundaries
|
||||
# ---------------------------------------------------------------------------
|
||||
letter_starts = []
|
||||
for i, tag in enumerate(all_tags):
|
||||
cls = get_css_class(tag)
|
||||
text = tag.get_text().strip()
|
||||
if cls == "p7" and text.startswith("Letter to:"):
|
||||
letter_starts.append(i)
|
||||
|
||||
print(f"\n Found {len(letter_starts)} letters")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 2: Extract each letter
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Classes that are just empty spacers
|
||||
SKIP_CLASSES = {"p2", "p15", "p23", "p24", "p26"}
|
||||
|
||||
# p12 = centered (verse-like)
|
||||
CENTERED_CLASSES = {"p12", "p18", "p19", "p20", "p21", "p27"}
|
||||
|
||||
# p13, p34, p38 = signature/closing blocks (colored)
|
||||
SIGNATURE_CLASSES = {"p13", "p34", "p38"}
|
||||
|
||||
letters = []
|
||||
slug_counts = Counter()
|
||||
|
||||
for li, start_idx in enumerate(letter_starts):
|
||||
# Determine end boundary
|
||||
end_idx = (
|
||||
letter_starts[li + 1] if li + 1 < len(letter_starts) else len(all_tags)
|
||||
)
|
||||
|
||||
# Parse header
|
||||
header_tag = all_tags[start_idx]
|
||||
recipient, location, date_str = parse_header(header_tag)
|
||||
|
||||
# Parse date into components
|
||||
year, month, day = parse_date(date_str) if date_str else (None, None, None)
|
||||
|
||||
# Find date reference code (first p8 after header that matches YY-MM-DD)
|
||||
date_ref = None
|
||||
salutation = None
|
||||
paragraphs = []
|
||||
first_p8_seen = False
|
||||
|
||||
for i in range(start_idx + 1, end_idx):
|
||||
tag = all_tags[i]
|
||||
cls = get_css_class(tag)
|
||||
text = tag.get_text().strip()
|
||||
|
||||
if cls in SKIP_CLASSES:
|
||||
continue
|
||||
if not text:
|
||||
continue
|
||||
|
||||
# p8 = reference code or body text
|
||||
if cls == "p8":
|
||||
if not first_p8_seen:
|
||||
# First p8 after header — check if it's a date reference
|
||||
if re.match(r"\d{2}-\d{2}", text):
|
||||
date_ref = text.strip()
|
||||
first_p8_seen = True
|
||||
|
||||
# Try to extract year/month/day from ref if not already parsed
|
||||
if not year and date_ref:
|
||||
ref_parts = date_ref.split("-")
|
||||
if len(ref_parts) >= 3:
|
||||
try:
|
||||
yr = int(ref_parts[0])
|
||||
mo = int(ref_parts[1])
|
||||
dy = int(ref_parts[2])
|
||||
year = 1900 + yr if yr < 100 else yr
|
||||
month = mo if 1 <= mo <= 12 else None
|
||||
day = dy if 1 <= dy <= 31 else None
|
||||
except ValueError:
|
||||
pass
|
||||
continue
|
||||
else:
|
||||
first_p8_seen = True
|
||||
# Not a date ref — treat as body text
|
||||
html = inner_html(tag)
|
||||
if html:
|
||||
paragraphs.append(html)
|
||||
continue
|
||||
else:
|
||||
# Subsequent p8 tags — body/closing text
|
||||
html = inner_html(tag)
|
||||
if html:
|
||||
paragraphs.append(html)
|
||||
continue
|
||||
|
||||
# p9 = recipient address block
|
||||
if cls == "p9":
|
||||
html = inner_html(tag)
|
||||
if html:
|
||||
paragraphs.append(html)
|
||||
continue
|
||||
|
||||
# p10 = salutation
|
||||
if cls == "p10":
|
||||
salutation = decode_scagoudy(text)
|
||||
html = inner_html(tag)
|
||||
if html:
|
||||
paragraphs.append(html)
|
||||
continue
|
||||
|
||||
# Centered classes (verse blocks)
|
||||
if cls in CENTERED_CLASSES:
|
||||
html = inner_html(tag)
|
||||
if html:
|
||||
# Check if it has line breaks (verse-like)
|
||||
if "<br" in html:
|
||||
html = f'<div style="text-align:center">{html}</div>'
|
||||
paragraphs.append(html)
|
||||
continue
|
||||
|
||||
# Signature/closing blocks
|
||||
if cls in SIGNATURE_CLASSES:
|
||||
html = inner_html(tag)
|
||||
if html:
|
||||
paragraphs.append(html)
|
||||
continue
|
||||
|
||||
# All other body text classes (p11, p14, p16, p22, p28, p29, etc.)
|
||||
html = inner_html(tag)
|
||||
if html and html != "<br>":
|
||||
# Check for verse-like content
|
||||
stripped = html.strip()
|
||||
if (
|
||||
stripped.startswith("<i>")
|
||||
and "<br" in stripped
|
||||
and '<div style="text-align:center">' not in stripped
|
||||
):
|
||||
plain = re.sub(r"</?[bi]>", "", stripped)
|
||||
lines = [
|
||||
l.strip() for l in re.split(r"<br\s*/?>", plain) if l.strip()
|
||||
]
|
||||
if len(lines) >= 2:
|
||||
html = f'<div style="text-align:center">{stripped}</div>'
|
||||
|
||||
paragraphs.append(html)
|
||||
|
||||
# Clean recipient name — strip trailing punctuation
|
||||
if recipient:
|
||||
recipient = recipient.rstrip(":,.; ").strip()
|
||||
# Also clean " :" (space before colon)
|
||||
recipient = re.sub(r"\s+:$", "", recipient).strip()
|
||||
|
||||
# Split multi-recipient strings into individual names
|
||||
# Patterns: "Mukunda, Janaki" or "Mukunda and Janaki"
|
||||
recipient_display = recipient or "Unknown"
|
||||
if recipient:
|
||||
# Split by comma, semicolon, " and ", " & "
|
||||
parts = re.split(r"\s*[,;]\s*|\s+and\s+|\s+&\s+", recipient)
|
||||
recipients_list = []
|
||||
for p in parts:
|
||||
name = p.strip().rstrip(":,.; ").strip()
|
||||
# Clean " :" pattern
|
||||
name = re.sub(r"\s+:$", "", name).strip()
|
||||
if name:
|
||||
recipients_list.append(name)
|
||||
if not recipients_list:
|
||||
recipients_list = ["Unknown"]
|
||||
else:
|
||||
recipients_list = ["Unknown"]
|
||||
|
||||
# Build slug
|
||||
slug = make_slug(recipient_display, date_ref, year, month, day, li)
|
||||
|
||||
# Ensure unique slug
|
||||
if slug in slug_counts:
|
||||
slug_counts[slug] += 1
|
||||
slug = f"{slug}-{slug_counts[slug]}"
|
||||
else:
|
||||
slug_counts[slug] = 1
|
||||
|
||||
# Normalize location
|
||||
if location:
|
||||
# Clean up common variants
|
||||
location = location.rstrip(",.")
|
||||
# Normalize "Vrndavana" variants
|
||||
if location.lower() in ("vrndavana", "vrndävana"):
|
||||
location = "Vrindaban"
|
||||
|
||||
letter = {
|
||||
"slug": slug,
|
||||
"recipients": recipients_list,
|
||||
"recipientDisplay": recipient_display,
|
||||
"date": date_str,
|
||||
"dateRef": date_ref,
|
||||
"year": year,
|
||||
"month": month,
|
||||
"day": day,
|
||||
"location": location,
|
||||
"salutation": salutation,
|
||||
"paragraphs": paragraphs,
|
||||
}
|
||||
letters.append(letter)
|
||||
|
||||
print(f"\n Extracted {len(letters)} letters")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 3: Statistics
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Year distribution
|
||||
year_counts = Counter()
|
||||
for letter in letters:
|
||||
if letter["year"]:
|
||||
year_counts[letter["year"]] += 1
|
||||
print(f"\n Year distribution:")
|
||||
for yr in sorted(year_counts.keys()):
|
||||
if 1940 <= yr <= 1980:
|
||||
print(f" {yr}: {year_counts[yr]} letters")
|
||||
|
||||
# Location distribution (top 20)
|
||||
loc_counts = Counter()
|
||||
for letter in letters:
|
||||
if letter["location"]:
|
||||
loc_counts[letter["location"]] += 1
|
||||
print(f"\n Top 20 locations:")
|
||||
for loc, count in loc_counts.most_common(20):
|
||||
print(f" {loc}: {count}")
|
||||
|
||||
# Recipient distribution (by individual name, not group)
|
||||
recip_counts = Counter()
|
||||
for letter in letters:
|
||||
for r in letter["recipients"]:
|
||||
recip_counts[r] += 1
|
||||
print(f"\n Top 20 recipients (individual):")
|
||||
for recip, count in recip_counts.most_common(20):
|
||||
print(f" {recip}: {count}")
|
||||
|
||||
# Letters with missing data
|
||||
no_year = sum(1 for l in letters if not l["year"])
|
||||
no_location = sum(1 for l in letters if not l["location"])
|
||||
no_date = sum(1 for l in letters if not l["date"])
|
||||
no_recipient = sum(1 for l in letters if l["recipients"] == ["Unknown"])
|
||||
print(f"\n Missing data:")
|
||||
print(f" No year: {no_year}")
|
||||
print(f" No location: {no_location}")
|
||||
print(f" No date: {no_date}")
|
||||
print(f" No recipient (Unknown): {no_recipient}")
|
||||
|
||||
# Unique counts
|
||||
all_individual_recipients = set()
|
||||
for l in letters:
|
||||
for r in l["recipients"]:
|
||||
all_individual_recipients.add(r)
|
||||
unique_recipients = len(all_individual_recipients)
|
||||
unique_locations = len(set(l["location"] for l in letters if l["location"]))
|
||||
unique_years = len(set(l["year"] for l in letters if l["year"]))
|
||||
print(f"\n Unique counts:")
|
||||
print(f" Recipients: {unique_recipients}")
|
||||
print(f" Locations: {unique_locations}")
|
||||
print(f" Years: {unique_years}")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 4: Write output
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Write individual letter files grouped by year for manageable sizes
|
||||
years_with_letters = sorted(
|
||||
set(l["year"] for l in letters if l["year"] and 1940 <= l["year"] <= 1980)
|
||||
)
|
||||
|
||||
# Write a summary/index file
|
||||
index_data = {
|
||||
"totalLetters": len(letters),
|
||||
"uniqueRecipients": unique_recipients,
|
||||
"uniqueLocations": unique_locations,
|
||||
"years": [],
|
||||
"topRecipients": [
|
||||
{"name": r, "count": c} for r, c in recip_counts.most_common(50)
|
||||
],
|
||||
"topLocations": [
|
||||
{"name": l, "count": c} for l, c in loc_counts.most_common(50)
|
||||
],
|
||||
}
|
||||
|
||||
for yr in years_with_letters:
|
||||
yr_letters = [l for l in letters if l["year"] == yr]
|
||||
index_data["years"].append(
|
||||
{
|
||||
"year": yr,
|
||||
"count": len(yr_letters),
|
||||
}
|
||||
)
|
||||
|
||||
# Also count letters without a valid year
|
||||
undated = [
|
||||
l for l in letters if not l["year"] or l["year"] < 1940 or l["year"] > 1980
|
||||
]
|
||||
if undated:
|
||||
index_data["years"].append(
|
||||
{
|
||||
"year": 0,
|
||||
"count": len(undated),
|
||||
}
|
||||
)
|
||||
|
||||
write_json(index_data, os.path.join(output_dir, "index.json"))
|
||||
print(f"\n Written: index.json")
|
||||
|
||||
# Write letters grouped by year (one JSON file per year)
|
||||
for yr in years_with_letters:
|
||||
yr_letters = [l for l in letters if l["year"] == yr]
|
||||
yr_letters.sort(key=lambda l: (l["month"] or 0, l["day"] or 0))
|
||||
write_json(yr_letters, os.path.join(output_dir, f"{yr}.json"))
|
||||
|
||||
# Write undated letters
|
||||
if undated:
|
||||
write_json(undated, os.path.join(output_dir, "undated.json"))
|
||||
|
||||
print(f" Written: {len(years_with_letters)} year files + undated.json")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 5: Quality checks
|
||||
# ---------------------------------------------------------------------------
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f" QUALITY CHECKS")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
# Check for ScaGoudy remnants
|
||||
sca_chars = {"å", "ë", "ö"}
|
||||
sca_remnants = 0
|
||||
for letter in letters[:100]: # Check first 100
|
||||
for p in letter["paragraphs"]:
|
||||
if any(c in p for c in sca_chars):
|
||||
sca_remnants += 1
|
||||
break
|
||||
if sca_remnants == 0:
|
||||
print(f" ✅ No ScaGoudy remnants detected (checked first 100)")
|
||||
else:
|
||||
print(f" ⚠ {sca_remnants} letters may have ScaGoudy remnants")
|
||||
|
||||
# Check empty letters
|
||||
empty = [l for l in letters if len(l["paragraphs"]) == 0]
|
||||
if empty:
|
||||
print(f" ⚠ {len(empty)} empty letters (no paragraphs)")
|
||||
else:
|
||||
print(f" ✅ No empty letters")
|
||||
|
||||
# Check duplicate slugs
|
||||
slug_list = [l["slug"] for l in letters]
|
||||
dupes = len(slug_list) - len(set(slug_list))
|
||||
if dupes:
|
||||
print(f" ⚠ {dupes} duplicate slugs")
|
||||
else:
|
||||
print(f" ✅ No duplicate slugs")
|
||||
|
||||
# Verse count
|
||||
verse_count = 0
|
||||
for letter in letters:
|
||||
for p in letter["paragraphs"]:
|
||||
if '<div style="text-align:center">' in p:
|
||||
verse_count += 1
|
||||
print(f" ℹ️ {verse_count} center-aligned verse paragraphs")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Summary
|
||||
# ---------------------------------------------------------------------------
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f" EXTRACTION SUMMARY")
|
||||
print(f"{'=' * 60}")
|
||||
print(f" Total letters: {len(letters)}")
|
||||
print(f" Unique recipients: {unique_recipients}")
|
||||
print(f" Unique locations: {unique_locations}")
|
||||
print(f" Year range: {min(years_with_letters)}–{max(years_with_letters)}")
|
||||
print(f" Total paragraphs: {sum(len(l['paragraphs']) for l in letters)}")
|
||||
print(f"{'=' * 60}")
|
||||
print(f"\nDone! ✨")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,158 @@
|
||||
/**
|
||||
* seed-letters.ts — Push letters into MongoDB.
|
||||
*
|
||||
* Collection written:
|
||||
* - letters → 6,866 individual letter documents
|
||||
*
|
||||
* JSON sources:
|
||||
* - lib/jsons/letters/1947.json through 1977.json
|
||||
* - lib/jsons/letters/undated.json
|
||||
*
|
||||
* Usage:
|
||||
* npx tsx scripts/seed-letters.ts
|
||||
*/
|
||||
|
||||
import dotenv from "dotenv";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { MongoClient } from "mongodb";
|
||||
|
||||
dotenv.config({ path: ".env.local" });
|
||||
dotenv.config({ path: ".env" });
|
||||
|
||||
const DATABASE_URL = process.env.DATABASE_URL;
|
||||
if (!DATABASE_URL) {
|
||||
console.error("❌ DATABASE_URL is not set");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const BASE_DIR = path.resolve(__dirname, "..");
|
||||
const LETTERS_DIR = path.join(BASE_DIR, "lib", "jsons", "letters");
|
||||
|
||||
function readJson<T>(filePath: string): T {
|
||||
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
|
||||
}
|
||||
|
||||
function sanitizeString(s: string): string {
|
||||
s = s.normalize("NFC");
|
||||
s = s.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, "");
|
||||
s = s.replace(/[\x7f-\x9f]/g, "");
|
||||
s = s.replace(/[\uD800-\uDFFF]/g, "");
|
||||
s = s.replace(/[\u200B-\u200F\u2028-\u202F\uFEFF]/g, "");
|
||||
s = s.replace(/[\uFDD0-\uFDEF\uFFFE\uFFFF]/g, "");
|
||||
s = Buffer.from(s, "utf-8").toString("utf-8");
|
||||
s = s.split("\0").join("");
|
||||
return s;
|
||||
}
|
||||
|
||||
function sanitizeDoc(obj: unknown): unknown {
|
||||
if (typeof obj === "string") return sanitizeString(obj);
|
||||
if (Array.isArray(obj)) return obj.map(sanitizeDoc);
|
||||
if (obj !== null && typeof obj === "object") {
|
||||
const result: Record<string, unknown> = {};
|
||||
for (const [key, value] of Object.entries(obj as Record<string, unknown>))
|
||||
result[key] = sanitizeDoc(value);
|
||||
return result;
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
type LetterJson = {
|
||||
slug: string;
|
||||
recipients: string[];
|
||||
recipientDisplay: string;
|
||||
date: string | null;
|
||||
dateRef: string | null;
|
||||
year: number | null;
|
||||
month: number | null;
|
||||
day: number | null;
|
||||
location: string | null;
|
||||
salutation: string | null;
|
||||
paragraphs: string[];
|
||||
};
|
||||
|
||||
async function main() {
|
||||
console.log("🌱 Seeding Letters...\n");
|
||||
|
||||
const client = new MongoClient(DATABASE_URL!);
|
||||
await client.connect();
|
||||
console.log("📡 Connected to MongoDB\n");
|
||||
|
||||
const db = client.db();
|
||||
const col = db.collection("letters");
|
||||
const start = Date.now();
|
||||
|
||||
// Delete all existing letters
|
||||
const deleted = await col.deleteMany({});
|
||||
console.log(` 🗑️ Deleted ${deleted.deletedCount} existing letters`);
|
||||
|
||||
// Ensure indexes
|
||||
await col.createIndex({ slug: 1 }, { unique: true });
|
||||
await col.createIndex({ year: 1 });
|
||||
await col.createIndex({ recipient: 1 });
|
||||
await col.createIndex({ location: 1 });
|
||||
await col.createIndex({ year: 1, month: 1 });
|
||||
|
||||
// Read all year files
|
||||
const files = fs
|
||||
.readdirSync(LETTERS_DIR)
|
||||
.filter((f) => f.endsWith(".json") && f !== "index.json")
|
||||
.sort();
|
||||
|
||||
console.log(`\n 📂 Found ${files.length} letter files`);
|
||||
|
||||
let inserted = 0;
|
||||
let errors = 0;
|
||||
|
||||
for (const file of files) {
|
||||
const letters = readJson<LetterJson[]>(path.join(LETTERS_DIR, file));
|
||||
console.log(`\n 📄 ${file}: ${letters.length} letters`);
|
||||
|
||||
for (const letter of letters) {
|
||||
const doc = sanitizeDoc(letter) as Record<string, unknown>;
|
||||
|
||||
try {
|
||||
await col.insertOne(doc);
|
||||
inserted++;
|
||||
|
||||
if (inserted % 500 === 0) {
|
||||
console.log(` ... ${inserted} letters inserted`);
|
||||
}
|
||||
} catch (e: unknown) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
if (msg.includes("duplicate key")) {
|
||||
// Skip duplicates silently
|
||||
continue;
|
||||
}
|
||||
console.error(
|
||||
` ❌ Error inserting ${letter.slug}: ${msg.substring(0, 100)}`,
|
||||
);
|
||||
errors++;
|
||||
if (errors > 10) {
|
||||
console.error(" Too many errors, stopping.");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (errors > 10) break;
|
||||
}
|
||||
|
||||
// Verify
|
||||
const totalDocs = await col.countDocuments();
|
||||
|
||||
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
|
||||
console.log("\n" + "─".repeat(50));
|
||||
console.log(`📊 Summary:`);
|
||||
console.log(` Inserted: ${inserted} letters`);
|
||||
console.log(` Errors: ${errors}`);
|
||||
console.log(` Total in collection: ${totalDocs}`);
|
||||
console.log(`\n🎉 Done in ${elapsed}s`);
|
||||
|
||||
await client.close();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("❌ Seed failed:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,189 @@
|
||||
/**
|
||||
* seed-magazines.ts — Push Magazines into MongoDB.
|
||||
*
|
||||
* Collections written:
|
||||
* - magazine_meta → one doc per magazine
|
||||
* - magazine_issues → N issue docs per magazine
|
||||
*
|
||||
* Usage:
|
||||
* npx tsx scripts/seed-magazines.ts
|
||||
*/
|
||||
|
||||
import dotenv from "dotenv";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { MongoClient } from "mongodb";
|
||||
|
||||
dotenv.config({ path: ".env.local" });
|
||||
dotenv.config({ path: ".env" });
|
||||
|
||||
const DATABASE_URL = process.env.DATABASE_URL;
|
||||
if (!DATABASE_URL) {
|
||||
console.error("❌ DATABASE_URL is not set");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const BASE_DIR = path.resolve(__dirname, "..");
|
||||
const MAGAZINES_DIR = path.join(BASE_DIR, "lib", "jsons", "magazines");
|
||||
|
||||
function readJson<T>(filePath: string): T {
|
||||
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
|
||||
}
|
||||
|
||||
function sanitizeString(s: string): string {
|
||||
// Normalize to NFC
|
||||
s = s.normalize("NFC");
|
||||
// Remove null bytes and C0 control chars (keep \t \n \r)
|
||||
s = s.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, "");
|
||||
// Remove C1 control chars
|
||||
s = s.replace(/[\x7f-\x9f]/g, "");
|
||||
// Remove lone surrogates (U+D800–U+DFFF) that are invalid in BSON
|
||||
s = s.replace(/[\uD800-\uDFFF]/g, "");
|
||||
// Remove zero-width characters that can confuse BSON length calculations
|
||||
s = s.replace(/[\u200B-\u200F\u2028-\u202F\uFEFF]/g, "");
|
||||
// Remove non-characters (U+FDD0–U+FDEF, U+FFFE, U+FFFF)
|
||||
s = s.replace(/[\uFDD0-\uFDEF\uFFFE\uFFFF]/g, "");
|
||||
// Remove any remaining non-BMP characters that could cause issues
|
||||
// (keep common supplementary planes like emoji, but remove PUA etc.)
|
||||
s = s.replace(/[\uDB80-\uDBFF][\uDC00-\uDFFF]/g, ""); // Supplementary PUA
|
||||
// Round-trip through Buffer to guarantee valid UTF-8 byte sequence
|
||||
s = Buffer.from(s, "utf-8").toString("utf-8");
|
||||
// Final safety: ensure the string is null-terminated compatible
|
||||
// by removing any embedded nulls that survived
|
||||
s = s.split("\0").join("");
|
||||
return s;
|
||||
}
|
||||
|
||||
function sanitizeDoc(obj: unknown): unknown {
|
||||
if (typeof obj === "string") {
|
||||
return sanitizeString(obj);
|
||||
}
|
||||
if (Array.isArray(obj)) return obj.map(sanitizeDoc);
|
||||
if (obj !== null && typeof obj === "object") {
|
||||
const result: Record<string, unknown> = {};
|
||||
for (const [key, value] of Object.entries(obj as Record<string, unknown>))
|
||||
result[key] = sanitizeDoc(value);
|
||||
return result;
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
async function seedOneMagazine(
|
||||
db: ReturnType<MongoClient["db"]>,
|
||||
magDir: string,
|
||||
): Promise<{ bookCode: string; issues: number }> {
|
||||
const introPath = path.join(magDir, "intro.json");
|
||||
if (!fs.existsSync(introPath)) throw new Error(`No intro.json in ${magDir}`);
|
||||
const intro = sanitizeDoc(
|
||||
readJson<Record<string, unknown>>(introPath),
|
||||
) as Record<string, unknown>;
|
||||
const bookCode = intro.book as string;
|
||||
|
||||
const metaCol = db.collection("magazine_meta");
|
||||
await metaCol.replaceOne(
|
||||
{ book: bookCode },
|
||||
{ book: bookCode, data: intro },
|
||||
{ upsert: true },
|
||||
);
|
||||
console.log(` 📖 Upserted magazine_meta for "${intro.title}" (${bookCode})`);
|
||||
|
||||
const issuesCol = db.collection("magazine_issues");
|
||||
// Delete old issues for this magazine
|
||||
await issuesCol.deleteMany({ book: bookCode });
|
||||
|
||||
const issuesDir = path.join(magDir, "issues");
|
||||
let inserted = 0;
|
||||
if (fs.existsSync(issuesDir)) {
|
||||
const files = fs
|
||||
.readdirSync(issuesDir)
|
||||
.filter((f) => f.endsWith(".json"))
|
||||
.sort();
|
||||
for (const file of files) {
|
||||
const issue = sanitizeDoc(
|
||||
readJson<Record<string, unknown>>(path.join(issuesDir, file)),
|
||||
) as Record<string, unknown>;
|
||||
|
||||
// Deep-sanitize each paragraph string
|
||||
const rawParas = (issue.paragraphs as string[]) || [];
|
||||
const cleanParas = rawParas.map((p: string) => sanitizeString(String(p)));
|
||||
|
||||
// Store paragraphs as a single JSON string to avoid MongoDB BSON
|
||||
// wire protocol issues with large string arrays.
|
||||
// The data layer will JSON.parse() this back into an array on read.
|
||||
const paragraphsJson = JSON.stringify(cleanParas);
|
||||
|
||||
const doc = {
|
||||
book: bookCode,
|
||||
volume: issue.volume,
|
||||
part: issue.part,
|
||||
title: issue.title,
|
||||
slug: issue.slug,
|
||||
order: issue.order,
|
||||
paragraphsJson,
|
||||
paragraphs: [] as string[], // Keep empty array for schema compat
|
||||
};
|
||||
|
||||
await issuesCol.replaceOne({ book: bookCode, slug: issue.slug }, doc, {
|
||||
upsert: true,
|
||||
});
|
||||
inserted++;
|
||||
}
|
||||
console.log(` 📄 Issues: ${inserted}`);
|
||||
}
|
||||
|
||||
const total = await issuesCol.countDocuments({ book: bookCode });
|
||||
console.log(` ✅ ${bookCode}: ${total} issue entries`);
|
||||
return { bookCode, issues: inserted };
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log("🌱 Seeding Magazines...\n");
|
||||
const client = new MongoClient(DATABASE_URL!);
|
||||
await client.connect();
|
||||
console.log("📡 Connected to MongoDB\n");
|
||||
const db = client.db();
|
||||
const start = Date.now();
|
||||
|
||||
if (!fs.existsSync(MAGAZINES_DIR)) {
|
||||
console.error(`❌ ${MAGAZINES_DIR} not found`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const subdirs = fs
|
||||
.readdirSync(MAGAZINES_DIR)
|
||||
.filter((e) => {
|
||||
const p = path.join(MAGAZINES_DIR, e);
|
||||
return (
|
||||
fs.statSync(p).isDirectory() &&
|
||||
fs.existsSync(path.join(p, "intro.json"))
|
||||
);
|
||||
})
|
||||
.sort();
|
||||
|
||||
if (subdirs.length === 0) {
|
||||
console.error("❌ No magazine directories found");
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(`📂 Found ${subdirs.length} magazine(s)\n`);
|
||||
|
||||
let totalIssues = 0;
|
||||
const bookCodes: string[] = [];
|
||||
for (const subdir of subdirs) {
|
||||
console.log(`\n📝 Processing: ${subdir}`);
|
||||
const result = await seedOneMagazine(db, path.join(MAGAZINES_DIR, subdir));
|
||||
bookCodes.push(result.bookCode);
|
||||
totalIssues += result.issues;
|
||||
}
|
||||
|
||||
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
|
||||
console.log("\n" + "─".repeat(50));
|
||||
console.log(`📊 Summary: ${subdirs.length} magazines, ${totalIssues} issues`);
|
||||
console.log(` Book codes: ${bookCodes.join(", ")}`);
|
||||
console.log(`\n🎉 Done in ${elapsed}s`);
|
||||
await client.close();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("❌ Seed failed:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user