690 lines
22 KiB
Python
690 lines
22 KiB
Python
#!/usr/bin/env python3
|
||
"""Extract letters from letters.html.
|
||
|
||
Each letter starts with a p7 tag containing "Letter to: <recipient>".
|
||
The header contains recipient, location, and date information.
|
||
|
||
Structure:
|
||
- p7 = Letter header (recipient, dash, location, date)
|
||
- p8 = Reference code (YY-MM-DD) and also postscript/closing lines
|
||
- p9 = Recipient address block
|
||
- p10 = Salutation ("Dear...")
|
||
- p11 = Body paragraphs
|
||
- p12 = Centered text (verses, centered quotes)
|
||
- p13 = Closing/signature block (color: #408080)
|
||
- p14 = Indented/numbered items
|
||
- p16-p22, p28-p37 = Various body text styles
|
||
|
||
Output:
|
||
lib/jsons/letters/letters.json (array of all letters with metadata)
|
||
|
||
Usage:
|
||
python3 scripts/letters_extract.py
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import re
|
||
import sys
|
||
from collections import Counter
|
||
|
||
from bs4 import BeautifulSoup, Tag
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# ScaGoudy → IAST Unicode mapping
|
||
# ---------------------------------------------------------------------------
|
||
SCAGOUDY_TO_IAST = {
|
||
"ä": "ā",
|
||
"Ä": "Ā",
|
||
"å": "ṛ",
|
||
"Å": "Ṛ",
|
||
"ñ": "ṣ",
|
||
"Ñ": "Ṣ",
|
||
"ö": "ṭ",
|
||
"Ö": "Ṭ",
|
||
"ë": "ṇ",
|
||
"Ë": "Ṇ",
|
||
"ò": "ḍ",
|
||
"Ò": "Ḍ",
|
||
"ç": "ś",
|
||
"Ç": "Ś",
|
||
"ì": "ṅ",
|
||
"Ì": "Ṅ",
|
||
"ï": "ñ",
|
||
"Ï": "Ñ",
|
||
"ü": "ū",
|
||
"Ü": "Ū",
|
||
"é": "ī",
|
||
"É": "Ī",
|
||
"à": "ṁ",
|
||
"ù": "ḥ",
|
||
"è": "ṝ",
|
||
"ÿ": "ḷ",
|
||
}
|
||
|
||
|
||
def decode_scagoudy(text: str) -> str:
|
||
"""Replace ScaGoudy-encoded characters with proper IAST Unicode."""
|
||
for src, dst in SCAGOUDY_TO_IAST.items():
|
||
text = text.replace(src, dst)
|
||
return text
|
||
|
||
|
||
def get_css_class(tag: Tag) -> str:
|
||
classes = tag.get("class", [])
|
||
return classes[0] if classes else ""
|
||
|
||
|
||
def inner_html(tag: Tag) -> str:
|
||
"""Get inner HTML of a tag, decode ScaGoudy, clean up spans."""
|
||
html = "".join(str(child) for child in tag.children)
|
||
html = decode_scagoudy(html)
|
||
|
||
# <span class="s1">text</span> → keep inner text (reference style)
|
||
html = re.sub(
|
||
r'<span[^>]*class="s\d+"[^>]*>(.*?)</span>', r"\1", html, flags=re.DOTALL
|
||
)
|
||
# Remove any remaining spans but keep content
|
||
html = re.sub(r"<span[^>]*>(.*?)</span>", r"\1", html, flags=re.DOTALL)
|
||
# Remove tab characters
|
||
html = html.replace("\t", " ")
|
||
# Collapse multiple spaces
|
||
html = re.sub(r"[ ]+", " ", html)
|
||
html = html.strip()
|
||
# Remove leading <br> tags
|
||
html = re.sub(r"^(<br\s*/?>[\s]*)+", "", html)
|
||
# Remove trailing <br> tags
|
||
html = re.sub(r"(<br\s*/?>[\s]*)+$", "", html)
|
||
return html
|
||
|
||
|
||
def write_json(data, path: str):
|
||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||
with open(path, "w", encoding="utf-8") as f:
|
||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||
f.write("\n")
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Month name → number mapping
|
||
# ---------------------------------------------------------------------------
|
||
MONTH_MAP = {
|
||
"january": 1,
|
||
"february": 2,
|
||
"march": 3,
|
||
"april": 4,
|
||
"may": 5,
|
||
"june": 6,
|
||
"july": 7,
|
||
"august": 8,
|
||
"september": 9,
|
||
"october": 10,
|
||
"november": 11,
|
||
"december": 12,
|
||
"jan": 1,
|
||
"feb": 2,
|
||
"mar": 3,
|
||
"apr": 4,
|
||
"jun": 6,
|
||
"jul": 7,
|
||
"aug": 8,
|
||
"sep": 9,
|
||
"sept": 9,
|
||
"oct": 10,
|
||
"nov": 11,
|
||
"dec": 12,
|
||
}
|
||
|
||
|
||
def parse_date(date_str: str):
|
||
"""Parse a date string like '12 July, 1947' or 'May 1949'.
|
||
|
||
Returns (year, month, day) where month and day can be None.
|
||
"""
|
||
if not date_str:
|
||
return None, None, None
|
||
|
||
date_str = date_str.strip().rstrip(".")
|
||
|
||
# Try "DD Month, YYYY" or "DD Month YYYY"
|
||
m = re.match(r"(\d{1,2})\s+([A-Za-z]+),?\s*(\d{4})", date_str)
|
||
if m:
|
||
day = int(m.group(1))
|
||
month_name = m.group(2).lower()
|
||
year = int(m.group(3))
|
||
month = MONTH_MAP.get(month_name)
|
||
if year >= 1900 and year <= 2000:
|
||
return year, month, day
|
||
|
||
# Try "Month DD, YYYY" or "Month DD YYYY"
|
||
m = re.match(r"([A-Za-z]+)\s+(\d{1,2}),?\s*(\d{4})", date_str)
|
||
if m:
|
||
month_name = m.group(1).lower()
|
||
day = int(m.group(2))
|
||
year = int(m.group(3))
|
||
month = MONTH_MAP.get(month_name)
|
||
if year >= 1900 and year <= 2000:
|
||
return year, month, day
|
||
|
||
# Try "Month, YYYY" or "Month YYYY"
|
||
m = re.match(r"([A-Za-z]+),?\s*(\d{4})", date_str)
|
||
if m:
|
||
month_name = m.group(1).lower()
|
||
year = int(m.group(2))
|
||
month = MONTH_MAP.get(month_name)
|
||
if month and year >= 1900 and year <= 2000:
|
||
return year, month, None
|
||
|
||
# Try just "YYYY"
|
||
m = re.match(r"(\d{4})$", date_str.strip())
|
||
if m:
|
||
year = int(m.group(1))
|
||
if year >= 1900 and year <= 2000:
|
||
return year, None, None
|
||
|
||
return None, None, None
|
||
|
||
|
||
def parse_header(tag: Tag):
|
||
"""Parse a p7 letter header tag.
|
||
|
||
Expected format:
|
||
Letter to: <recipient>
|
||
—
|
||
<location>
|
||
<date>
|
||
|
||
Or sometimes:
|
||
Letter to: <recipient>
|
||
<date>
|
||
|
||
Returns (recipient, location, date_str).
|
||
"""
|
||
text = tag.get_text().strip()
|
||
lines = [l.strip() for l in text.split("\n") if l.strip()]
|
||
|
||
recipient = None
|
||
location = None
|
||
date_str = None
|
||
|
||
if not lines:
|
||
return recipient, location, date_str
|
||
|
||
# First line: "Letter to: <recipient>"
|
||
first = lines[0]
|
||
m = re.match(r"Letter to:\s*(.*)", first, re.IGNORECASE)
|
||
if m:
|
||
recipient = m.group(1).strip().rstrip(",.")
|
||
recipient = decode_scagoudy(recipient)
|
||
|
||
# Remove "—" lines
|
||
content_lines = [l for l in lines[1:] if l != "—" and l != "-"]
|
||
|
||
if len(content_lines) >= 2:
|
||
# Last line is usually the date
|
||
potential_date = content_lines[-1]
|
||
year, month, day = parse_date(potential_date)
|
||
if year:
|
||
date_str = potential_date
|
||
# Everything before the date is location
|
||
loc_lines = content_lines[:-1]
|
||
if loc_lines:
|
||
location = loc_lines[0].strip().rstrip(",.")
|
||
location = decode_scagoudy(location)
|
||
else:
|
||
# Maybe the second-to-last is location and last is something else
|
||
# Try to find a date anywhere
|
||
for cl in reversed(content_lines):
|
||
yr, mo, dy = parse_date(cl)
|
||
if yr:
|
||
date_str = cl
|
||
idx = content_lines.index(cl)
|
||
if idx > 0:
|
||
location = content_lines[0].strip().rstrip(",.")
|
||
location = decode_scagoudy(location)
|
||
break
|
||
elif len(content_lines) == 1:
|
||
# Could be just a date, or just a location
|
||
potential_date = content_lines[0]
|
||
year, month, day = parse_date(potential_date)
|
||
if year:
|
||
date_str = potential_date
|
||
else:
|
||
location = potential_date.strip().rstrip(",.")
|
||
location = decode_scagoudy(location)
|
||
|
||
return recipient, location, date_str
|
||
|
||
|
||
def make_slug(recipient: str, date_ref: str, year, month, day, index: int) -> str:
|
||
"""Create a unique slug for a letter."""
|
||
parts = []
|
||
|
||
# Date part
|
||
if date_ref and re.match(r"\d{2}-\d{2}", date_ref):
|
||
parts.append(date_ref)
|
||
elif year:
|
||
yr = str(year)[2:] # Last 2 digits
|
||
if month:
|
||
if day:
|
||
parts.append(f"{yr}-{month:02d}-{day:02d}")
|
||
else:
|
||
parts.append(f"{yr}-{month:02d}-00")
|
||
else:
|
||
parts.append(f"{yr}-00-00")
|
||
else:
|
||
parts.append(f"unknown-{index}")
|
||
|
||
# Recipient part
|
||
if recipient:
|
||
# Slugify recipient name
|
||
name = recipient.lower()
|
||
name = re.sub(r"[^a-z0-9]+", "-", name)
|
||
name = name.strip("-")[:40]
|
||
parts.append(name)
|
||
|
||
slug = "-".join(parts)
|
||
return slug
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Main extraction
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def main():
|
||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
html_path = os.path.join(base_dir, "public", "html", "letters.html")
|
||
output_dir = os.path.join(base_dir, "lib", "jsons", "letters")
|
||
|
||
if not os.path.exists(html_path):
|
||
print(f"ERROR: {html_path} not found!")
|
||
sys.exit(1)
|
||
|
||
print(f"Reading {html_path}...")
|
||
with open(html_path, "r", encoding="utf-8") as f:
|
||
content = f.read()
|
||
|
||
soup = BeautifulSoup(content, "html.parser")
|
||
all_tags = soup.find_all("p")
|
||
print(f" Parsed {len(all_tags)} <p> tags total")
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Phase 1: Find all letter boundaries
|
||
# ---------------------------------------------------------------------------
|
||
letter_starts = []
|
||
for i, tag in enumerate(all_tags):
|
||
cls = get_css_class(tag)
|
||
text = tag.get_text().strip()
|
||
if cls == "p7" and text.startswith("Letter to:"):
|
||
letter_starts.append(i)
|
||
|
||
print(f"\n Found {len(letter_starts)} letters")
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Phase 2: Extract each letter
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# Classes that are just empty spacers
|
||
SKIP_CLASSES = {"p2", "p15", "p23", "p24", "p26"}
|
||
|
||
# p12 = centered (verse-like)
|
||
CENTERED_CLASSES = {"p12", "p18", "p19", "p20", "p21", "p27"}
|
||
|
||
# p13, p34, p38 = signature/closing blocks (colored)
|
||
SIGNATURE_CLASSES = {"p13", "p34", "p38"}
|
||
|
||
letters = []
|
||
slug_counts = Counter()
|
||
|
||
for li, start_idx in enumerate(letter_starts):
|
||
# Determine end boundary
|
||
end_idx = (
|
||
letter_starts[li + 1] if li + 1 < len(letter_starts) else len(all_tags)
|
||
)
|
||
|
||
# Parse header
|
||
header_tag = all_tags[start_idx]
|
||
recipient, location, date_str = parse_header(header_tag)
|
||
|
||
# Parse date into components
|
||
year, month, day = parse_date(date_str) if date_str else (None, None, None)
|
||
|
||
# Find date reference code (first p8 after header that matches YY-MM-DD)
|
||
date_ref = None
|
||
salutation = None
|
||
paragraphs = []
|
||
first_p8_seen = False
|
||
|
||
for i in range(start_idx + 1, end_idx):
|
||
tag = all_tags[i]
|
||
cls = get_css_class(tag)
|
||
text = tag.get_text().strip()
|
||
|
||
if cls in SKIP_CLASSES:
|
||
continue
|
||
if not text:
|
||
continue
|
||
|
||
# p8 = reference code or body text
|
||
if cls == "p8":
|
||
if not first_p8_seen:
|
||
# First p8 after header — check if it's a date reference
|
||
if re.match(r"\d{2}-\d{2}", text):
|
||
date_ref = text.strip()
|
||
first_p8_seen = True
|
||
|
||
# Try to extract year/month/day from ref if not already parsed
|
||
if not year and date_ref:
|
||
ref_parts = date_ref.split("-")
|
||
if len(ref_parts) >= 3:
|
||
try:
|
||
yr = int(ref_parts[0])
|
||
mo = int(ref_parts[1])
|
||
dy = int(ref_parts[2])
|
||
year = 1900 + yr if yr < 100 else yr
|
||
month = mo if 1 <= mo <= 12 else None
|
||
day = dy if 1 <= dy <= 31 else None
|
||
except ValueError:
|
||
pass
|
||
continue
|
||
else:
|
||
first_p8_seen = True
|
||
# Not a date ref — treat as body text
|
||
html = inner_html(tag)
|
||
if html:
|
||
paragraphs.append(html)
|
||
continue
|
||
else:
|
||
# Subsequent p8 tags — body/closing text
|
||
html = inner_html(tag)
|
||
if html:
|
||
paragraphs.append(html)
|
||
continue
|
||
|
||
# p9 = recipient address block
|
||
if cls == "p9":
|
||
html = inner_html(tag)
|
||
if html:
|
||
paragraphs.append(html)
|
||
continue
|
||
|
||
# p10 = salutation
|
||
if cls == "p10":
|
||
salutation = decode_scagoudy(text)
|
||
html = inner_html(tag)
|
||
if html:
|
||
paragraphs.append(html)
|
||
continue
|
||
|
||
# Centered classes (verse blocks)
|
||
if cls in CENTERED_CLASSES:
|
||
html = inner_html(tag)
|
||
if html:
|
||
# Check if it has line breaks (verse-like)
|
||
if "<br" in html:
|
||
html = f'<div style="text-align:center">{html}</div>'
|
||
paragraphs.append(html)
|
||
continue
|
||
|
||
# Signature/closing blocks
|
||
if cls in SIGNATURE_CLASSES:
|
||
html = inner_html(tag)
|
||
if html:
|
||
paragraphs.append(html)
|
||
continue
|
||
|
||
# All other body text classes (p11, p14, p16, p22, p28, p29, etc.)
|
||
html = inner_html(tag)
|
||
if html and html != "<br>":
|
||
# Check for verse-like content
|
||
stripped = html.strip()
|
||
if (
|
||
stripped.startswith("<i>")
|
||
and "<br" in stripped
|
||
and '<div style="text-align:center">' not in stripped
|
||
):
|
||
plain = re.sub(r"</?[bi]>", "", stripped)
|
||
lines = [
|
||
l.strip() for l in re.split(r"<br\s*/?>", plain) if l.strip()
|
||
]
|
||
if len(lines) >= 2:
|
||
html = f'<div style="text-align:center">{stripped}</div>'
|
||
|
||
paragraphs.append(html)
|
||
|
||
# Clean recipient name — strip trailing punctuation
|
||
if recipient:
|
||
recipient = recipient.rstrip(":,.; ").strip()
|
||
# Also clean " :" (space before colon)
|
||
recipient = re.sub(r"\s+:$", "", recipient).strip()
|
||
|
||
# Split multi-recipient strings into individual names
|
||
# Patterns: "Mukunda, Janaki" or "Mukunda and Janaki"
|
||
recipient_display = recipient or "Unknown"
|
||
if recipient:
|
||
# Split by comma, semicolon, " and ", " & "
|
||
parts = re.split(r"\s*[,;]\s*|\s+and\s+|\s+&\s+", recipient)
|
||
recipients_list = []
|
||
for p in parts:
|
||
name = p.strip().rstrip(":,.; ").strip()
|
||
# Clean " :" pattern
|
||
name = re.sub(r"\s+:$", "", name).strip()
|
||
if name:
|
||
recipients_list.append(name)
|
||
if not recipients_list:
|
||
recipients_list = ["Unknown"]
|
||
else:
|
||
recipients_list = ["Unknown"]
|
||
|
||
# Build slug
|
||
slug = make_slug(recipient_display, date_ref, year, month, day, li)
|
||
|
||
# Ensure unique slug
|
||
if slug in slug_counts:
|
||
slug_counts[slug] += 1
|
||
slug = f"{slug}-{slug_counts[slug]}"
|
||
else:
|
||
slug_counts[slug] = 1
|
||
|
||
# Normalize location
|
||
if location:
|
||
# Clean up common variants
|
||
location = location.rstrip(",.")
|
||
# Normalize "Vrndavana" variants
|
||
if location.lower() in ("vrndavana", "vrndävana"):
|
||
location = "Vrindaban"
|
||
|
||
letter = {
|
||
"slug": slug,
|
||
"recipients": recipients_list,
|
||
"recipientDisplay": recipient_display,
|
||
"date": date_str,
|
||
"dateRef": date_ref,
|
||
"year": year,
|
||
"month": month,
|
||
"day": day,
|
||
"location": location,
|
||
"salutation": salutation,
|
||
"paragraphs": paragraphs,
|
||
}
|
||
letters.append(letter)
|
||
|
||
print(f"\n Extracted {len(letters)} letters")
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Phase 3: Statistics
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# Year distribution
|
||
year_counts = Counter()
|
||
for letter in letters:
|
||
if letter["year"]:
|
||
year_counts[letter["year"]] += 1
|
||
print(f"\n Year distribution:")
|
||
for yr in sorted(year_counts.keys()):
|
||
if 1940 <= yr <= 1980:
|
||
print(f" {yr}: {year_counts[yr]} letters")
|
||
|
||
# Location distribution (top 20)
|
||
loc_counts = Counter()
|
||
for letter in letters:
|
||
if letter["location"]:
|
||
loc_counts[letter["location"]] += 1
|
||
print(f"\n Top 20 locations:")
|
||
for loc, count in loc_counts.most_common(20):
|
||
print(f" {loc}: {count}")
|
||
|
||
# Recipient distribution (by individual name, not group)
|
||
recip_counts = Counter()
|
||
for letter in letters:
|
||
for r in letter["recipients"]:
|
||
recip_counts[r] += 1
|
||
print(f"\n Top 20 recipients (individual):")
|
||
for recip, count in recip_counts.most_common(20):
|
||
print(f" {recip}: {count}")
|
||
|
||
# Letters with missing data
|
||
no_year = sum(1 for l in letters if not l["year"])
|
||
no_location = sum(1 for l in letters if not l["location"])
|
||
no_date = sum(1 for l in letters if not l["date"])
|
||
no_recipient = sum(1 for l in letters if l["recipients"] == ["Unknown"])
|
||
print(f"\n Missing data:")
|
||
print(f" No year: {no_year}")
|
||
print(f" No location: {no_location}")
|
||
print(f" No date: {no_date}")
|
||
print(f" No recipient (Unknown): {no_recipient}")
|
||
|
||
# Unique counts
|
||
all_individual_recipients = set()
|
||
for l in letters:
|
||
for r in l["recipients"]:
|
||
all_individual_recipients.add(r)
|
||
unique_recipients = len(all_individual_recipients)
|
||
unique_locations = len(set(l["location"] for l in letters if l["location"]))
|
||
unique_years = len(set(l["year"] for l in letters if l["year"]))
|
||
print(f"\n Unique counts:")
|
||
print(f" Recipients: {unique_recipients}")
|
||
print(f" Locations: {unique_locations}")
|
||
print(f" Years: {unique_years}")
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Phase 4: Write output
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# Write individual letter files grouped by year for manageable sizes
|
||
years_with_letters = sorted(
|
||
set(l["year"] for l in letters if l["year"] and 1940 <= l["year"] <= 1980)
|
||
)
|
||
|
||
# Write a summary/index file
|
||
index_data = {
|
||
"totalLetters": len(letters),
|
||
"uniqueRecipients": unique_recipients,
|
||
"uniqueLocations": unique_locations,
|
||
"years": [],
|
||
"topRecipients": [
|
||
{"name": r, "count": c} for r, c in recip_counts.most_common(50)
|
||
],
|
||
"topLocations": [
|
||
{"name": l, "count": c} for l, c in loc_counts.most_common(50)
|
||
],
|
||
}
|
||
|
||
for yr in years_with_letters:
|
||
yr_letters = [l for l in letters if l["year"] == yr]
|
||
index_data["years"].append(
|
||
{
|
||
"year": yr,
|
||
"count": len(yr_letters),
|
||
}
|
||
)
|
||
|
||
# Also count letters without a valid year
|
||
undated = [
|
||
l for l in letters if not l["year"] or l["year"] < 1940 or l["year"] > 1980
|
||
]
|
||
if undated:
|
||
index_data["years"].append(
|
||
{
|
||
"year": 0,
|
||
"count": len(undated),
|
||
}
|
||
)
|
||
|
||
write_json(index_data, os.path.join(output_dir, "index.json"))
|
||
print(f"\n Written: index.json")
|
||
|
||
# Write letters grouped by year (one JSON file per year)
|
||
for yr in years_with_letters:
|
||
yr_letters = [l for l in letters if l["year"] == yr]
|
||
yr_letters.sort(key=lambda l: (l["month"] or 0, l["day"] or 0))
|
||
write_json(yr_letters, os.path.join(output_dir, f"{yr}.json"))
|
||
|
||
# Write undated letters
|
||
if undated:
|
||
write_json(undated, os.path.join(output_dir, "undated.json"))
|
||
|
||
print(f" Written: {len(years_with_letters)} year files + undated.json")
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Phase 5: Quality checks
|
||
# ---------------------------------------------------------------------------
|
||
print(f"\n{'=' * 60}")
|
||
print(f" QUALITY CHECKS")
|
||
print(f"{'=' * 60}")
|
||
|
||
# Check for ScaGoudy remnants
|
||
sca_chars = {"å", "ë", "ö"}
|
||
sca_remnants = 0
|
||
for letter in letters[:100]: # Check first 100
|
||
for p in letter["paragraphs"]:
|
||
if any(c in p for c in sca_chars):
|
||
sca_remnants += 1
|
||
break
|
||
if sca_remnants == 0:
|
||
print(f" ✅ No ScaGoudy remnants detected (checked first 100)")
|
||
else:
|
||
print(f" ⚠ {sca_remnants} letters may have ScaGoudy remnants")
|
||
|
||
# Check empty letters
|
||
empty = [l for l in letters if len(l["paragraphs"]) == 0]
|
||
if empty:
|
||
print(f" ⚠ {len(empty)} empty letters (no paragraphs)")
|
||
else:
|
||
print(f" ✅ No empty letters")
|
||
|
||
# Check duplicate slugs
|
||
slug_list = [l["slug"] for l in letters]
|
||
dupes = len(slug_list) - len(set(slug_list))
|
||
if dupes:
|
||
print(f" ⚠ {dupes} duplicate slugs")
|
||
else:
|
||
print(f" ✅ No duplicate slugs")
|
||
|
||
# Verse count
|
||
verse_count = 0
|
||
for letter in letters:
|
||
for p in letter["paragraphs"]:
|
||
if '<div style="text-align:center">' in p:
|
||
verse_count += 1
|
||
print(f" ℹ️ {verse_count} center-aligned verse paragraphs")
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Summary
|
||
# ---------------------------------------------------------------------------
|
||
print(f"\n{'=' * 60}")
|
||
print(f" EXTRACTION SUMMARY")
|
||
print(f"{'=' * 60}")
|
||
print(f" Total letters: {len(letters)}")
|
||
print(f" Unique recipients: {unique_recipients}")
|
||
print(f" Unique locations: {unique_locations}")
|
||
print(f" Year range: {min(years_with_letters)}–{max(years_with_letters)}")
|
||
print(f" Total paragraphs: {sum(len(l['paragraphs']) for l in letters)}")
|
||
print(f"{'=' * 60}")
|
||
print(f"\nDone! ✨")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|