krispire/spsays-training
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: letters added

Browse Source
This commit is contained in:
gosangam
2026-05-04 00:08:31 +05:30
parent 35d1d14f37
commit 401766928a
20 changed files with 4272 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/letters_extract.py
+689
View File
@@ -0,0 +1,689 @@
#!/usr/bin/env python3
"""Extract letters from letters.html.
Each letter starts with a p7 tag containing "Letter to: <recipient>".
The header contains recipient, location, and date information.
Structure:
- p7 = Letter header (recipient, dash, location, date)
- p8 = Reference code (YY-MM-DD) and also postscript/closing lines
- p9 = Recipient address block
- p10 = Salutation ("Dear...")
- p11 = Body paragraphs
- p12 = Centered text (verses, centered quotes)
- p13 = Closing/signature block (color: #408080)
- p14 = Indented/numbered items
- p16-p22, p28-p37 = Various body text styles
Output:
lib/jsons/letters/letters.json (array of all letters with metadata)
Usage:
python3 scripts/letters_extract.py
"""
import json
import os
import re
import sys
from collections import Counter
from bs4 import BeautifulSoup, Tag
# ---------------------------------------------------------------------------
# ScaGoudy → IAST Unicode mapping
# ---------------------------------------------------------------------------
SCAGOUDY_TO_IAST = {
"ä": "ā",
"Ä": "Ā",
"å": "ṛ",
"Å": "Ṛ",
"ñ": "ṣ",
"Ñ": "Ṣ",
"ö": "ṭ",
"Ö": "Ṭ",
"ë": "ṇ",
"Ë": "Ṇ",
"ò": "ḍ",
"Ò": "Ḍ",
"ç": "ś",
"Ç": "Ś",
"ì": "ṅ",
"Ì": "Ṅ",
"ï": "ñ",
"Ï": "Ñ",
"ü": "ū",
"Ü": "Ū",
"é": "ī",
"É": "Ī",
"à": "ṁ",
"ù": "ḥ",
"è": "ṝ",
"ÿ": "ḷ",
}
def decode_scagoudy(text: str) -> str:
"""Replace ScaGoudy-encoded characters with proper IAST Unicode."""
for src, dst in SCAGOUDY_TO_IAST.items():
text = text.replace(src, dst)
return text
def get_css_class(tag: Tag) -> str:
classes = tag.get("class", [])
return classes[0] if classes else ""
def inner_html(tag: Tag) -> str:
"""Get inner HTML of a tag, decode ScaGoudy, clean up spans."""
html = "".join(str(child) for child in tag.children)
html = decode_scagoudy(html)
# <span class="s1">text</span> → keep inner text (reference style)
html = re.sub(
r'<span[^>]*class="s\d+"[^>]*>(.*?)</span>', r"\1", html, flags=re.DOTALL
)
# Remove any remaining spans but keep content
html = re.sub(r"<span[^>]*>(.*?)</span>", r"\1", html, flags=re.DOTALL)
# Remove tab characters
html = html.replace("\t", " ")
# Collapse multiple spaces
html = re.sub(r"[ ]+", " ", html)
html = html.strip()
# Remove leading <br> tags
html = re.sub(r"^(<br\s*/?>[\s]*)+", "", html)
# Remove trailing <br> tags
html = re.sub(r"(<br\s*/?>[\s]*)+$", "", html)
return html
def write_json(data, path: str):
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w", encoding="utf-8") as f:
json.dump(data, f, indent=2, ensure_ascii=False)
f.write("\n")
# ---------------------------------------------------------------------------
# Month name → number mapping
# ---------------------------------------------------------------------------
MONTH_MAP = {
"january": 1,
"february": 2,
"march": 3,
"april": 4,
"may": 5,
"june": 6,
"july": 7,
"august": 8,
"september": 9,
"october": 10,
"november": 11,
"december": 12,
"jan": 1,
"feb": 2,
"mar": 3,
"apr": 4,
"jun": 6,
"jul": 7,
"aug": 8,
"sep": 9,
"sept": 9,
"oct": 10,
"nov": 11,
"dec": 12,
}
def parse_date(date_str: str):
"""Parse a date string like '12 July, 1947' or 'May 1949'.
Returns (year, month, day) where month and day can be None.
"""
if not date_str:
return None, None, None
date_str = date_str.strip().rstrip(".")
# Try "DD Month, YYYY" or "DD Month YYYY"
m = re.match(r"(\d{1,2})\s+([A-Za-z]+),?\s*(\d{4})", date_str)
if m:
day = int(m.group(1))
month_name = m.group(2).lower()
year = int(m.group(3))
month = MONTH_MAP.get(month_name)
if year >= 1900 and year <= 2000:
return year, month, day
# Try "Month DD, YYYY" or "Month DD YYYY"
m = re.match(r"([A-Za-z]+)\s+(\d{1,2}),?\s*(\d{4})", date_str)
if m:
month_name = m.group(1).lower()
day = int(m.group(2))
year = int(m.group(3))
month = MONTH_MAP.get(month_name)
if year >= 1900 and year <= 2000:
return year, month, day
# Try "Month, YYYY" or "Month YYYY"
m = re.match(r"([A-Za-z]+),?\s*(\d{4})", date_str)
if m:
month_name = m.group(1).lower()
year = int(m.group(2))
month = MONTH_MAP.get(month_name)
if month and year >= 1900 and year <= 2000:
return year, month, None
# Try just "YYYY"
m = re.match(r"(\d{4})$", date_str.strip())
if m:
year = int(m.group(1))
if year >= 1900 and year <= 2000:
return year, None, None
return None, None, None
def parse_header(tag: Tag):
"""Parse a p7 letter header tag.
Expected format:
Letter to: <recipient>
—
<location>
<date>
Or sometimes:
Letter to: <recipient>
<date>
Returns (recipient, location, date_str).
"""
text = tag.get_text().strip()
lines = [l.strip() for l in text.split("\n") if l.strip()]
recipient = None
location = None
date_str = None
if not lines:
return recipient, location, date_str
# First line: "Letter to: <recipient>"
first = lines[0]
m = re.match(r"Letter to:\s*(.*)", first, re.IGNORECASE)
if m:
recipient = m.group(1).strip().rstrip(",.")
recipient = decode_scagoudy(recipient)
# Remove "—" lines
content_lines = [l for l in lines[1:] if l != "—" and l != "-"]
if len(content_lines) >= 2:
# Last line is usually the date
potential_date = content_lines[-1]
year, month, day = parse_date(potential_date)
if year:
date_str = potential_date
# Everything before the date is location
loc_lines = content_lines[:-1]
if loc_lines:
location = loc_lines[0].strip().rstrip(",.")
location = decode_scagoudy(location)
else:
# Maybe the second-to-last is location and last is something else
# Try to find a date anywhere
for cl in reversed(content_lines):
yr, mo, dy = parse_date(cl)
if yr:
date_str = cl
idx = content_lines.index(cl)
if idx > 0:
location = content_lines[0].strip().rstrip(",.")
location = decode_scagoudy(location)
break
elif len(content_lines) == 1:
# Could be just a date, or just a location
potential_date = content_lines[0]
year, month, day = parse_date(potential_date)
if year:
date_str = potential_date
else:
location = potential_date.strip().rstrip(",.")
location = decode_scagoudy(location)
return recipient, location, date_str
def make_slug(recipient: str, date_ref: str, year, month, day, index: int) -> str:
"""Create a unique slug for a letter."""
parts = []
# Date part
if date_ref and re.match(r"\d{2}-\d{2}", date_ref):
parts.append(date_ref)
elif year:
yr = str(year)[2:] # Last 2 digits
if month:
if day:
parts.append(f"{yr}-{month:02d}-{day:02d}")
else:
parts.append(f"{yr}-{month:02d}-00")
else:
parts.append(f"{yr}-00-00")
else:
parts.append(f"unknown-{index}")
# Recipient part
if recipient:
# Slugify recipient name
name = recipient.lower()
name = re.sub(r"[^a-z0-9]+", "-", name)
name = name.strip("-")[:40]
parts.append(name)
slug = "-".join(parts)
return slug
# ---------------------------------------------------------------------------
# Main extraction
# ---------------------------------------------------------------------------
def main():
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
html_path = os.path.join(base_dir, "public", "html", "letters.html")
output_dir = os.path.join(base_dir, "lib", "jsons", "letters")
if not os.path.exists(html_path):
print(f"ERROR: {html_path} not found!")
sys.exit(1)
print(f"Reading {html_path}...")
with open(html_path, "r", encoding="utf-8") as f:
content = f.read()
soup = BeautifulSoup(content, "html.parser")
all_tags = soup.find_all("p")
print(f" Parsed {len(all_tags)} <p> tags total")
# ---------------------------------------------------------------------------
# Phase 1: Find all letter boundaries
# ---------------------------------------------------------------------------
letter_starts = []
for i, tag in enumerate(all_tags):
cls = get_css_class(tag)
text = tag.get_text().strip()
if cls == "p7" and text.startswith("Letter to:"):
letter_starts.append(i)
print(f"\n Found {len(letter_starts)} letters")
# ---------------------------------------------------------------------------
# Phase 2: Extract each letter
# ---------------------------------------------------------------------------
# Classes that are just empty spacers
SKIP_CLASSES = {"p2", "p15", "p23", "p24", "p26"}
# p12 = centered (verse-like)
CENTERED_CLASSES = {"p12", "p18", "p19", "p20", "p21", "p27"}
# p13, p34, p38 = signature/closing blocks (colored)
SIGNATURE_CLASSES = {"p13", "p34", "p38"}
letters = []
slug_counts = Counter()
for li, start_idx in enumerate(letter_starts):
# Determine end boundary
end_idx = (
letter_starts[li + 1] if li + 1 < len(letter_starts) else len(all_tags)
)
# Parse header
header_tag = all_tags[start_idx]
recipient, location, date_str = parse_header(header_tag)
# Parse date into components
year, month, day = parse_date(date_str) if date_str else (None, None, None)
# Find date reference code (first p8 after header that matches YY-MM-DD)
date_ref = None
salutation = None
paragraphs = []
first_p8_seen = False
for i in range(start_idx + 1, end_idx):
tag = all_tags[i]
cls = get_css_class(tag)
text = tag.get_text().strip()
if cls in SKIP_CLASSES:
continue
if not text:
continue
# p8 = reference code or body text
if cls == "p8":
if not first_p8_seen:
# First p8 after header — check if it's a date reference
if re.match(r"\d{2}-\d{2}", text):
date_ref = text.strip()
first_p8_seen = True
# Try to extract year/month/day from ref if not already parsed
if not year and date_ref:
ref_parts = date_ref.split("-")
if len(ref_parts) >= 3:
try:
yr = int(ref_parts[0])
mo = int(ref_parts[1])
dy = int(ref_parts[2])
year = 1900 + yr if yr < 100 else yr
month = mo if 1 <= mo <= 12 else None
day = dy if 1 <= dy <= 31 else None
except ValueError:
pass
continue
else:
first_p8_seen = True
# Not a date ref — treat as body text
html = inner_html(tag)
if html:
paragraphs.append(html)
continue
else:
# Subsequent p8 tags — body/closing text
html = inner_html(tag)
if html:
paragraphs.append(html)
continue
# p9 = recipient address block
if cls == "p9":
html = inner_html(tag)
if html:
paragraphs.append(html)
continue
# p10 = salutation
if cls == "p10":
salutation = decode_scagoudy(text)
html = inner_html(tag)
if html:
paragraphs.append(html)
continue
# Centered classes (verse blocks)
if cls in CENTERED_CLASSES:
html = inner_html(tag)
if html:
# Check if it has line breaks (verse-like)
if "<br" in html:
html = f'<div style="text-align:center">{html}</div>'
paragraphs.append(html)
continue
# Signature/closing blocks
if cls in SIGNATURE_CLASSES:
html = inner_html(tag)
if html:
paragraphs.append(html)
continue
# All other body text classes (p11, p14, p16, p22, p28, p29, etc.)
html = inner_html(tag)
if html and html != "<br>":
# Check for verse-like content
stripped = html.strip()
if (
stripped.startswith("<i>")
and "<br" in stripped
and '<div style="text-align:center">' not in stripped
):
plain = re.sub(r"</?[bi]>", "", stripped)
lines = [
l.strip() for l in re.split(r"<br\s*/?>", plain) if l.strip()
]
if len(lines) >= 2:
html = f'<div style="text-align:center">{stripped}</div>'
paragraphs.append(html)
# Clean recipient name — strip trailing punctuation
if recipient:
recipient = recipient.rstrip(":,.; ").strip()
# Also clean " :" (space before colon)
recipient = re.sub(r"\s+:$", "", recipient).strip()
# Split multi-recipient strings into individual names
# Patterns: "Mukunda, Janaki" or "Mukunda and Janaki"
recipient_display = recipient or "Unknown"
if recipient:
# Split by comma, semicolon, " and ", " & "
parts = re.split(r"\s*[,;]\s*|\s+and\s+|\s+&\s+", recipient)
recipients_list = []
for p in parts:
name = p.strip().rstrip(":,.; ").strip()
# Clean " :" pattern
name = re.sub(r"\s+:$", "", name).strip()
if name:
recipients_list.append(name)
if not recipients_list:
recipients_list = ["Unknown"]
else:
recipients_list = ["Unknown"]
# Build slug
slug = make_slug(recipient_display, date_ref, year, month, day, li)
# Ensure unique slug
if slug in slug_counts:
slug_counts[slug] += 1
slug = f"{slug}-{slug_counts[slug]}"
else:
slug_counts[slug] = 1
# Normalize location
if location:
# Clean up common variants
location = location.rstrip(",.")
# Normalize "Vrndavana" variants
if location.lower() in ("vrndavana", "vrndävana"):
location = "Vrindaban"
letter = {
"slug": slug,
"recipients": recipients_list,
"recipientDisplay": recipient_display,
"date": date_str,
"dateRef": date_ref,
"year": year,
"month": month,
"day": day,
"location": location,
"salutation": salutation,
"paragraphs": paragraphs,
}
letters.append(letter)
print(f"\n Extracted {len(letters)} letters")
# ---------------------------------------------------------------------------
# Phase 3: Statistics
# ---------------------------------------------------------------------------
# Year distribution
year_counts = Counter()
for letter in letters:
if letter["year"]:
year_counts[letter["year"]] += 1
print(f"\n Year distribution:")
for yr in sorted(year_counts.keys()):
if 1940 <= yr <= 1980:
print(f" {yr}: {year_counts[yr]} letters")
# Location distribution (top 20)
loc_counts = Counter()
for letter in letters:
if letter["location"]:
loc_counts[letter["location"]] += 1
print(f"\n Top 20 locations:")
for loc, count in loc_counts.most_common(20):
print(f" {loc}: {count}")
# Recipient distribution (by individual name, not group)
recip_counts = Counter()
for letter in letters:
for r in letter["recipients"]:
recip_counts[r] += 1
print(f"\n Top 20 recipients (individual):")
for recip, count in recip_counts.most_common(20):
print(f" {recip}: {count}")
# Letters with missing data
no_year = sum(1 for l in letters if not l["year"])
no_location = sum(1 for l in letters if not l["location"])
no_date = sum(1 for l in letters if not l["date"])
no_recipient = sum(1 for l in letters if l["recipients"] == ["Unknown"])
print(f"\n Missing data:")
print(f" No year: {no_year}")
print(f" No location: {no_location}")
print(f" No date: {no_date}")
print(f" No recipient (Unknown): {no_recipient}")
# Unique counts
all_individual_recipients = set()
for l in letters:
for r in l["recipients"]:
all_individual_recipients.add(r)
unique_recipients = len(all_individual_recipients)
unique_locations = len(set(l["location"] for l in letters if l["location"]))
unique_years = len(set(l["year"] for l in letters if l["year"]))
print(f"\n Unique counts:")
print(f" Recipients: {unique_recipients}")
print(f" Locations: {unique_locations}")
print(f" Years: {unique_years}")
# ---------------------------------------------------------------------------
# Phase 4: Write output
# ---------------------------------------------------------------------------
# Write individual letter files grouped by year for manageable sizes
years_with_letters = sorted(
set(l["year"] for l in letters if l["year"] and 1940 <= l["year"] <= 1980)
)
# Write a summary/index file
index_data = {
"totalLetters": len(letters),
"uniqueRecipients": unique_recipients,
"uniqueLocations": unique_locations,
"years": [],
"topRecipients": [
{"name": r, "count": c} for r, c in recip_counts.most_common(50)
],
"topLocations": [
{"name": l, "count": c} for l, c in loc_counts.most_common(50)
],
}
for yr in years_with_letters:
yr_letters = [l for l in letters if l["year"] == yr]
index_data["years"].append(
{
"year": yr,
"count": len(yr_letters),
}
)
# Also count letters without a valid year
undated = [
l for l in letters if not l["year"] or l["year"] < 1940 or l["year"] > 1980
]
if undated:
index_data["years"].append(
{
"year": 0,
"count": len(undated),
}
)
write_json(index_data, os.path.join(output_dir, "index.json"))
print(f"\n Written: index.json")
# Write letters grouped by year (one JSON file per year)
for yr in years_with_letters:
yr_letters = [l for l in letters if l["year"] == yr]
yr_letters.sort(key=lambda l: (l["month"] or 0, l["day"] or 0))
write_json(yr_letters, os.path.join(output_dir, f"{yr}.json"))
# Write undated letters
if undated:
write_json(undated, os.path.join(output_dir, "undated.json"))
print(f" Written: {len(years_with_letters)} year files + undated.json")
# ---------------------------------------------------------------------------
# Phase 5: Quality checks
# ---------------------------------------------------------------------------
print(f"\n{'=' * 60}")
print(f" QUALITY CHECKS")
print(f"{'=' * 60}")
# Check for ScaGoudy remnants
sca_chars = {"å", "ë", "ö"}
sca_remnants = 0
for letter in letters[:100]: # Check first 100
for p in letter["paragraphs"]:
if any(c in p for c in sca_chars):
sca_remnants += 1
break
if sca_remnants == 0:
print(f" ✅ No ScaGoudy remnants detected (checked first 100)")
else:
print(f" ⚠ {sca_remnants} letters may have ScaGoudy remnants")
# Check empty letters
empty = [l for l in letters if len(l["paragraphs"]) == 0]
if empty:
print(f" ⚠ {len(empty)} empty letters (no paragraphs)")
else:
print(f" ✅ No empty letters")
# Check duplicate slugs
slug_list = [l["slug"] for l in letters]
dupes = len(slug_list) - len(set(slug_list))
if dupes:
print(f" ⚠ {dupes} duplicate slugs")
else:
print(f" ✅ No duplicate slugs")
# Verse count
verse_count = 0
for letter in letters:
for p in letter["paragraphs"]:
if '<div style="text-align:center">' in p:
verse_count += 1
print(f" ℹ️ {verse_count} center-aligned verse paragraphs")
# ---------------------------------------------------------------------------
# Summary
# ---------------------------------------------------------------------------
print(f"\n{'=' * 60}")
print(f" EXTRACTION SUMMARY")
print(f"{'=' * 60}")
print(f" Total letters: {len(letters)}")
print(f" Unique recipients: {unique_recipients}")
print(f" Unique locations: {unique_locations}")
print(f" Year range: {min(years_with_letters)}–{max(years_with_letters)}")
print(f" Total paragraphs: {sum(len(l['paragraphs']) for l in letters)}")
print(f"{'=' * 60}")
print(f"\nDone! ✨")
if __name__ == "__main__":
main()