feat: letters added
This commit is contained in:
gosangam
@@ -0,0 +1,546 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract Back to Godhead magazine from btg-1960.html.
|
||||
|
||||
Each magazine issue becomes its own JSON file.
|
||||
Issues are grouped by volume in the intro.json metadata.
|
||||
|
||||
Structure:
|
||||
- p2 = Volume/Part bookmark (e.g., "BTGPY1: Volume 1, Part 1, 1944")
|
||||
- p3 = Volume/Part display title
|
||||
- p8 = Article bookmark within an issue (e.g., "BTGPY1a: Message of His Divine Grace")
|
||||
- p9 = Article display title (bold)
|
||||
- p12, p35 = Body paragraphs
|
||||
- p14, p17, p88 = Continuation notes / misc text
|
||||
- p15 = Verse blocks (italic, with ScaGoudy encoding)
|
||||
- p33, p40, p47, p54 = Magazine mastheads/headers
|
||||
|
||||
Output:
|
||||
lib/jsons/magazines/btg/intro.json
|
||||
lib/jsons/magazines/btg/issues/v1-p1.json
|
||||
lib/jsons/magazines/btg/issues/v1-p2.json
|
||||
...
|
||||
lib/jsons/magazines/btg/issues/v4-p1.json
|
||||
|
||||
Usage:
|
||||
python3 scripts/btg_extract.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ScaGoudy → IAST Unicode mapping
|
||||
# ---------------------------------------------------------------------------
|
||||
SCAGOUDY_TO_IAST = {
|
||||
"ä": "ā",
|
||||
"Ä": "Ā",
|
||||
"å": "ṛ",
|
||||
"Å": "Ṛ",
|
||||
"ñ": "ṣ",
|
||||
"Ñ": "Ṣ",
|
||||
"ö": "ṭ",
|
||||
"Ö": "Ṭ",
|
||||
"ë": "ṇ",
|
||||
"Ë": "Ṇ",
|
||||
"ò": "ḍ",
|
||||
"Ò": "Ḍ",
|
||||
"ç": "ś",
|
||||
"Ç": "Ś",
|
||||
"ì": "ṅ",
|
||||
"Ì": "Ṅ",
|
||||
"ï": "ñ",
|
||||
"Ï": "Ñ",
|
||||
"ü": "ū",
|
||||
"Ü": "Ū",
|
||||
"é": "ī",
|
||||
"É": "Ī",
|
||||
"à": "ṁ",
|
||||
"ù": "ḥ",
|
||||
"è": "ṝ",
|
||||
"ÿ": "ḷ",
|
||||
}
|
||||
|
||||
|
||||
def decode_scagoudy(text: str) -> str:
|
||||
"""Replace ScaGoudy-encoded characters with proper IAST Unicode."""
|
||||
for src, dst in SCAGOUDY_TO_IAST.items():
|
||||
text = text.replace(src, dst)
|
||||
return text
|
||||
|
||||
|
||||
def get_css_class(tag: Tag) -> str:
|
||||
classes = tag.get("class", [])
|
||||
return classes[0] if classes else ""
|
||||
|
||||
|
||||
def inner_html(tag: Tag) -> str:
|
||||
"""Get inner HTML of a tag, decode ScaGoudy, clean up spans."""
|
||||
html = "".join(str(child) for child in tag.children)
|
||||
html = decode_scagoudy(html)
|
||||
# <span class="s1">text</span> → keep inner text (s1 is a ref style)
|
||||
html = re.sub(
|
||||
r'<span[^>]*class="s1"[^>]*>(.*?)</span>', r"\1", html, flags=re.DOTALL
|
||||
)
|
||||
# <span class="s3">text</span> → keep inner text (superscript digit)
|
||||
html = re.sub(
|
||||
r'<span[^>]*class="s3"[^>]*>(.*?)</span>', r"\1", html, flags=re.DOTALL
|
||||
)
|
||||
# <span class="s4">text</span> → keep inner text
|
||||
html = re.sub(
|
||||
r'<span[^>]*class="s4"[^>]*>(.*?)</span>', r"\1", html, flags=re.DOTALL
|
||||
)
|
||||
# <span class="s5">text</span> → keep inner text
|
||||
html = re.sub(
|
||||
r'<span[^>]*class="s5"[^>]*>(.*?)</span>', r"\1", html, flags=re.DOTALL
|
||||
)
|
||||
# <span class="s7">text</span> → keep inner text
|
||||
html = re.sub(
|
||||
r'<span[^>]*class="s7"[^>]*>(.*?)</span>', r"\1", html, flags=re.DOTALL
|
||||
)
|
||||
# <span class="s8">text</span> → keep inner text
|
||||
html = re.sub(
|
||||
r'<span[^>]*class="s8"[^>]*>(.*?)</span>', r"\1", html, flags=re.DOTALL
|
||||
)
|
||||
# Remove any remaining spans but keep content
|
||||
html = re.sub(r"<span[^>]*>(.*?)</span>", r"\1", html, flags=re.DOTALL)
|
||||
# Remove Apple-tab-span
|
||||
html = html.replace("\t", " ")
|
||||
# Collapse multiple spaces (but not around <br>)
|
||||
html = re.sub(r"[ ]+", " ", html)
|
||||
html = html.strip()
|
||||
# Remove leading <br> tags
|
||||
html = re.sub(r"^(<br\s*/?>[\s]*)+", "", html)
|
||||
# Remove trailing <br> tags
|
||||
html = re.sub(r"(<br\s*/?>[\s]*)+$", "", html)
|
||||
return html
|
||||
|
||||
|
||||
def is_verse_paragraph(tag: Tag) -> bool:
|
||||
"""Check if a paragraph is a verse block (italic with line breaks)."""
|
||||
cls = get_css_class(tag)
|
||||
# p15 is commonly used for verse blocks
|
||||
if cls == "p15":
|
||||
return True
|
||||
# Also check for paragraphs that are entirely italic with <br> breaks
|
||||
# (some verses use p14 or other classes but are entirely in <i> tags)
|
||||
return False
|
||||
|
||||
|
||||
def format_verse(tag: Tag) -> str:
|
||||
"""Format a verse paragraph as a center-aligned div."""
|
||||
html = inner_html(tag)
|
||||
if not html:
|
||||
return ""
|
||||
# Wrap in center-aligned div
|
||||
return f'<div style="text-align:center">{html}</div>'
|
||||
|
||||
|
||||
def write_json(data: dict, path: str):
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
f.write("\n")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Parse volume/part info from p2 bookmark text
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def parse_volume_part(text: str):
|
||||
"""Parse 'BTGPY1: Volume 1, Part 1, 1944' → (volume, part, date_str, slug).
|
||||
Returns (volume_num, part_num, date_string, slug) or None.
|
||||
"""
|
||||
# Remove the BTGPY prefix
|
||||
text = text.strip()
|
||||
# Match "Volume X, Part Y, ..."
|
||||
m = re.search(r"Volume\s+(\d+),?\s*Part\s+(\d+),?\s*(.*)", text, re.IGNORECASE)
|
||||
if m:
|
||||
vol = int(m.group(1))
|
||||
part = int(m.group(2))
|
||||
date_str = m.group(3).strip().rstrip(" ")
|
||||
slug = f"v{vol}-p{part}"
|
||||
return vol, part, date_str, slug
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "btg-1960.html")
|
||||
output_dir = os.path.join(base_dir, "lib", "jsons", "magazines", "btg")
|
||||
issues_dir = os.path.join(output_dir, "issues")
|
||||
|
||||
if not os.path.exists(html_path):
|
||||
print(f"ERROR: {html_path} not found!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Reading {html_path}...")
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
|
||||
soup = BeautifulSoup(content, "html.parser")
|
||||
all_tags = soup.find_all("p")
|
||||
print(f" Parsed {len(all_tags)} <p> tags total")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 1: Find all issue boundaries
|
||||
# ---------------------------------------------------------------------------
|
||||
# Each issue starts with a p2 bookmark tag
|
||||
issue_boundaries = [] # List of (tag_index, volume, part, date_str, slug)
|
||||
|
||||
for i, tag in enumerate(all_tags):
|
||||
cls = get_css_class(tag)
|
||||
if cls == "p2":
|
||||
text = tag.get_text().strip()
|
||||
parsed = parse_volume_part(text)
|
||||
if parsed:
|
||||
vol, part, date_str, slug = parsed
|
||||
issue_boundaries.append((i, vol, part, date_str, slug))
|
||||
|
||||
print(f"\n Found {len(issue_boundaries)} issues")
|
||||
for ib in issue_boundaries:
|
||||
print(f" tag[{ib[0]}]: Vol {ib[1]}, Part {ib[2]}, {ib[3]} → {ib[4]}")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 2: Extract articles and paragraphs for each issue
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Classes to skip entirely
|
||||
SKIP_CLASSES = {"p2", "p11", "p24", "p28"}
|
||||
|
||||
# Classes that are header/masthead material (include as paragraphs)
|
||||
MASTHEAD_CLASSES = {"p3", "p4", "p5", "p6", "p7", "p32", "p33", "p34", "p40", "p47"}
|
||||
|
||||
# Classes that are body text
|
||||
BODY_CLASSES = {
|
||||
"p10",
|
||||
"p12",
|
||||
"p13",
|
||||
"p14",
|
||||
"p15",
|
||||
"p16",
|
||||
"p17",
|
||||
"p35",
|
||||
"p36",
|
||||
"p37",
|
||||
"p38",
|
||||
"p39",
|
||||
"p41",
|
||||
"p42",
|
||||
"p43",
|
||||
"p44",
|
||||
"p45",
|
||||
"p46",
|
||||
"p48",
|
||||
"p49",
|
||||
"p50",
|
||||
"p51",
|
||||
"p52",
|
||||
"p53",
|
||||
"p54",
|
||||
"p55",
|
||||
"p56",
|
||||
"p57",
|
||||
"p58",
|
||||
"p59",
|
||||
"p60",
|
||||
"p61",
|
||||
"p62",
|
||||
"p63",
|
||||
"p64",
|
||||
"p65",
|
||||
"p66",
|
||||
"p67",
|
||||
"p68",
|
||||
"p69",
|
||||
"p70",
|
||||
"p71",
|
||||
"p72",
|
||||
"p73",
|
||||
"p74",
|
||||
"p75",
|
||||
"p76",
|
||||
"p77",
|
||||
"p78",
|
||||
"p79",
|
||||
"p80",
|
||||
"p81",
|
||||
"p82",
|
||||
"p83",
|
||||
"p84",
|
||||
"p85",
|
||||
"p86",
|
||||
"p87",
|
||||
"p88",
|
||||
"p89",
|
||||
"p90",
|
||||
"p91",
|
||||
"p92",
|
||||
"p93",
|
||||
"p94",
|
||||
"p95",
|
||||
"p96",
|
||||
"p97",
|
||||
"p98",
|
||||
"p99",
|
||||
}
|
||||
|
||||
issues_data = []
|
||||
|
||||
for idx, (start_tag_idx, vol, part, date_str, slug) in enumerate(issue_boundaries):
|
||||
# Determine end boundary
|
||||
if idx + 1 < len(issue_boundaries):
|
||||
end_tag_idx = issue_boundaries[idx + 1][0]
|
||||
else:
|
||||
end_tag_idx = len(all_tags)
|
||||
|
||||
issue_tags = all_tags[start_tag_idx:end_tag_idx]
|
||||
|
||||
paragraphs = []
|
||||
articles = [] # List of { title, slug }
|
||||
|
||||
for tag in issue_tags:
|
||||
cls = get_css_class(tag)
|
||||
text = tag.get_text().strip()
|
||||
|
||||
# Skip bookmark tags and empty lines
|
||||
if cls in SKIP_CLASSES:
|
||||
continue
|
||||
if not text and cls not in ("p15",):
|
||||
continue
|
||||
|
||||
# p8 = Article bookmark — extract title for metadata
|
||||
if cls == "p8":
|
||||
# Extract article title from bookmark text (remove BTGPYxx: prefix)
|
||||
article_text = text
|
||||
m = re.match(r"BTGPY\d+[a-z]?:\s*(.*)", article_text)
|
||||
if m:
|
||||
article_title = m.group(1).strip()
|
||||
else:
|
||||
article_title = article_text
|
||||
article_title = decode_scagoudy(article_title)
|
||||
articles.append({"title": article_title})
|
||||
continue
|
||||
|
||||
# p9 = Article display title — render as bold heading
|
||||
if cls == "p9":
|
||||
html = inner_html(tag)
|
||||
if html:
|
||||
# Make it a bold article heading
|
||||
if not html.startswith("<b>"):
|
||||
html = f"<b>{html}</b>"
|
||||
paragraphs.append(html)
|
||||
continue
|
||||
|
||||
# p15 = Verse block — center align
|
||||
if cls == "p15":
|
||||
html = format_verse(tag)
|
||||
if html:
|
||||
paragraphs.append(html)
|
||||
continue
|
||||
|
||||
# p3 = Volume title — skip (it's the display title of the issue header)
|
||||
if cls == "p3":
|
||||
continue
|
||||
|
||||
# Masthead classes — include but as regular paragraphs
|
||||
if cls in MASTHEAD_CLASSES:
|
||||
html = inner_html(tag)
|
||||
if html and html != "<br>":
|
||||
paragraphs.append(html)
|
||||
continue
|
||||
|
||||
# Body text classes
|
||||
html = inner_html(tag)
|
||||
if html and html != "<br>":
|
||||
# Check if this looks like a verse (entirely italic with <br> breaks)
|
||||
stripped = html.strip()
|
||||
if (
|
||||
stripped.startswith("<i>")
|
||||
and "<br" in stripped
|
||||
and not '<div style="text-align:center">' in stripped
|
||||
):
|
||||
# Check if it's predominantly a verse (italic block with line breaks)
|
||||
# by seeing if removing the <i> tags leaves mostly text with <br>
|
||||
plain = re.sub(r"</?[bi]>", "", stripped)
|
||||
lines = [
|
||||
l.strip() for l in re.split(r"<br\s*/?>", plain) if l.strip()
|
||||
]
|
||||
if len(lines) >= 2:
|
||||
html = f'<div style="text-align:center">{stripped}</div>'
|
||||
|
||||
paragraphs.append(html)
|
||||
|
||||
# Build display title
|
||||
if date_str:
|
||||
display_title = f"Volume {vol}, Part {part}, {date_str}"
|
||||
else:
|
||||
display_title = f"Volume {vol}, Part {part}"
|
||||
|
||||
issue_data = {
|
||||
"book": "btg",
|
||||
"volume": vol,
|
||||
"part": part,
|
||||
"title": display_title,
|
||||
"slug": slug,
|
||||
"articles": articles,
|
||||
"paragraphs": paragraphs,
|
||||
}
|
||||
issues_data.append(issue_data)
|
||||
|
||||
print(
|
||||
f" {slug}: {display_title} — "
|
||||
f"{len(articles)} articles, {len(paragraphs)} paragraphs"
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 3: Build intro.json with hierarchical volume groups
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Group issues by volume
|
||||
volume_groups = {}
|
||||
for issue in issues_data:
|
||||
vol = issue["volume"]
|
||||
if vol not in volume_groups:
|
||||
volume_groups[vol] = []
|
||||
volume_groups[vol].append(issue)
|
||||
|
||||
# Build chapters array for intro.json
|
||||
chapters = []
|
||||
for vol_num in sorted(volume_groups.keys()):
|
||||
vol_issues = volume_groups[vol_num]
|
||||
sub_chapters = []
|
||||
for issue in vol_issues:
|
||||
sub_chapters.append(
|
||||
{
|
||||
"slug": issue["slug"],
|
||||
"title": issue["title"],
|
||||
}
|
||||
)
|
||||
chapters.append(
|
||||
{
|
||||
"title": f"Volume {vol_num}",
|
||||
"subChapters": sub_chapters,
|
||||
}
|
||||
)
|
||||
|
||||
total_issues = len(issues_data)
|
||||
total_articles = sum(len(i["articles"]) for i in issues_data)
|
||||
|
||||
intro_json = {
|
||||
"book": "btg",
|
||||
"title": "Back to Godhead",
|
||||
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
||||
"year": 1944,
|
||||
"description": f"The original Back to Godhead magazine founded in 1944. {total_issues} issues across {len(volume_groups)} volumes, containing {total_articles} articles on Kṛṣṇa consciousness, Vedic philosophy, and spiritual life.",
|
||||
"totalIssues": total_issues,
|
||||
"totalArticles": total_articles,
|
||||
"chapters": chapters,
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 4: Write JSON files
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Clean output directory
|
||||
if os.path.exists(issues_dir):
|
||||
for f in os.listdir(issues_dir):
|
||||
os.remove(os.path.join(issues_dir, f))
|
||||
|
||||
# Write intro.json
|
||||
write_json(intro_json, os.path.join(output_dir, "intro.json"))
|
||||
print(f"\n Written: intro.json ({total_issues} issues, {total_articles} articles)")
|
||||
|
||||
# Write issue files
|
||||
order = 100
|
||||
for issue in issues_data:
|
||||
issue_json = {
|
||||
"book": "btg",
|
||||
"volume": issue["volume"],
|
||||
"part": issue["part"],
|
||||
"title": issue["title"],
|
||||
"slug": issue["slug"],
|
||||
"order": order,
|
||||
"articles": issue["articles"],
|
||||
"paragraphs": issue["paragraphs"],
|
||||
}
|
||||
filepath = os.path.join(issues_dir, f"{issue['slug']}.json")
|
||||
write_json(issue_json, filepath)
|
||||
order += 1
|
||||
|
||||
print(f" Written: {len(issues_data)} issue files")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 5: Quality checks
|
||||
# ---------------------------------------------------------------------------
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f" QUALITY CHECKS")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
# Check for ScaGoudy remnants
|
||||
sca_chars = set(SCAGOUDY_TO_IAST.keys())
|
||||
sca_remnants = 0
|
||||
for issue in issues_data:
|
||||
for p in issue["paragraphs"]:
|
||||
if any(c in p for c in sca_chars if c not in ("é", "ü", "à")):
|
||||
# Skip common French/German chars that might be legitimate
|
||||
# Focus on clearly ScaGoudy-only chars
|
||||
if "å" in p or "ë" in p or "ö" in p and "Ö" not in p:
|
||||
sca_remnants += 1
|
||||
break
|
||||
if sca_remnants == 0:
|
||||
print(f" ✅ No ScaGoudy remnants detected")
|
||||
else:
|
||||
print(f" ⚠ {sca_remnants} issues may have ScaGoudy remnants")
|
||||
|
||||
# Check for empty issues
|
||||
empty_issues = [i for i in issues_data if len(i["paragraphs"]) == 0]
|
||||
if empty_issues:
|
||||
print(
|
||||
f" ⚠ {len(empty_issues)} empty issues: {[i['slug'] for i in empty_issues]}"
|
||||
)
|
||||
else:
|
||||
print(f" ✅ No empty issues")
|
||||
|
||||
# Check centered verses
|
||||
verse_count = 0
|
||||
for issue in issues_data:
|
||||
for p in issue["paragraphs"]:
|
||||
if '<div style="text-align:center">' in p:
|
||||
verse_count += 1
|
||||
print(f" ℹ️ {verse_count} center-aligned verse paragraphs")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Summary
|
||||
# ---------------------------------------------------------------------------
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f" EXTRACTION SUMMARY")
|
||||
print(f"{'=' * 60}")
|
||||
for vol_num in sorted(volume_groups.keys()):
|
||||
vol_issues = volume_groups[vol_num]
|
||||
vol_articles = sum(len(i["articles"]) for i in vol_issues)
|
||||
vol_paras = sum(len(i["paragraphs"]) for i in vol_issues)
|
||||
print(
|
||||
f" Volume {vol_num}: {len(vol_issues)} issues, {vol_articles} articles, {vol_paras} paragraphs"
|
||||
)
|
||||
for issue in vol_issues:
|
||||
print(
|
||||
f" {issue['slug']}: {issue['title']} "
|
||||
f"({len(issue['articles'])} articles, {len(issue['paragraphs'])} paras)"
|
||||
)
|
||||
print(f"\n Total: {total_issues} issues, {total_articles} articles")
|
||||
print(f"{'=' * 60}")
|
||||
print(f"\nDone! ✨")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user