547 lines
17 KiB
Python
547 lines
17 KiB
Python
#!/usr/bin/env python3
|
||
"""Extract Back to Godhead magazine from btg-1960.html.
|
||
|
||
Each magazine issue becomes its own JSON file.
|
||
Issues are grouped by volume in the intro.json metadata.
|
||
|
||
Structure:
|
||
- p2 = Volume/Part bookmark (e.g., "BTGPY1: Volume 1, Part 1, 1944")
|
||
- p3 = Volume/Part display title
|
||
- p8 = Article bookmark within an issue (e.g., "BTGPY1a: Message of His Divine Grace")
|
||
- p9 = Article display title (bold)
|
||
- p12, p35 = Body paragraphs
|
||
- p14, p17, p88 = Continuation notes / misc text
|
||
- p15 = Verse blocks (italic, with ScaGoudy encoding)
|
||
- p33, p40, p47, p54 = Magazine mastheads/headers
|
||
|
||
Output:
|
||
lib/jsons/magazines/btg/intro.json
|
||
lib/jsons/magazines/btg/issues/v1-p1.json
|
||
lib/jsons/magazines/btg/issues/v1-p2.json
|
||
...
|
||
lib/jsons/magazines/btg/issues/v4-p1.json
|
||
|
||
Usage:
|
||
python3 scripts/btg_extract.py
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import re
|
||
import sys
|
||
|
||
from bs4 import BeautifulSoup, Tag
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# ScaGoudy → IAST Unicode mapping
|
||
# ---------------------------------------------------------------------------
|
||
SCAGOUDY_TO_IAST = {
|
||
"ä": "ā",
|
||
"Ä": "Ā",
|
||
"å": "ṛ",
|
||
"Å": "Ṛ",
|
||
"ñ": "ṣ",
|
||
"Ñ": "Ṣ",
|
||
"ö": "ṭ",
|
||
"Ö": "Ṭ",
|
||
"ë": "ṇ",
|
||
"Ë": "Ṇ",
|
||
"ò": "ḍ",
|
||
"Ò": "Ḍ",
|
||
"ç": "ś",
|
||
"Ç": "Ś",
|
||
"ì": "ṅ",
|
||
"Ì": "Ṅ",
|
||
"ï": "ñ",
|
||
"Ï": "Ñ",
|
||
"ü": "ū",
|
||
"Ü": "Ū",
|
||
"é": "ī",
|
||
"É": "Ī",
|
||
"à": "ṁ",
|
||
"ù": "ḥ",
|
||
"è": "ṝ",
|
||
"ÿ": "ḷ",
|
||
}
|
||
|
||
|
||
def decode_scagoudy(text: str) -> str:
|
||
"""Replace ScaGoudy-encoded characters with proper IAST Unicode."""
|
||
for src, dst in SCAGOUDY_TO_IAST.items():
|
||
text = text.replace(src, dst)
|
||
return text
|
||
|
||
|
||
def get_css_class(tag: Tag) -> str:
|
||
classes = tag.get("class", [])
|
||
return classes[0] if classes else ""
|
||
|
||
|
||
def inner_html(tag: Tag) -> str:
|
||
"""Get inner HTML of a tag, decode ScaGoudy, clean up spans."""
|
||
html = "".join(str(child) for child in tag.children)
|
||
html = decode_scagoudy(html)
|
||
# <span class="s1">text</span> → keep inner text (s1 is a ref style)
|
||
html = re.sub(
|
||
r'<span[^>]*class="s1"[^>]*>(.*?)</span>', r"\1", html, flags=re.DOTALL
|
||
)
|
||
# <span class="s3">text</span> → keep inner text (superscript digit)
|
||
html = re.sub(
|
||
r'<span[^>]*class="s3"[^>]*>(.*?)</span>', r"\1", html, flags=re.DOTALL
|
||
)
|
||
# <span class="s4">text</span> → keep inner text
|
||
html = re.sub(
|
||
r'<span[^>]*class="s4"[^>]*>(.*?)</span>', r"\1", html, flags=re.DOTALL
|
||
)
|
||
# <span class="s5">text</span> → keep inner text
|
||
html = re.sub(
|
||
r'<span[^>]*class="s5"[^>]*>(.*?)</span>', r"\1", html, flags=re.DOTALL
|
||
)
|
||
# <span class="s7">text</span> → keep inner text
|
||
html = re.sub(
|
||
r'<span[^>]*class="s7"[^>]*>(.*?)</span>', r"\1", html, flags=re.DOTALL
|
||
)
|
||
# <span class="s8">text</span> → keep inner text
|
||
html = re.sub(
|
||
r'<span[^>]*class="s8"[^>]*>(.*?)</span>', r"\1", html, flags=re.DOTALL
|
||
)
|
||
# Remove any remaining spans but keep content
|
||
html = re.sub(r"<span[^>]*>(.*?)</span>", r"\1", html, flags=re.DOTALL)
|
||
# Remove Apple-tab-span
|
||
html = html.replace("\t", " ")
|
||
# Collapse multiple spaces (but not around <br>)
|
||
html = re.sub(r"[ ]+", " ", html)
|
||
html = html.strip()
|
||
# Remove leading <br> tags
|
||
html = re.sub(r"^(<br\s*/?>[\s]*)+", "", html)
|
||
# Remove trailing <br> tags
|
||
html = re.sub(r"(<br\s*/?>[\s]*)+$", "", html)
|
||
return html
|
||
|
||
|
||
def is_verse_paragraph(tag: Tag) -> bool:
|
||
"""Check if a paragraph is a verse block (italic with line breaks)."""
|
||
cls = get_css_class(tag)
|
||
# p15 is commonly used for verse blocks
|
||
if cls == "p15":
|
||
return True
|
||
# Also check for paragraphs that are entirely italic with <br> breaks
|
||
# (some verses use p14 or other classes but are entirely in <i> tags)
|
||
return False
|
||
|
||
|
||
def format_verse(tag: Tag) -> str:
|
||
"""Format a verse paragraph as a center-aligned div."""
|
||
html = inner_html(tag)
|
||
if not html:
|
||
return ""
|
||
# Wrap in center-aligned div
|
||
return f'<div style="text-align:center">{html}</div>'
|
||
|
||
|
||
def write_json(data: dict, path: str):
|
||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||
with open(path, "w", encoding="utf-8") as f:
|
||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||
f.write("\n")
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Parse volume/part info from p2 bookmark text
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def parse_volume_part(text: str):
|
||
"""Parse 'BTGPY1: Volume 1, Part 1, 1944' → (volume, part, date_str, slug).
|
||
Returns (volume_num, part_num, date_string, slug) or None.
|
||
"""
|
||
# Remove the BTGPY prefix
|
||
text = text.strip()
|
||
# Match "Volume X, Part Y, ..."
|
||
m = re.search(r"Volume\s+(\d+),?\s*Part\s+(\d+),?\s*(.*)", text, re.IGNORECASE)
|
||
if m:
|
||
vol = int(m.group(1))
|
||
part = int(m.group(2))
|
||
date_str = m.group(3).strip().rstrip(" ")
|
||
slug = f"v{vol}-p{part}"
|
||
return vol, part, date_str, slug
|
||
return None
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Main extraction
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def main():
|
||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
html_path = os.path.join(base_dir, "public", "html", "btg-1960.html")
|
||
output_dir = os.path.join(base_dir, "lib", "jsons", "magazines", "btg")
|
||
issues_dir = os.path.join(output_dir, "issues")
|
||
|
||
if not os.path.exists(html_path):
|
||
print(f"ERROR: {html_path} not found!")
|
||
sys.exit(1)
|
||
|
||
print(f"Reading {html_path}...")
|
||
with open(html_path, "r", encoding="utf-8") as f:
|
||
content = f.read()
|
||
|
||
soup = BeautifulSoup(content, "html.parser")
|
||
all_tags = soup.find_all("p")
|
||
print(f" Parsed {len(all_tags)} <p> tags total")
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Phase 1: Find all issue boundaries
|
||
# ---------------------------------------------------------------------------
|
||
# Each issue starts with a p2 bookmark tag
|
||
issue_boundaries = [] # List of (tag_index, volume, part, date_str, slug)
|
||
|
||
for i, tag in enumerate(all_tags):
|
||
cls = get_css_class(tag)
|
||
if cls == "p2":
|
||
text = tag.get_text().strip()
|
||
parsed = parse_volume_part(text)
|
||
if parsed:
|
||
vol, part, date_str, slug = parsed
|
||
issue_boundaries.append((i, vol, part, date_str, slug))
|
||
|
||
print(f"\n Found {len(issue_boundaries)} issues")
|
||
for ib in issue_boundaries:
|
||
print(f" tag[{ib[0]}]: Vol {ib[1]}, Part {ib[2]}, {ib[3]} → {ib[4]}")
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Phase 2: Extract articles and paragraphs for each issue
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# Classes to skip entirely
|
||
SKIP_CLASSES = {"p2", "p11", "p24", "p28"}
|
||
|
||
# Classes that are header/masthead material (include as paragraphs)
|
||
MASTHEAD_CLASSES = {"p3", "p4", "p5", "p6", "p7", "p32", "p33", "p34", "p40", "p47"}
|
||
|
||
# Classes that are body text
|
||
BODY_CLASSES = {
|
||
"p10",
|
||
"p12",
|
||
"p13",
|
||
"p14",
|
||
"p15",
|
||
"p16",
|
||
"p17",
|
||
"p35",
|
||
"p36",
|
||
"p37",
|
||
"p38",
|
||
"p39",
|
||
"p41",
|
||
"p42",
|
||
"p43",
|
||
"p44",
|
||
"p45",
|
||
"p46",
|
||
"p48",
|
||
"p49",
|
||
"p50",
|
||
"p51",
|
||
"p52",
|
||
"p53",
|
||
"p54",
|
||
"p55",
|
||
"p56",
|
||
"p57",
|
||
"p58",
|
||
"p59",
|
||
"p60",
|
||
"p61",
|
||
"p62",
|
||
"p63",
|
||
"p64",
|
||
"p65",
|
||
"p66",
|
||
"p67",
|
||
"p68",
|
||
"p69",
|
||
"p70",
|
||
"p71",
|
||
"p72",
|
||
"p73",
|
||
"p74",
|
||
"p75",
|
||
"p76",
|
||
"p77",
|
||
"p78",
|
||
"p79",
|
||
"p80",
|
||
"p81",
|
||
"p82",
|
||
"p83",
|
||
"p84",
|
||
"p85",
|
||
"p86",
|
||
"p87",
|
||
"p88",
|
||
"p89",
|
||
"p90",
|
||
"p91",
|
||
"p92",
|
||
"p93",
|
||
"p94",
|
||
"p95",
|
||
"p96",
|
||
"p97",
|
||
"p98",
|
||
"p99",
|
||
}
|
||
|
||
issues_data = []
|
||
|
||
for idx, (start_tag_idx, vol, part, date_str, slug) in enumerate(issue_boundaries):
|
||
# Determine end boundary
|
||
if idx + 1 < len(issue_boundaries):
|
||
end_tag_idx = issue_boundaries[idx + 1][0]
|
||
else:
|
||
end_tag_idx = len(all_tags)
|
||
|
||
issue_tags = all_tags[start_tag_idx:end_tag_idx]
|
||
|
||
paragraphs = []
|
||
articles = [] # List of { title, slug }
|
||
|
||
for tag in issue_tags:
|
||
cls = get_css_class(tag)
|
||
text = tag.get_text().strip()
|
||
|
||
# Skip bookmark tags and empty lines
|
||
if cls in SKIP_CLASSES:
|
||
continue
|
||
if not text and cls not in ("p15",):
|
||
continue
|
||
|
||
# p8 = Article bookmark — extract title for metadata
|
||
if cls == "p8":
|
||
# Extract article title from bookmark text (remove BTGPYxx: prefix)
|
||
article_text = text
|
||
m = re.match(r"BTGPY\d+[a-z]?:\s*(.*)", article_text)
|
||
if m:
|
||
article_title = m.group(1).strip()
|
||
else:
|
||
article_title = article_text
|
||
article_title = decode_scagoudy(article_title)
|
||
articles.append({"title": article_title})
|
||
continue
|
||
|
||
# p9 = Article display title — render as bold heading
|
||
if cls == "p9":
|
||
html = inner_html(tag)
|
||
if html:
|
||
# Make it a bold article heading
|
||
if not html.startswith("<b>"):
|
||
html = f"<b>{html}</b>"
|
||
paragraphs.append(html)
|
||
continue
|
||
|
||
# p15 = Verse block — center align
|
||
if cls == "p15":
|
||
html = format_verse(tag)
|
||
if html:
|
||
paragraphs.append(html)
|
||
continue
|
||
|
||
# p3 = Volume title — skip (it's the display title of the issue header)
|
||
if cls == "p3":
|
||
continue
|
||
|
||
# Masthead classes — include but as regular paragraphs
|
||
if cls in MASTHEAD_CLASSES:
|
||
html = inner_html(tag)
|
||
if html and html != "<br>":
|
||
paragraphs.append(html)
|
||
continue
|
||
|
||
# Body text classes
|
||
html = inner_html(tag)
|
||
if html and html != "<br>":
|
||
# Check if this looks like a verse (entirely italic with <br> breaks)
|
||
stripped = html.strip()
|
||
if (
|
||
stripped.startswith("<i>")
|
||
and "<br" in stripped
|
||
and not '<div style="text-align:center">' in stripped
|
||
):
|
||
# Check if it's predominantly a verse (italic block with line breaks)
|
||
# by seeing if removing the <i> tags leaves mostly text with <br>
|
||
plain = re.sub(r"</?[bi]>", "", stripped)
|
||
lines = [
|
||
l.strip() for l in re.split(r"<br\s*/?>", plain) if l.strip()
|
||
]
|
||
if len(lines) >= 2:
|
||
html = f'<div style="text-align:center">{stripped}</div>'
|
||
|
||
paragraphs.append(html)
|
||
|
||
# Build display title
|
||
if date_str:
|
||
display_title = f"Volume {vol}, Part {part}, {date_str}"
|
||
else:
|
||
display_title = f"Volume {vol}, Part {part}"
|
||
|
||
issue_data = {
|
||
"book": "btg",
|
||
"volume": vol,
|
||
"part": part,
|
||
"title": display_title,
|
||
"slug": slug,
|
||
"articles": articles,
|
||
"paragraphs": paragraphs,
|
||
}
|
||
issues_data.append(issue_data)
|
||
|
||
print(
|
||
f" {slug}: {display_title} — "
|
||
f"{len(articles)} articles, {len(paragraphs)} paragraphs"
|
||
)
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Phase 3: Build intro.json with hierarchical volume groups
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# Group issues by volume
|
||
volume_groups = {}
|
||
for issue in issues_data:
|
||
vol = issue["volume"]
|
||
if vol not in volume_groups:
|
||
volume_groups[vol] = []
|
||
volume_groups[vol].append(issue)
|
||
|
||
# Build chapters array for intro.json
|
||
chapters = []
|
||
for vol_num in sorted(volume_groups.keys()):
|
||
vol_issues = volume_groups[vol_num]
|
||
sub_chapters = []
|
||
for issue in vol_issues:
|
||
sub_chapters.append(
|
||
{
|
||
"slug": issue["slug"],
|
||
"title": issue["title"],
|
||
}
|
||
)
|
||
chapters.append(
|
||
{
|
||
"title": f"Volume {vol_num}",
|
||
"subChapters": sub_chapters,
|
||
}
|
||
)
|
||
|
||
total_issues = len(issues_data)
|
||
total_articles = sum(len(i["articles"]) for i in issues_data)
|
||
|
||
intro_json = {
|
||
"book": "btg",
|
||
"title": "Back to Godhead",
|
||
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
||
"year": 1944,
|
||
"description": f"The original Back to Godhead magazine founded in 1944. {total_issues} issues across {len(volume_groups)} volumes, containing {total_articles} articles on Kṛṣṇa consciousness, Vedic philosophy, and spiritual life.",
|
||
"totalIssues": total_issues,
|
||
"totalArticles": total_articles,
|
||
"chapters": chapters,
|
||
}
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Phase 4: Write JSON files
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# Clean output directory
|
||
if os.path.exists(issues_dir):
|
||
for f in os.listdir(issues_dir):
|
||
os.remove(os.path.join(issues_dir, f))
|
||
|
||
# Write intro.json
|
||
write_json(intro_json, os.path.join(output_dir, "intro.json"))
|
||
print(f"\n Written: intro.json ({total_issues} issues, {total_articles} articles)")
|
||
|
||
# Write issue files
|
||
order = 100
|
||
for issue in issues_data:
|
||
issue_json = {
|
||
"book": "btg",
|
||
"volume": issue["volume"],
|
||
"part": issue["part"],
|
||
"title": issue["title"],
|
||
"slug": issue["slug"],
|
||
"order": order,
|
||
"articles": issue["articles"],
|
||
"paragraphs": issue["paragraphs"],
|
||
}
|
||
filepath = os.path.join(issues_dir, f"{issue['slug']}.json")
|
||
write_json(issue_json, filepath)
|
||
order += 1
|
||
|
||
print(f" Written: {len(issues_data)} issue files")
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Phase 5: Quality checks
|
||
# ---------------------------------------------------------------------------
|
||
print(f"\n{'=' * 60}")
|
||
print(f" QUALITY CHECKS")
|
||
print(f"{'=' * 60}")
|
||
|
||
# Check for ScaGoudy remnants
|
||
sca_chars = set(SCAGOUDY_TO_IAST.keys())
|
||
sca_remnants = 0
|
||
for issue in issues_data:
|
||
for p in issue["paragraphs"]:
|
||
if any(c in p for c in sca_chars if c not in ("é", "ü", "à")):
|
||
# Skip common French/German chars that might be legitimate
|
||
# Focus on clearly ScaGoudy-only chars
|
||
if "å" in p or "ë" in p or "ö" in p and "Ö" not in p:
|
||
sca_remnants += 1
|
||
break
|
||
if sca_remnants == 0:
|
||
print(f" ✅ No ScaGoudy remnants detected")
|
||
else:
|
||
print(f" ⚠ {sca_remnants} issues may have ScaGoudy remnants")
|
||
|
||
# Check for empty issues
|
||
empty_issues = [i for i in issues_data if len(i["paragraphs"]) == 0]
|
||
if empty_issues:
|
||
print(
|
||
f" ⚠ {len(empty_issues)} empty issues: {[i['slug'] for i in empty_issues]}"
|
||
)
|
||
else:
|
||
print(f" ✅ No empty issues")
|
||
|
||
# Check centered verses
|
||
verse_count = 0
|
||
for issue in issues_data:
|
||
for p in issue["paragraphs"]:
|
||
if '<div style="text-align:center">' in p:
|
||
verse_count += 1
|
||
print(f" ℹ️ {verse_count} center-aligned verse paragraphs")
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Summary
|
||
# ---------------------------------------------------------------------------
|
||
print(f"\n{'=' * 60}")
|
||
print(f" EXTRACTION SUMMARY")
|
||
print(f"{'=' * 60}")
|
||
for vol_num in sorted(volume_groups.keys()):
|
||
vol_issues = volume_groups[vol_num]
|
||
vol_articles = sum(len(i["articles"]) for i in vol_issues)
|
||
vol_paras = sum(len(i["paragraphs"]) for i in vol_issues)
|
||
print(
|
||
f" Volume {vol_num}: {len(vol_issues)} issues, {vol_articles} articles, {vol_paras} paragraphs"
|
||
)
|
||
for issue in vol_issues:
|
||
print(
|
||
f" {issue['slug']}: {issue['title']} "
|
||
f"({len(issue['articles'])} articles, {len(issue['paragraphs'])} paras)"
|
||
)
|
||
print(f"\n Total: {total_issues} issues, {total_articles} articles")
|
||
print(f"{'=' * 60}")
|
||
print(f"\nDone! ✨")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|