718 lines
24 KiB
Python
718 lines
24 KiB
Python
#!/usr/bin/env python3
|
||
"""Extract Back to Godhead magazine (1966-1973) from btg-1978.html.
|
||
|
||
Each magazine issue becomes its own JSON file.
|
||
Issues are grouped by volume/era in the intro.json metadata.
|
||
Advertisement sections are excluded.
|
||
|
||
Structure:
|
||
- p2 = Issue bookmark (bold, e.g., "Back to Godhead Magazine. Vol. 1, No. 01, October 23, 1966")
|
||
- p8 = Article bookmark within an issue
|
||
- p9 = Article display title (bold)
|
||
- p7, p5, p12, p4, p3, etc. = Body text
|
||
- Articles titled "Advertisement" are skipped entirely
|
||
|
||
Output:
|
||
lib/jsons/magazines/btg-1978/intro.json
|
||
lib/jsons/magazines/btg-1978/issues/no-01.json
|
||
lib/jsons/magazines/btg-1978/issues/no-02.json
|
||
...
|
||
lib/jsons/magazines/btg-1978/issues/no-54.json
|
||
|
||
Usage:
|
||
python3 scripts/btg_1978_extract.py
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import re
|
||
import sys
|
||
|
||
from bs4 import BeautifulSoup, Tag
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# ScaGoudy → IAST Unicode mapping
|
||
# ---------------------------------------------------------------------------
|
||
SCAGOUDY_TO_IAST = {
|
||
"ä": "ā",
|
||
"Ä": "Ā",
|
||
"å": "ṛ",
|
||
"Å": "Ṛ",
|
||
"ñ": "ṣ",
|
||
"Ñ": "Ṣ",
|
||
"ö": "ṭ",
|
||
"Ö": "Ṭ",
|
||
"ë": "ṇ",
|
||
"Ë": "Ṇ",
|
||
"ò": "ḍ",
|
||
"Ò": "Ḍ",
|
||
"ç": "ś",
|
||
"Ç": "Ś",
|
||
"ì": "ṅ",
|
||
"Ì": "Ṅ",
|
||
"ï": "ñ",
|
||
"Ï": "Ñ",
|
||
"ü": "ū",
|
||
"Ü": "Ū",
|
||
"é": "ī",
|
||
"É": "Ī",
|
||
"à": "ṁ",
|
||
"ù": "ḥ",
|
||
"è": "ṝ",
|
||
"ÿ": "ḷ",
|
||
}
|
||
|
||
|
||
def decode_scagoudy(text: str) -> str:
|
||
"""Replace ScaGoudy-encoded characters with proper IAST Unicode."""
|
||
for src, dst in SCAGOUDY_TO_IAST.items():
|
||
text = text.replace(src, dst)
|
||
return text
|
||
|
||
|
||
def get_css_class(tag: Tag) -> str:
|
||
classes = tag.get("class", [])
|
||
return classes[0] if classes else ""
|
||
|
||
|
||
def inner_html(tag: Tag) -> str:
|
||
"""Get inner HTML of a tag, decode ScaGoudy, clean up spans."""
|
||
html = "".join(str(child) for child in tag.children)
|
||
html = decode_scagoudy(html)
|
||
|
||
# <span class="s2">text</span> → <i>text</i> (italic marker in btg-1978)
|
||
html = re.sub(
|
||
r'<span[^>]*class="s2"[^>]*>(.*?)</span>', r"<i>\1</i>", html, flags=re.DOTALL
|
||
)
|
||
|
||
# Remove all other span tags but keep inner content
|
||
html = re.sub(r"<span[^>]*>(.*?)</span>", r"\1", html, flags=re.DOTALL)
|
||
|
||
# Remove Apple-tab-span and tab characters
|
||
html = html.replace("\t", " ")
|
||
|
||
# Collapse multiple spaces (but not around <br>)
|
||
html = re.sub(r"[ ]+", " ", html)
|
||
html = html.strip()
|
||
|
||
# Remove leading <br> tags
|
||
html = re.sub(r"^(<br\s*/?>[\s]*)+", "", html)
|
||
# Remove trailing <br> tags
|
||
html = re.sub(r"(<br\s*/?>[\s]*)+$", "", html)
|
||
|
||
return html
|
||
|
||
|
||
def write_json(data: dict, path: str):
|
||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||
with open(path, "w", encoding="utf-8") as f:
|
||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||
f.write("\n")
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Parse issue info from p2 text
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def parse_issue_info(text: str):
|
||
"""Parse issue info from the p2 bookmark text.
|
||
|
||
Handles various formats:
|
||
"Back to Godhead Magazine. Vol. 1, No. 01, October 23, 1966"
|
||
"Back to Godhead Magazine. Vol. 1, No. 06, January 20, 1967"
|
||
"Back to Godhead Magazine. [VOL II No 1.] No. 16, 1968"
|
||
"Back to Godhead Magazine VOL II. No 2. [#17], 1968"
|
||
"Back to Godhead Magazine no. 30, 1969"
|
||
"Back to Godhead Magazine No. 36, 1970"
|
||
|
||
Returns (issue_number, volume, display_title, date_str, slug) or None.
|
||
"""
|
||
text = text.strip()
|
||
# Remove "Back to Godhead Magazine" prefix variations
|
||
text_clean = re.sub(
|
||
r"^Back to Godhead Magazine\.?\s*", "", text, flags=re.IGNORECASE
|
||
).strip()
|
||
|
||
issue_num = None
|
||
volume = None
|
||
date_str = ""
|
||
|
||
# Try to extract issue number from various patterns
|
||
|
||
# Pattern 1: "Vol. X, No. YY, date" or "Vol. X No. YY, date"
|
||
m = re.search(
|
||
r"Vol\.?\s*(\d+)\.?\s*,?\s*No\.?\s*(\d+)\s*,?\s*(.*)", text_clean, re.IGNORECASE
|
||
)
|
||
if m:
|
||
volume = int(m.group(1))
|
||
issue_num = int(m.group(2))
|
||
date_str = m.group(3).strip().rstrip(",. ")
|
||
|
||
# Pattern 2: "[VOL II No 1.] No. 16, 1968" — use the absolute number
|
||
if issue_num is None:
|
||
m = re.search(r"No\.?\s*(\d+)\s*,?\s*(\d{4})", text_clean, re.IGNORECASE)
|
||
if m:
|
||
issue_num = int(m.group(1))
|
||
date_str = m.group(2).strip()
|
||
|
||
# Pattern 3: "VOL II. No 2. [#17], 1968"
|
||
if issue_num is None:
|
||
m = re.search(r"#(\d+)", text_clean)
|
||
if m:
|
||
issue_num = int(m.group(1))
|
||
|
||
# Pattern 4: "no. 30, 1969" or "no 31, 1969"
|
||
if issue_num is None:
|
||
m = re.search(r"no\.?\s*(\d+)\s*,?\s*(.*)", text_clean, re.IGNORECASE)
|
||
if m:
|
||
issue_num = int(m.group(1))
|
||
date_str = m.group(2).strip().rstrip(",. ")
|
||
|
||
# Try to extract volume from text if not found yet
|
||
if volume is None:
|
||
m = re.search(r"VOL\.?\s*(?:II|2)", text_clean, re.IGNORECASE)
|
||
if m:
|
||
volume = 2
|
||
else:
|
||
m = re.search(r"VOL\.?\s*(?:I|1)", text_clean, re.IGNORECASE)
|
||
if m:
|
||
volume = 1
|
||
|
||
if issue_num is None:
|
||
print(f" ⚠ Could not parse issue number from: {text}")
|
||
return None
|
||
|
||
# Extract year from date_str if present
|
||
year_match = re.search(r"(\d{4})", date_str)
|
||
year = int(year_match.group(1)) if year_match else None
|
||
|
||
# Build display title
|
||
if volume and issue_num <= 17:
|
||
display_title = f"Vol. {volume}, No. {issue_num:02d}"
|
||
if date_str:
|
||
display_title += f", {date_str}"
|
||
else:
|
||
display_title = f"No. {issue_num}"
|
||
if date_str:
|
||
display_title += f", {date_str}"
|
||
|
||
slug = f"no-{issue_num:02d}"
|
||
|
||
return issue_num, volume, display_title, date_str, year, slug
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Main extraction
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def main():
|
||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
html_path = os.path.join(base_dir, "public", "html", "btg-1978.html")
|
||
output_dir = os.path.join(base_dir, "lib", "jsons", "magazines", "btg-1978")
|
||
issues_dir = os.path.join(output_dir, "issues")
|
||
|
||
if not os.path.exists(html_path):
|
||
print(f"ERROR: {html_path} not found!")
|
||
sys.exit(1)
|
||
|
||
print(f"Reading {html_path}...")
|
||
with open(html_path, "r", encoding="utf-8") as f:
|
||
content = f.read()
|
||
|
||
soup = BeautifulSoup(content, "html.parser")
|
||
all_tags = soup.find_all("p")
|
||
print(f" Parsed {len(all_tags)} <p> tags total")
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Phase 1: Find all issue boundaries
|
||
# ---------------------------------------------------------------------------
|
||
issue_boundaries = [] # List of (tag_index, issue_num, volume, display_title, date_str, year, slug)
|
||
|
||
for i, tag in enumerate(all_tags):
|
||
cls = get_css_class(tag)
|
||
if cls == "p2":
|
||
text = tag.get_text().strip()
|
||
if "Back to Godhead" in text:
|
||
parsed = parse_issue_info(text)
|
||
if parsed:
|
||
issue_num, volume, display_title, date_str, year, slug = parsed
|
||
issue_boundaries.append(
|
||
(i, issue_num, volume, display_title, date_str, year, slug)
|
||
)
|
||
|
||
print(f"\n Found {len(issue_boundaries)} issues")
|
||
for ib in issue_boundaries:
|
||
tag_idx, inum, vol, title, ds, yr, slug = ib
|
||
print(f" tag[{tag_idx}]: #{inum} (Vol {vol}) {title} → {slug}")
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Phase 2: Extract articles and paragraphs for each issue
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# Known skip classes (bookmarks, empty lines)
|
||
SKIP_CLASSES = {"p2", "p11", "p28", "p24"}
|
||
|
||
issues_data = []
|
||
skipped_ads = 0
|
||
|
||
# Maximum paragraphs per issue — prevents mega-issues from absorbing
|
||
# unmarked content that belongs to missing intermediate issues.
|
||
MAX_PARAGRAPHS_PER_ISSUE = 500
|
||
|
||
for idx, (
|
||
start_tag_idx,
|
||
issue_num,
|
||
vol,
|
||
display_title,
|
||
date_str,
|
||
year,
|
||
slug,
|
||
) in enumerate(issue_boundaries):
|
||
# Determine end boundary
|
||
if idx + 1 < len(issue_boundaries):
|
||
end_tag_idx = issue_boundaries[idx + 1][0]
|
||
else:
|
||
end_tag_idx = len(all_tags)
|
||
|
||
issue_tags = all_tags[start_tag_idx:end_tag_idx]
|
||
|
||
paragraphs = []
|
||
articles = []
|
||
in_advertisement = False
|
||
|
||
for tag in issue_tags:
|
||
cls = get_css_class(tag)
|
||
text = tag.get_text().strip()
|
||
|
||
# Skip bookmark tags and empty lines
|
||
if cls in SKIP_CLASSES:
|
||
continue
|
||
if not text:
|
||
continue
|
||
|
||
# p8 = Article bookmark
|
||
if cls == "p8":
|
||
article_title = decode_scagoudy(text).strip()
|
||
|
||
# Check if this is an advertisement
|
||
if re.match(r"^Advertisements?", article_title, re.IGNORECASE):
|
||
in_advertisement = True
|
||
skipped_ads += 1
|
||
continue
|
||
else:
|
||
in_advertisement = False
|
||
articles.append({"title": article_title})
|
||
continue
|
||
|
||
# p122 and other classes can also be advertisement markers
|
||
if re.match(r"^Advertisements?\s*$", text, re.IGNORECASE):
|
||
in_advertisement = True
|
||
skipped_ads += 1
|
||
continue
|
||
|
||
# p9 = Article display title
|
||
if cls == "p9":
|
||
if in_advertisement:
|
||
continue
|
||
# Also check the p9 display title itself for advertisement
|
||
plain_title = decode_scagoudy(text).strip()
|
||
if re.match(r"^Advertisements?\s*$", plain_title, re.IGNORECASE):
|
||
in_advertisement = True
|
||
skipped_ads += 1
|
||
continue
|
||
html = inner_html(tag)
|
||
if html:
|
||
# Make it a bold article heading if not already
|
||
if not html.startswith("<b>"):
|
||
html = f"<b>{html}</b>"
|
||
paragraphs.append(html)
|
||
continue
|
||
|
||
# Skip content within advertisement sections
|
||
if in_advertisement:
|
||
# Check if next p8 will reset it — for now just skip
|
||
continue
|
||
|
||
# p15 = Verse block — center align
|
||
if cls == "p15":
|
||
html = inner_html(tag)
|
||
if html:
|
||
paragraphs.append(f'<div style="text-align:center">{html}</div>')
|
||
continue
|
||
|
||
# All other classes — extract as body text
|
||
html = inner_html(tag)
|
||
if html and html != "<br>":
|
||
# Check if this looks like a verse (entirely italic with <br> breaks)
|
||
stripped = html.strip()
|
||
if (
|
||
stripped.startswith("<i>")
|
||
and "<br" in stripped
|
||
and '<div style="text-align:center">' not in stripped
|
||
):
|
||
plain = re.sub(r"</?[bi]>", "", stripped)
|
||
lines = [
|
||
l.strip() for l in re.split(r"<br\s*/?>", plain) if l.strip()
|
||
]
|
||
if len(lines) >= 2:
|
||
html = f'<div style="text-align:center">{stripped}</div>'
|
||
|
||
paragraphs.append(html)
|
||
|
||
# Post-process: remove any ad-like paragraphs that slipped through.
|
||
# Two-pass approach:
|
||
# 1. Detect ad section start markers and skip until next real article
|
||
# 2. Filter individual ad-like paragraphs by content patterns
|
||
|
||
# --- Pass 1: Section-level ad detection ---
|
||
ad_section_patterns = [
|
||
r"^Advertisements?\s*$",
|
||
r"^CIRCULATION AND SUBSCRIPTIONS",
|
||
r"^Order Form",
|
||
r"^SPECIAL OFFER",
|
||
r"^Subscriptions?:",
|
||
r"^WORLDWIDE IMPORTS",
|
||
r"^THE\s+ORIENT\s+SHOP",
|
||
r"^PROTECT YOUR CHECKS",
|
||
]
|
||
ad_section_re = re.compile("|".join(ad_section_patterns), re.IGNORECASE)
|
||
|
||
cleaned_pass1 = []
|
||
skip_until_next_bold = False
|
||
for p in paragraphs:
|
||
plain = re.sub(r"<[^>]+>", "", p).strip()
|
||
# Detect ad section start
|
||
if ad_section_re.search(plain):
|
||
skip_until_next_bold = True
|
||
continue
|
||
if skip_until_next_bold:
|
||
# A bold article heading that's NOT an ad resets the skip
|
||
if p.startswith("<b>") and len(plain) < 200:
|
||
if ad_section_re.search(plain):
|
||
continue
|
||
else:
|
||
skip_until_next_bold = False
|
||
cleaned_pass1.append(p)
|
||
continue
|
||
# If it looks like real content (long paragraph), stop skipping
|
||
if len(plain) > 150:
|
||
skip_until_next_bold = False
|
||
cleaned_pass1.append(p)
|
||
continue
|
||
# Short paragraphs in ad zone — skip
|
||
continue
|
||
cleaned_pass1.append(p)
|
||
|
||
# --- Pass 2: Individual paragraph ad detection ---
|
||
# These patterns catch standalone ad fragments that weren't part of
|
||
# a marked advertisement section.
|
||
ad_para_keywords = [
|
||
"subscription",
|
||
"per copy",
|
||
"enclosed is",
|
||
"please send me",
|
||
"order form",
|
||
"postage paid",
|
||
"money order",
|
||
"check or money order",
|
||
"zip code",
|
||
"state_____",
|
||
"address_____",
|
||
"city_____",
|
||
"available from iskcon",
|
||
"waterpipes",
|
||
"djellabas",
|
||
"panchos",
|
||
"orient shop",
|
||
"worldwide imports",
|
||
"sullivan st",
|
||
"bleecker",
|
||
"check protector",
|
||
"safeguards your checks",
|
||
"satisfaction guaranteed",
|
||
"import-export",
|
||
"davis & manswell",
|
||
"davis & manswell",
|
||
"bedford ave",
|
||
"brooklyn, n.y.",
|
||
"immediate delivery",
|
||
"free gift guide",
|
||
"write for free",
|
||
"free catalog",
|
||
"free catalogue",
|
||
"protect your checks",
|
||
"name of purchaser",
|
||
"such dresses at $",
|
||
"and gowns yet $",
|
||
"price per copy",
|
||
"one-year subscription",
|
||
"year subscription costs",
|
||
"back issues available",
|
||
"a subscription to",
|
||
"send check or money",
|
||
"your name & address",
|
||
"your name & address",
|
||
"674-9511",
|
||
]
|
||
# Also detect short paragraphs that look like addresses or phone numbers
|
||
address_re = re.compile(
|
||
r"^\d+\s+\w+\s+(st|street|ave|avenue|blvd|road|rd|lane|ln|dr|drive)\b",
|
||
re.IGNORECASE,
|
||
)
|
||
phone_re = re.compile(r"^\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}\s*$")
|
||
zipcode_re = re.compile(
|
||
r"(N\.?Y\.?\s+\d{5}|CA\s+\d{5}|Brooklyn|Manhattan)",
|
||
re.IGNORECASE,
|
||
)
|
||
|
||
cleaned = []
|
||
for p in cleaned_pass1:
|
||
plain = re.sub(r"<[^>]+>", "", p).strip()
|
||
plain_lower = plain.lower()
|
||
|
||
# Skip if matches any ad keyword
|
||
if any(kw in plain_lower for kw in ad_para_keywords):
|
||
continue
|
||
|
||
# Skip short paragraphs that look like addresses
|
||
if len(plain) < 80 and address_re.search(plain):
|
||
continue
|
||
|
||
# Skip standalone phone numbers
|
||
if phone_re.match(plain):
|
||
continue
|
||
|
||
# Skip short paragraphs with zip codes (likely address fragments)
|
||
if len(plain) < 60 and zipcode_re.search(plain):
|
||
continue
|
||
|
||
# Skip price-only paragraphs like "$1 plus 25c"
|
||
if re.match(
|
||
r"^\$?\d+[\.\d]*\s*(plus|postage|each|per)", plain, re.IGNORECASE
|
||
):
|
||
continue
|
||
|
||
cleaned.append(p)
|
||
paragraphs = cleaned
|
||
|
||
# Cap paragraphs to prevent mega-issues from absorbing unmarked
|
||
# intermediate issues (e.g., issues 18-29 have no p2 markers and
|
||
# would otherwise be absorbed into issue 17).
|
||
if len(paragraphs) > MAX_PARAGRAPHS_PER_ISSUE:
|
||
print(
|
||
f" ⚠ {slug}: Capped from {len(paragraphs)} to {MAX_PARAGRAPHS_PER_ISSUE} paragraphs "
|
||
f"(likely absorbed unmarked intermediate issues)"
|
||
)
|
||
paragraphs = paragraphs[:MAX_PARAGRAPHS_PER_ISSUE]
|
||
|
||
issue_data = {
|
||
"book": "btg-1978",
|
||
"volume": vol,
|
||
"part": issue_num,
|
||
"title": display_title,
|
||
"slug": slug,
|
||
"year": year,
|
||
"articles": articles,
|
||
"paragraphs": paragraphs,
|
||
}
|
||
issues_data.append(issue_data)
|
||
|
||
print(
|
||
f" {slug}: {display_title} — "
|
||
f"{len(articles)} articles, {len(paragraphs)} paragraphs"
|
||
)
|
||
|
||
print(f"\n Skipped {skipped_ads} advertisement sections")
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Phase 3: Build intro.json with hierarchical groups
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# Group issues by era/volume
|
||
# Early issues (1-17) have Vol 1/Vol 2 designations
|
||
# Later issues (30+) just have issue numbers
|
||
group_early = [] # Issues 1-17 (1966-1968)
|
||
group_later = [] # Issues 30+ (1969-1973)
|
||
|
||
for issue in issues_data:
|
||
if issue["part"] <= 17:
|
||
group_early.append(issue)
|
||
else:
|
||
group_later.append(issue)
|
||
|
||
chapters = []
|
||
|
||
if group_early:
|
||
sub = []
|
||
for issue in group_early:
|
||
sub.append({"slug": issue["slug"], "title": issue["title"]})
|
||
chapters.append(
|
||
{
|
||
"title": "Issues #1–17 (1966–1968)",
|
||
"subChapters": sub,
|
||
}
|
||
)
|
||
|
||
if group_later:
|
||
sub = []
|
||
for issue in group_later:
|
||
sub.append({"slug": issue["slug"], "title": issue["title"]})
|
||
chapters.append(
|
||
{
|
||
"title": "Issues #30–54 (1969–1973)",
|
||
"subChapters": sub,
|
||
}
|
||
)
|
||
|
||
total_issues = len(issues_data)
|
||
total_articles = sum(len(i["articles"]) for i in issues_data)
|
||
|
||
# Determine year range
|
||
years = [i["year"] for i in issues_data if i["year"]]
|
||
min_year = min(years) if years else None
|
||
max_year = max(years) if years else None
|
||
|
||
intro_json = {
|
||
"book": "btg-1978",
|
||
"title": "Back to Godhead (1966–1973)",
|
||
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
||
"year": min_year,
|
||
"description": (
|
||
f"Back to Godhead magazine issues from the Western era ({min_year}–{max_year}). "
|
||
f"{total_issues} issues containing {total_articles} articles — lectures, essays, "
|
||
f"poems, and philosophical discussions on Kṛṣṇa consciousness."
|
||
),
|
||
"totalIssues": total_issues,
|
||
"totalArticles": total_articles,
|
||
"flag": "Pre 1978",
|
||
"chapters": chapters,
|
||
}
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Phase 4: Write JSON files
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# Clean output directory
|
||
if os.path.exists(issues_dir):
|
||
for f in os.listdir(issues_dir):
|
||
os.remove(os.path.join(issues_dir, f))
|
||
|
||
# Write intro.json
|
||
write_json(intro_json, os.path.join(output_dir, "intro.json"))
|
||
print(f"\n Written: intro.json ({total_issues} issues, {total_articles} articles)")
|
||
|
||
# Write issue files
|
||
order = 100
|
||
for issue in issues_data:
|
||
issue_json = {
|
||
"book": "btg-1978",
|
||
"volume": issue["volume"],
|
||
"part": issue["part"],
|
||
"title": issue["title"],
|
||
"slug": issue["slug"],
|
||
"order": order,
|
||
"articles": issue["articles"],
|
||
"paragraphs": issue["paragraphs"],
|
||
}
|
||
filepath = os.path.join(issues_dir, f"{issue['slug']}.json")
|
||
write_json(issue_json, filepath)
|
||
order += 1
|
||
|
||
print(f" Written: {len(issues_data)} issue files")
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Phase 5: Quality checks
|
||
# ---------------------------------------------------------------------------
|
||
print(f"\n{'=' * 60}")
|
||
print(f" QUALITY CHECKS")
|
||
print(f"{'=' * 60}")
|
||
|
||
# Check for ScaGoudy remnants
|
||
sca_chars = set(SCAGOUDY_TO_IAST.keys())
|
||
sca_remnants = 0
|
||
for issue in issues_data:
|
||
for p in issue["paragraphs"]:
|
||
if any(c in p for c in ("å", "ë", "ö")):
|
||
sca_remnants += 1
|
||
break
|
||
if sca_remnants == 0:
|
||
print(f" ✅ No ScaGoudy remnants detected")
|
||
else:
|
||
print(f" ⚠ {sca_remnants} issues may have ScaGoudy remnants")
|
||
|
||
# Check for empty issues
|
||
empty_issues = [i for i in issues_data if len(i["paragraphs"]) == 0]
|
||
if empty_issues:
|
||
print(
|
||
f" ⚠ {len(empty_issues)} empty issues: {[i['slug'] for i in empty_issues]}"
|
||
)
|
||
else:
|
||
print(f" ✅ No empty issues")
|
||
|
||
# Check centered verses
|
||
verse_count = 0
|
||
for issue in issues_data:
|
||
for p in issue["paragraphs"]:
|
||
if '<div style="text-align:center">' in p:
|
||
verse_count += 1
|
||
print(f" ℹ️ {verse_count} center-aligned verse paragraphs")
|
||
|
||
# Check for remaining advertisement content
|
||
ad_check = 0
|
||
ad_keywords = ["advertisement", "subscribe", "subscription form"]
|
||
for issue in issues_data:
|
||
for p in issue["paragraphs"]:
|
||
lower_p = p.lower()
|
||
if any(kw in lower_p for kw in ad_keywords):
|
||
# Only flag if the paragraph is primarily an ad
|
||
if "advertisement" in lower_p and len(p) < 200:
|
||
ad_check += 1
|
||
if ad_check == 0:
|
||
print(f" ✅ No advertisement content detected in output")
|
||
else:
|
||
print(f" ⚠ {ad_check} paragraphs may contain advertisement content")
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Summary
|
||
# ---------------------------------------------------------------------------
|
||
print(f"\n{'=' * 60}")
|
||
print(f" EXTRACTION SUMMARY")
|
||
print(f"{'=' * 60}")
|
||
|
||
if group_early:
|
||
early_arts = sum(len(i["articles"]) for i in group_early)
|
||
early_paras = sum(len(i["paragraphs"]) for i in group_early)
|
||
print(
|
||
f" Issues #1–17 (1966–1968): {len(group_early)} issues, "
|
||
f"{early_arts} articles, {early_paras} paragraphs"
|
||
)
|
||
for issue in group_early:
|
||
print(
|
||
f" {issue['slug']}: {issue['title']} "
|
||
f"({len(issue['articles'])} articles, {len(issue['paragraphs'])} paras)"
|
||
)
|
||
|
||
if group_later:
|
||
later_arts = sum(len(i["articles"]) for i in group_later)
|
||
later_paras = sum(len(i["paragraphs"]) for i in group_later)
|
||
print(
|
||
f" Issues #30–54 (1969–1973): {len(group_later)} issues, "
|
||
f"{later_arts} articles, {later_paras} paragraphs"
|
||
)
|
||
for issue in group_later:
|
||
print(
|
||
f" {issue['slug']}: {issue['title']} "
|
||
f"({len(issue['articles'])} articles, {len(issue['paragraphs'])} paras)"
|
||
)
|
||
|
||
print(f"\n Total: {total_issues} issues, {total_articles} articles")
|
||
print(f" Skipped: {skipped_ads} advertisement sections")
|
||
print(f"{'=' * 60}")
|
||
print(f"\nDone! ✨")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|