feat: letters added
This commit is contained in:
gosangam
@@ -0,0 +1,717 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract Back to Godhead magazine (1966-1973) from btg-1978.html.
|
||||
|
||||
Each magazine issue becomes its own JSON file.
|
||||
Issues are grouped by volume/era in the intro.json metadata.
|
||||
Advertisement sections are excluded.
|
||||
|
||||
Structure:
|
||||
- p2 = Issue bookmark (bold, e.g., "Back to Godhead Magazine. Vol. 1, No. 01, October 23, 1966")
|
||||
- p8 = Article bookmark within an issue
|
||||
- p9 = Article display title (bold)
|
||||
- p7, p5, p12, p4, p3, etc. = Body text
|
||||
- Articles titled "Advertisement" are skipped entirely
|
||||
|
||||
Output:
|
||||
lib/jsons/magazines/btg-1978/intro.json
|
||||
lib/jsons/magazines/btg-1978/issues/no-01.json
|
||||
lib/jsons/magazines/btg-1978/issues/no-02.json
|
||||
...
|
||||
lib/jsons/magazines/btg-1978/issues/no-54.json
|
||||
|
||||
Usage:
|
||||
python3 scripts/btg_1978_extract.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ScaGoudy → IAST Unicode mapping
|
||||
# ---------------------------------------------------------------------------
|
||||
SCAGOUDY_TO_IAST = {
|
||||
"ä": "ā",
|
||||
"Ä": "Ā",
|
||||
"å": "ṛ",
|
||||
"Å": "Ṛ",
|
||||
"ñ": "ṣ",
|
||||
"Ñ": "Ṣ",
|
||||
"ö": "ṭ",
|
||||
"Ö": "Ṭ",
|
||||
"ë": "ṇ",
|
||||
"Ë": "Ṇ",
|
||||
"ò": "ḍ",
|
||||
"Ò": "Ḍ",
|
||||
"ç": "ś",
|
||||
"Ç": "Ś",
|
||||
"ì": "ṅ",
|
||||
"Ì": "Ṅ",
|
||||
"ï": "ñ",
|
||||
"Ï": "Ñ",
|
||||
"ü": "ū",
|
||||
"Ü": "Ū",
|
||||
"é": "ī",
|
||||
"É": "Ī",
|
||||
"à": "ṁ",
|
||||
"ù": "ḥ",
|
||||
"è": "ṝ",
|
||||
"ÿ": "ḷ",
|
||||
}
|
||||
|
||||
|
||||
def decode_scagoudy(text: str) -> str:
|
||||
"""Replace ScaGoudy-encoded characters with proper IAST Unicode."""
|
||||
for src, dst in SCAGOUDY_TO_IAST.items():
|
||||
text = text.replace(src, dst)
|
||||
return text
|
||||
|
||||
|
||||
def get_css_class(tag: Tag) -> str:
|
||||
classes = tag.get("class", [])
|
||||
return classes[0] if classes else ""
|
||||
|
||||
|
||||
def inner_html(tag: Tag) -> str:
|
||||
"""Get inner HTML of a tag, decode ScaGoudy, clean up spans."""
|
||||
html = "".join(str(child) for child in tag.children)
|
||||
html = decode_scagoudy(html)
|
||||
|
||||
# <span class="s2">text</span> → <i>text</i> (italic marker in btg-1978)
|
||||
html = re.sub(
|
||||
r'<span[^>]*class="s2"[^>]*>(.*?)</span>', r"<i>\1</i>", html, flags=re.DOTALL
|
||||
)
|
||||
|
||||
# Remove all other span tags but keep inner content
|
||||
html = re.sub(r"<span[^>]*>(.*?)</span>", r"\1", html, flags=re.DOTALL)
|
||||
|
||||
# Remove Apple-tab-span and tab characters
|
||||
html = html.replace("\t", " ")
|
||||
|
||||
# Collapse multiple spaces (but not around <br>)
|
||||
html = re.sub(r"[ ]+", " ", html)
|
||||
html = html.strip()
|
||||
|
||||
# Remove leading <br> tags
|
||||
html = re.sub(r"^(<br\s*/?>[\s]*)+", "", html)
|
||||
# Remove trailing <br> tags
|
||||
html = re.sub(r"(<br\s*/?>[\s]*)+$", "", html)
|
||||
|
||||
return html
|
||||
|
||||
|
||||
def write_json(data: dict, path: str):
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
f.write("\n")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Parse issue info from p2 text
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def parse_issue_info(text: str):
|
||||
"""Parse issue info from the p2 bookmark text.
|
||||
|
||||
Handles various formats:
|
||||
"Back to Godhead Magazine. Vol. 1, No. 01, October 23, 1966"
|
||||
"Back to Godhead Magazine. Vol. 1, No. 06, January 20, 1967"
|
||||
"Back to Godhead Magazine. [VOL II No 1.] No. 16, 1968"
|
||||
"Back to Godhead Magazine VOL II. No 2. [#17], 1968"
|
||||
"Back to Godhead Magazine no. 30, 1969"
|
||||
"Back to Godhead Magazine No. 36, 1970"
|
||||
|
||||
Returns (issue_number, volume, display_title, date_str, slug) or None.
|
||||
"""
|
||||
text = text.strip()
|
||||
# Remove "Back to Godhead Magazine" prefix variations
|
||||
text_clean = re.sub(
|
||||
r"^Back to Godhead Magazine\.?\s*", "", text, flags=re.IGNORECASE
|
||||
).strip()
|
||||
|
||||
issue_num = None
|
||||
volume = None
|
||||
date_str = ""
|
||||
|
||||
# Try to extract issue number from various patterns
|
||||
|
||||
# Pattern 1: "Vol. X, No. YY, date" or "Vol. X No. YY, date"
|
||||
m = re.search(
|
||||
r"Vol\.?\s*(\d+)\.?\s*,?\s*No\.?\s*(\d+)\s*,?\s*(.*)", text_clean, re.IGNORECASE
|
||||
)
|
||||
if m:
|
||||
volume = int(m.group(1))
|
||||
issue_num = int(m.group(2))
|
||||
date_str = m.group(3).strip().rstrip(",. ")
|
||||
|
||||
# Pattern 2: "[VOL II No 1.] No. 16, 1968" — use the absolute number
|
||||
if issue_num is None:
|
||||
m = re.search(r"No\.?\s*(\d+)\s*,?\s*(\d{4})", text_clean, re.IGNORECASE)
|
||||
if m:
|
||||
issue_num = int(m.group(1))
|
||||
date_str = m.group(2).strip()
|
||||
|
||||
# Pattern 3: "VOL II. No 2. [#17], 1968"
|
||||
if issue_num is None:
|
||||
m = re.search(r"#(\d+)", text_clean)
|
||||
if m:
|
||||
issue_num = int(m.group(1))
|
||||
|
||||
# Pattern 4: "no. 30, 1969" or "no 31, 1969"
|
||||
if issue_num is None:
|
||||
m = re.search(r"no\.?\s*(\d+)\s*,?\s*(.*)", text_clean, re.IGNORECASE)
|
||||
if m:
|
||||
issue_num = int(m.group(1))
|
||||
date_str = m.group(2).strip().rstrip(",. ")
|
||||
|
||||
# Try to extract volume from text if not found yet
|
||||
if volume is None:
|
||||
m = re.search(r"VOL\.?\s*(?:II|2)", text_clean, re.IGNORECASE)
|
||||
if m:
|
||||
volume = 2
|
||||
else:
|
||||
m = re.search(r"VOL\.?\s*(?:I|1)", text_clean, re.IGNORECASE)
|
||||
if m:
|
||||
volume = 1
|
||||
|
||||
if issue_num is None:
|
||||
print(f" ⚠ Could not parse issue number from: {text}")
|
||||
return None
|
||||
|
||||
# Extract year from date_str if present
|
||||
year_match = re.search(r"(\d{4})", date_str)
|
||||
year = int(year_match.group(1)) if year_match else None
|
||||
|
||||
# Build display title
|
||||
if volume and issue_num <= 17:
|
||||
display_title = f"Vol. {volume}, No. {issue_num:02d}"
|
||||
if date_str:
|
||||
display_title += f", {date_str}"
|
||||
else:
|
||||
display_title = f"No. {issue_num}"
|
||||
if date_str:
|
||||
display_title += f", {date_str}"
|
||||
|
||||
slug = f"no-{issue_num:02d}"
|
||||
|
||||
return issue_num, volume, display_title, date_str, year, slug
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "btg-1978.html")
|
||||
output_dir = os.path.join(base_dir, "lib", "jsons", "magazines", "btg-1978")
|
||||
issues_dir = os.path.join(output_dir, "issues")
|
||||
|
||||
if not os.path.exists(html_path):
|
||||
print(f"ERROR: {html_path} not found!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Reading {html_path}...")
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
|
||||
soup = BeautifulSoup(content, "html.parser")
|
||||
all_tags = soup.find_all("p")
|
||||
print(f" Parsed {len(all_tags)} <p> tags total")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 1: Find all issue boundaries
|
||||
# ---------------------------------------------------------------------------
|
||||
issue_boundaries = [] # List of (tag_index, issue_num, volume, display_title, date_str, year, slug)
|
||||
|
||||
for i, tag in enumerate(all_tags):
|
||||
cls = get_css_class(tag)
|
||||
if cls == "p2":
|
||||
text = tag.get_text().strip()
|
||||
if "Back to Godhead" in text:
|
||||
parsed = parse_issue_info(text)
|
||||
if parsed:
|
||||
issue_num, volume, display_title, date_str, year, slug = parsed
|
||||
issue_boundaries.append(
|
||||
(i, issue_num, volume, display_title, date_str, year, slug)
|
||||
)
|
||||
|
||||
print(f"\n Found {len(issue_boundaries)} issues")
|
||||
for ib in issue_boundaries:
|
||||
tag_idx, inum, vol, title, ds, yr, slug = ib
|
||||
print(f" tag[{tag_idx}]: #{inum} (Vol {vol}) {title} → {slug}")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 2: Extract articles and paragraphs for each issue
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Known skip classes (bookmarks, empty lines)
|
||||
SKIP_CLASSES = {"p2", "p11", "p28", "p24"}
|
||||
|
||||
issues_data = []
|
||||
skipped_ads = 0
|
||||
|
||||
# Maximum paragraphs per issue — prevents mega-issues from absorbing
|
||||
# unmarked content that belongs to missing intermediate issues.
|
||||
MAX_PARAGRAPHS_PER_ISSUE = 500
|
||||
|
||||
for idx, (
|
||||
start_tag_idx,
|
||||
issue_num,
|
||||
vol,
|
||||
display_title,
|
||||
date_str,
|
||||
year,
|
||||
slug,
|
||||
) in enumerate(issue_boundaries):
|
||||
# Determine end boundary
|
||||
if idx + 1 < len(issue_boundaries):
|
||||
end_tag_idx = issue_boundaries[idx + 1][0]
|
||||
else:
|
||||
end_tag_idx = len(all_tags)
|
||||
|
||||
issue_tags = all_tags[start_tag_idx:end_tag_idx]
|
||||
|
||||
paragraphs = []
|
||||
articles = []
|
||||
in_advertisement = False
|
||||
|
||||
for tag in issue_tags:
|
||||
cls = get_css_class(tag)
|
||||
text = tag.get_text().strip()
|
||||
|
||||
# Skip bookmark tags and empty lines
|
||||
if cls in SKIP_CLASSES:
|
||||
continue
|
||||
if not text:
|
||||
continue
|
||||
|
||||
# p8 = Article bookmark
|
||||
if cls == "p8":
|
||||
article_title = decode_scagoudy(text).strip()
|
||||
|
||||
# Check if this is an advertisement
|
||||
if re.match(r"^Advertisements?", article_title, re.IGNORECASE):
|
||||
in_advertisement = True
|
||||
skipped_ads += 1
|
||||
continue
|
||||
else:
|
||||
in_advertisement = False
|
||||
articles.append({"title": article_title})
|
||||
continue
|
||||
|
||||
# p122 and other classes can also be advertisement markers
|
||||
if re.match(r"^Advertisements?\s*$", text, re.IGNORECASE):
|
||||
in_advertisement = True
|
||||
skipped_ads += 1
|
||||
continue
|
||||
|
||||
# p9 = Article display title
|
||||
if cls == "p9":
|
||||
if in_advertisement:
|
||||
continue
|
||||
# Also check the p9 display title itself for advertisement
|
||||
plain_title = decode_scagoudy(text).strip()
|
||||
if re.match(r"^Advertisements?\s*$", plain_title, re.IGNORECASE):
|
||||
in_advertisement = True
|
||||
skipped_ads += 1
|
||||
continue
|
||||
html = inner_html(tag)
|
||||
if html:
|
||||
# Make it a bold article heading if not already
|
||||
if not html.startswith("<b>"):
|
||||
html = f"<b>{html}</b>"
|
||||
paragraphs.append(html)
|
||||
continue
|
||||
|
||||
# Skip content within advertisement sections
|
||||
if in_advertisement:
|
||||
# Check if next p8 will reset it — for now just skip
|
||||
continue
|
||||
|
||||
# p15 = Verse block — center align
|
||||
if cls == "p15":
|
||||
html = inner_html(tag)
|
||||
if html:
|
||||
paragraphs.append(f'<div style="text-align:center">{html}</div>')
|
||||
continue
|
||||
|
||||
# All other classes — extract as body text
|
||||
html = inner_html(tag)
|
||||
if html and html != "<br>":
|
||||
# Check if this looks like a verse (entirely italic with <br> breaks)
|
||||
stripped = html.strip()
|
||||
if (
|
||||
stripped.startswith("<i>")
|
||||
and "<br" in stripped
|
||||
and '<div style="text-align:center">' not in stripped
|
||||
):
|
||||
plain = re.sub(r"</?[bi]>", "", stripped)
|
||||
lines = [
|
||||
l.strip() for l in re.split(r"<br\s*/?>", plain) if l.strip()
|
||||
]
|
||||
if len(lines) >= 2:
|
||||
html = f'<div style="text-align:center">{stripped}</div>'
|
||||
|
||||
paragraphs.append(html)
|
||||
|
||||
# Post-process: remove any ad-like paragraphs that slipped through.
|
||||
# Two-pass approach:
|
||||
# 1. Detect ad section start markers and skip until next real article
|
||||
# 2. Filter individual ad-like paragraphs by content patterns
|
||||
|
||||
# --- Pass 1: Section-level ad detection ---
|
||||
ad_section_patterns = [
|
||||
r"^Advertisements?\s*$",
|
||||
r"^CIRCULATION AND SUBSCRIPTIONS",
|
||||
r"^Order Form",
|
||||
r"^SPECIAL OFFER",
|
||||
r"^Subscriptions?:",
|
||||
r"^WORLDWIDE IMPORTS",
|
||||
r"^THE\s+ORIENT\s+SHOP",
|
||||
r"^PROTECT YOUR CHECKS",
|
||||
]
|
||||
ad_section_re = re.compile("|".join(ad_section_patterns), re.IGNORECASE)
|
||||
|
||||
cleaned_pass1 = []
|
||||
skip_until_next_bold = False
|
||||
for p in paragraphs:
|
||||
plain = re.sub(r"<[^>]+>", "", p).strip()
|
||||
# Detect ad section start
|
||||
if ad_section_re.search(plain):
|
||||
skip_until_next_bold = True
|
||||
continue
|
||||
if skip_until_next_bold:
|
||||
# A bold article heading that's NOT an ad resets the skip
|
||||
if p.startswith("<b>") and len(plain) < 200:
|
||||
if ad_section_re.search(plain):
|
||||
continue
|
||||
else:
|
||||
skip_until_next_bold = False
|
||||
cleaned_pass1.append(p)
|
||||
continue
|
||||
# If it looks like real content (long paragraph), stop skipping
|
||||
if len(plain) > 150:
|
||||
skip_until_next_bold = False
|
||||
cleaned_pass1.append(p)
|
||||
continue
|
||||
# Short paragraphs in ad zone — skip
|
||||
continue
|
||||
cleaned_pass1.append(p)
|
||||
|
||||
# --- Pass 2: Individual paragraph ad detection ---
|
||||
# These patterns catch standalone ad fragments that weren't part of
|
||||
# a marked advertisement section.
|
||||
ad_para_keywords = [
|
||||
"subscription",
|
||||
"per copy",
|
||||
"enclosed is",
|
||||
"please send me",
|
||||
"order form",
|
||||
"postage paid",
|
||||
"money order",
|
||||
"check or money order",
|
||||
"zip code",
|
||||
"state_____",
|
||||
"address_____",
|
||||
"city_____",
|
||||
"available from iskcon",
|
||||
"waterpipes",
|
||||
"djellabas",
|
||||
"panchos",
|
||||
"orient shop",
|
||||
"worldwide imports",
|
||||
"sullivan st",
|
||||
"bleecker",
|
||||
"check protector",
|
||||
"safeguards your checks",
|
||||
"satisfaction guaranteed",
|
||||
"import-export",
|
||||
"davis & manswell",
|
||||
"davis & manswell",
|
||||
"bedford ave",
|
||||
"brooklyn, n.y.",
|
||||
"immediate delivery",
|
||||
"free gift guide",
|
||||
"write for free",
|
||||
"free catalog",
|
||||
"free catalogue",
|
||||
"protect your checks",
|
||||
"name of purchaser",
|
||||
"such dresses at $",
|
||||
"and gowns yet $",
|
||||
"price per copy",
|
||||
"one-year subscription",
|
||||
"year subscription costs",
|
||||
"back issues available",
|
||||
"a subscription to",
|
||||
"send check or money",
|
||||
"your name & address",
|
||||
"your name & address",
|
||||
"674-9511",
|
||||
]
|
||||
# Also detect short paragraphs that look like addresses or phone numbers
|
||||
address_re = re.compile(
|
||||
r"^\d+\s+\w+\s+(st|street|ave|avenue|blvd|road|rd|lane|ln|dr|drive)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
phone_re = re.compile(r"^\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}\s*$")
|
||||
zipcode_re = re.compile(
|
||||
r"(N\.?Y\.?\s+\d{5}|CA\s+\d{5}|Brooklyn|Manhattan)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
cleaned = []
|
||||
for p in cleaned_pass1:
|
||||
plain = re.sub(r"<[^>]+>", "", p).strip()
|
||||
plain_lower = plain.lower()
|
||||
|
||||
# Skip if matches any ad keyword
|
||||
if any(kw in plain_lower for kw in ad_para_keywords):
|
||||
continue
|
||||
|
||||
# Skip short paragraphs that look like addresses
|
||||
if len(plain) < 80 and address_re.search(plain):
|
||||
continue
|
||||
|
||||
# Skip standalone phone numbers
|
||||
if phone_re.match(plain):
|
||||
continue
|
||||
|
||||
# Skip short paragraphs with zip codes (likely address fragments)
|
||||
if len(plain) < 60 and zipcode_re.search(plain):
|
||||
continue
|
||||
|
||||
# Skip price-only paragraphs like "$1 plus 25c"
|
||||
if re.match(
|
||||
r"^\$?\d+[\.\d]*\s*(plus|postage|each|per)", plain, re.IGNORECASE
|
||||
):
|
||||
continue
|
||||
|
||||
cleaned.append(p)
|
||||
paragraphs = cleaned
|
||||
|
||||
# Cap paragraphs to prevent mega-issues from absorbing unmarked
|
||||
# intermediate issues (e.g., issues 18-29 have no p2 markers and
|
||||
# would otherwise be absorbed into issue 17).
|
||||
if len(paragraphs) > MAX_PARAGRAPHS_PER_ISSUE:
|
||||
print(
|
||||
f" ⚠ {slug}: Capped from {len(paragraphs)} to {MAX_PARAGRAPHS_PER_ISSUE} paragraphs "
|
||||
f"(likely absorbed unmarked intermediate issues)"
|
||||
)
|
||||
paragraphs = paragraphs[:MAX_PARAGRAPHS_PER_ISSUE]
|
||||
|
||||
issue_data = {
|
||||
"book": "btg-1978",
|
||||
"volume": vol,
|
||||
"part": issue_num,
|
||||
"title": display_title,
|
||||
"slug": slug,
|
||||
"year": year,
|
||||
"articles": articles,
|
||||
"paragraphs": paragraphs,
|
||||
}
|
||||
issues_data.append(issue_data)
|
||||
|
||||
print(
|
||||
f" {slug}: {display_title} — "
|
||||
f"{len(articles)} articles, {len(paragraphs)} paragraphs"
|
||||
)
|
||||
|
||||
print(f"\n Skipped {skipped_ads} advertisement sections")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 3: Build intro.json with hierarchical groups
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Group issues by era/volume
|
||||
# Early issues (1-17) have Vol 1/Vol 2 designations
|
||||
# Later issues (30+) just have issue numbers
|
||||
group_early = [] # Issues 1-17 (1966-1968)
|
||||
group_later = [] # Issues 30+ (1969-1973)
|
||||
|
||||
for issue in issues_data:
|
||||
if issue["part"] <= 17:
|
||||
group_early.append(issue)
|
||||
else:
|
||||
group_later.append(issue)
|
||||
|
||||
chapters = []
|
||||
|
||||
if group_early:
|
||||
sub = []
|
||||
for issue in group_early:
|
||||
sub.append({"slug": issue["slug"], "title": issue["title"]})
|
||||
chapters.append(
|
||||
{
|
||||
"title": "Issues #1–17 (1966–1968)",
|
||||
"subChapters": sub,
|
||||
}
|
||||
)
|
||||
|
||||
if group_later:
|
||||
sub = []
|
||||
for issue in group_later:
|
||||
sub.append({"slug": issue["slug"], "title": issue["title"]})
|
||||
chapters.append(
|
||||
{
|
||||
"title": "Issues #30–54 (1969–1973)",
|
||||
"subChapters": sub,
|
||||
}
|
||||
)
|
||||
|
||||
total_issues = len(issues_data)
|
||||
total_articles = sum(len(i["articles"]) for i in issues_data)
|
||||
|
||||
# Determine year range
|
||||
years = [i["year"] for i in issues_data if i["year"]]
|
||||
min_year = min(years) if years else None
|
||||
max_year = max(years) if years else None
|
||||
|
||||
intro_json = {
|
||||
"book": "btg-1978",
|
||||
"title": "Back to Godhead (1966–1973)",
|
||||
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
||||
"year": min_year,
|
||||
"description": (
|
||||
f"Back to Godhead magazine issues from the Western era ({min_year}–{max_year}). "
|
||||
f"{total_issues} issues containing {total_articles} articles — lectures, essays, "
|
||||
f"poems, and philosophical discussions on Kṛṣṇa consciousness."
|
||||
),
|
||||
"totalIssues": total_issues,
|
||||
"totalArticles": total_articles,
|
||||
"flag": "Pre 1978",
|
||||
"chapters": chapters,
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 4: Write JSON files
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Clean output directory
|
||||
if os.path.exists(issues_dir):
|
||||
for f in os.listdir(issues_dir):
|
||||
os.remove(os.path.join(issues_dir, f))
|
||||
|
||||
# Write intro.json
|
||||
write_json(intro_json, os.path.join(output_dir, "intro.json"))
|
||||
print(f"\n Written: intro.json ({total_issues} issues, {total_articles} articles)")
|
||||
|
||||
# Write issue files
|
||||
order = 100
|
||||
for issue in issues_data:
|
||||
issue_json = {
|
||||
"book": "btg-1978",
|
||||
"volume": issue["volume"],
|
||||
"part": issue["part"],
|
||||
"title": issue["title"],
|
||||
"slug": issue["slug"],
|
||||
"order": order,
|
||||
"articles": issue["articles"],
|
||||
"paragraphs": issue["paragraphs"],
|
||||
}
|
||||
filepath = os.path.join(issues_dir, f"{issue['slug']}.json")
|
||||
write_json(issue_json, filepath)
|
||||
order += 1
|
||||
|
||||
print(f" Written: {len(issues_data)} issue files")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 5: Quality checks
|
||||
# ---------------------------------------------------------------------------
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f" QUALITY CHECKS")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
# Check for ScaGoudy remnants
|
||||
sca_chars = set(SCAGOUDY_TO_IAST.keys())
|
||||
sca_remnants = 0
|
||||
for issue in issues_data:
|
||||
for p in issue["paragraphs"]:
|
||||
if any(c in p for c in ("å", "ë", "ö")):
|
||||
sca_remnants += 1
|
||||
break
|
||||
if sca_remnants == 0:
|
||||
print(f" ✅ No ScaGoudy remnants detected")
|
||||
else:
|
||||
print(f" ⚠ {sca_remnants} issues may have ScaGoudy remnants")
|
||||
|
||||
# Check for empty issues
|
||||
empty_issues = [i for i in issues_data if len(i["paragraphs"]) == 0]
|
||||
if empty_issues:
|
||||
print(
|
||||
f" ⚠ {len(empty_issues)} empty issues: {[i['slug'] for i in empty_issues]}"
|
||||
)
|
||||
else:
|
||||
print(f" ✅ No empty issues")
|
||||
|
||||
# Check centered verses
|
||||
verse_count = 0
|
||||
for issue in issues_data:
|
||||
for p in issue["paragraphs"]:
|
||||
if '<div style="text-align:center">' in p:
|
||||
verse_count += 1
|
||||
print(f" ℹ️ {verse_count} center-aligned verse paragraphs")
|
||||
|
||||
# Check for remaining advertisement content
|
||||
ad_check = 0
|
||||
ad_keywords = ["advertisement", "subscribe", "subscription form"]
|
||||
for issue in issues_data:
|
||||
for p in issue["paragraphs"]:
|
||||
lower_p = p.lower()
|
||||
if any(kw in lower_p for kw in ad_keywords):
|
||||
# Only flag if the paragraph is primarily an ad
|
||||
if "advertisement" in lower_p and len(p) < 200:
|
||||
ad_check += 1
|
||||
if ad_check == 0:
|
||||
print(f" ✅ No advertisement content detected in output")
|
||||
else:
|
||||
print(f" ⚠ {ad_check} paragraphs may contain advertisement content")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Summary
|
||||
# ---------------------------------------------------------------------------
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f" EXTRACTION SUMMARY")
|
||||
print(f"{'=' * 60}")
|
||||
|
||||
if group_early:
|
||||
early_arts = sum(len(i["articles"]) for i in group_early)
|
||||
early_paras = sum(len(i["paragraphs"]) for i in group_early)
|
||||
print(
|
||||
f" Issues #1–17 (1966–1968): {len(group_early)} issues, "
|
||||
f"{early_arts} articles, {early_paras} paragraphs"
|
||||
)
|
||||
for issue in group_early:
|
||||
print(
|
||||
f" {issue['slug']}: {issue['title']} "
|
||||
f"({len(issue['articles'])} articles, {len(issue['paragraphs'])} paras)"
|
||||
)
|
||||
|
||||
if group_later:
|
||||
later_arts = sum(len(i["articles"]) for i in group_later)
|
||||
later_paras = sum(len(i["paragraphs"]) for i in group_later)
|
||||
print(
|
||||
f" Issues #30–54 (1969–1973): {len(group_later)} issues, "
|
||||
f"{later_arts} articles, {later_paras} paragraphs"
|
||||
)
|
||||
for issue in group_later:
|
||||
print(
|
||||
f" {issue['slug']}: {issue['title']} "
|
||||
f"({len(issue['articles'])} articles, {len(issue['paragraphs'])} paras)"
|
||||
)
|
||||
|
||||
print(f"\n Total: {total_issues} issues, {total_articles} articles")
|
||||
print(f" Skipped: {skipped_ads} advertisement sections")
|
||||
print(f"{'=' * 60}")
|
||||
print(f"\nDone! ✨")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user