[\s]*)+$", "", html)
return html
def is_verse_paragraph(tag: Tag) -> bool:
"""Check if a paragraph is a verse block (italic with line breaks)."""
cls = get_css_class(tag)
# p15 is commonly used for verse blocks
if cls == "p15":
return True
# Also check for paragraphs that are entirely italic with
breaks
# (some verses use p14 or other classes but are entirely in tags)
return False
def format_verse(tag: Tag) -> str:
"""Format a verse paragraph as a center-aligned div."""
html = inner_html(tag)
if not html:
return ""
# Wrap in center-aligned div
return f'{html}
'
def write_json(data: dict, path: str):
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w", encoding="utf-8") as f:
json.dump(data, f, indent=2, ensure_ascii=False)
f.write("\n")
# ---------------------------------------------------------------------------
# Parse volume/part info from p2 bookmark text
# ---------------------------------------------------------------------------
def parse_volume_part(text: str):
"""Parse 'BTGPY1: Volume 1, Part 1, 1944' → (volume, part, date_str, slug).
Returns (volume_num, part_num, date_string, slug) or None.
"""
# Remove the BTGPY prefix
text = text.strip()
# Match "Volume X, Part Y, ..."
m = re.search(r"Volume\s+(\d+),?\s*Part\s+(\d+),?\s*(.*)", text, re.IGNORECASE)
if m:
vol = int(m.group(1))
part = int(m.group(2))
date_str = m.group(3).strip().rstrip(" ")
slug = f"v{vol}-p{part}"
return vol, part, date_str, slug
return None
# ---------------------------------------------------------------------------
# Main extraction
# ---------------------------------------------------------------------------
def main():
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
html_path = os.path.join(base_dir, "public", "html", "btg-1960.html")
output_dir = os.path.join(base_dir, "lib", "jsons", "magazines", "btg")
issues_dir = os.path.join(output_dir, "issues")
if not os.path.exists(html_path):
print(f"ERROR: {html_path} not found!")
sys.exit(1)
print(f"Reading {html_path}...")
with open(html_path, "r", encoding="utf-8") as f:
content = f.read()
soup = BeautifulSoup(content, "html.parser")
all_tags = soup.find_all("p")
print(f" Parsed {len(all_tags)} tags total")
# ---------------------------------------------------------------------------
# Phase 1: Find all issue boundaries
# ---------------------------------------------------------------------------
# Each issue starts with a p2 bookmark tag
issue_boundaries = [] # List of (tag_index, volume, part, date_str, slug)
for i, tag in enumerate(all_tags):
cls = get_css_class(tag)
if cls == "p2":
text = tag.get_text().strip()
parsed = parse_volume_part(text)
if parsed:
vol, part, date_str, slug = parsed
issue_boundaries.append((i, vol, part, date_str, slug))
print(f"\n Found {len(issue_boundaries)} issues")
for ib in issue_boundaries:
print(f" tag[{ib[0]}]: Vol {ib[1]}, Part {ib[2]}, {ib[3]} → {ib[4]}")
# ---------------------------------------------------------------------------
# Phase 2: Extract articles and paragraphs for each issue
# ---------------------------------------------------------------------------
# Classes to skip entirely
SKIP_CLASSES = {"p2", "p11", "p24", "p28"}
# Classes that are header/masthead material (include as paragraphs)
MASTHEAD_CLASSES = {"p3", "p4", "p5", "p6", "p7", "p32", "p33", "p34", "p40", "p47"}
# Classes that are body text
BODY_CLASSES = {
"p10",
"p12",
"p13",
"p14",
"p15",
"p16",
"p17",
"p35",
"p36",
"p37",
"p38",
"p39",
"p41",
"p42",
"p43",
"p44",
"p45",
"p46",
"p48",
"p49",
"p50",
"p51",
"p52",
"p53",
"p54",
"p55",
"p56",
"p57",
"p58",
"p59",
"p60",
"p61",
"p62",
"p63",
"p64",
"p65",
"p66",
"p67",
"p68",
"p69",
"p70",
"p71",
"p72",
"p73",
"p74",
"p75",
"p76",
"p77",
"p78",
"p79",
"p80",
"p81",
"p82",
"p83",
"p84",
"p85",
"p86",
"p87",
"p88",
"p89",
"p90",
"p91",
"p92",
"p93",
"p94",
"p95",
"p96",
"p97",
"p98",
"p99",
}
issues_data = []
for idx, (start_tag_idx, vol, part, date_str, slug) in enumerate(issue_boundaries):
# Determine end boundary
if idx + 1 < len(issue_boundaries):
end_tag_idx = issue_boundaries[idx + 1][0]
else:
end_tag_idx = len(all_tags)
issue_tags = all_tags[start_tag_idx:end_tag_idx]
paragraphs = []
articles = [] # List of { title, slug }
for tag in issue_tags:
cls = get_css_class(tag)
text = tag.get_text().strip()
# Skip bookmark tags and empty lines
if cls in SKIP_CLASSES:
continue
if not text and cls not in ("p15",):
continue
# p8 = Article bookmark — extract title for metadata
if cls == "p8":
# Extract article title from bookmark text (remove BTGPYxx: prefix)
article_text = text
m = re.match(r"BTGPY\d+[a-z]?:\s*(.*)", article_text)
if m:
article_title = m.group(1).strip()
else:
article_title = article_text
article_title = decode_scagoudy(article_title)
articles.append({"title": article_title})
continue
# p9 = Article display title — render as bold heading
if cls == "p9":
html = inner_html(tag)
if html:
# Make it a bold article heading
if not html.startswith(""):
html = f"{html}"
paragraphs.append(html)
continue
# p15 = Verse block — center align
if cls == "p15":
html = format_verse(tag)
if html:
paragraphs.append(html)
continue
# p3 = Volume title — skip (it's the display title of the issue header)
if cls == "p3":
continue
# Masthead classes — include but as regular paragraphs
if cls in MASTHEAD_CLASSES:
html = inner_html(tag)
if html and html != "
":
paragraphs.append(html)
continue
# Body text classes
html = inner_html(tag)
if html and html != "
":
# Check if this looks like a verse (entirely italic with
breaks)
stripped = html.strip()
if (
stripped.startswith("")
and "' in stripped
):
# Check if it's predominantly a verse (italic block with line breaks)
# by seeing if removing the tags leaves mostly text with
plain = re.sub(r"", "", stripped)
lines = [
l.strip() for l in re.split(r"", plain) if l.strip()
]
if len(lines) >= 2:
html = f'{stripped}
'
paragraphs.append(html)
# Build display title
if date_str:
display_title = f"Volume {vol}, Part {part}, {date_str}"
else:
display_title = f"Volume {vol}, Part {part}"
issue_data = {
"book": "btg",
"volume": vol,
"part": part,
"title": display_title,
"slug": slug,
"articles": articles,
"paragraphs": paragraphs,
}
issues_data.append(issue_data)
print(
f" {slug}: {display_title} — "
f"{len(articles)} articles, {len(paragraphs)} paragraphs"
)
# ---------------------------------------------------------------------------
# Phase 3: Build intro.json with hierarchical volume groups
# ---------------------------------------------------------------------------
# Group issues by volume
volume_groups = {}
for issue in issues_data:
vol = issue["volume"]
if vol not in volume_groups:
volume_groups[vol] = []
volume_groups[vol].append(issue)
# Build chapters array for intro.json
chapters = []
for vol_num in sorted(volume_groups.keys()):
vol_issues = volume_groups[vol_num]
sub_chapters = []
for issue in vol_issues:
sub_chapters.append(
{
"slug": issue["slug"],
"title": issue["title"],
}
)
chapters.append(
{
"title": f"Volume {vol_num}",
"subChapters": sub_chapters,
}
)
total_issues = len(issues_data)
total_articles = sum(len(i["articles"]) for i in issues_data)
intro_json = {
"book": "btg",
"title": "Back to Godhead",
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
"year": 1944,
"description": f"The original Back to Godhead magazine founded in 1944. {total_issues} issues across {len(volume_groups)} volumes, containing {total_articles} articles on Kṛṣṇa consciousness, Vedic philosophy, and spiritual life.",
"totalIssues": total_issues,
"totalArticles": total_articles,
"chapters": chapters,
}
# ---------------------------------------------------------------------------
# Phase 4: Write JSON files
# ---------------------------------------------------------------------------
# Clean output directory
if os.path.exists(issues_dir):
for f in os.listdir(issues_dir):
os.remove(os.path.join(issues_dir, f))
# Write intro.json
write_json(intro_json, os.path.join(output_dir, "intro.json"))
print(f"\n Written: intro.json ({total_issues} issues, {total_articles} articles)")
# Write issue files
order = 100
for issue in issues_data:
issue_json = {
"book": "btg",
"volume": issue["volume"],
"part": issue["part"],
"title": issue["title"],
"slug": issue["slug"],
"order": order,
"articles": issue["articles"],
"paragraphs": issue["paragraphs"],
}
filepath = os.path.join(issues_dir, f"{issue['slug']}.json")
write_json(issue_json, filepath)
order += 1
print(f" Written: {len(issues_data)} issue files")
# ---------------------------------------------------------------------------
# Phase 5: Quality checks
# ---------------------------------------------------------------------------
print(f"\n{'=' * 60}")
print(f" QUALITY CHECKS")
print(f"{'=' * 60}")
# Check for ScaGoudy remnants
sca_chars = set(SCAGOUDY_TO_IAST.keys())
sca_remnants = 0
for issue in issues_data:
for p in issue["paragraphs"]:
if any(c in p for c in sca_chars if c not in ("é", "ü", "à")):
# Skip common French/German chars that might be legitimate
# Focus on clearly ScaGoudy-only chars
if "å" in p or "ë" in p or "ö" in p and "Ö" not in p:
sca_remnants += 1
break
if sca_remnants == 0:
print(f" ✅ No ScaGoudy remnants detected")
else:
print(f" ⚠ {sca_remnants} issues may have ScaGoudy remnants")
# Check for empty issues
empty_issues = [i for i in issues_data if len(i["paragraphs"]) == 0]
if empty_issues:
print(
f" ⚠ {len(empty_issues)} empty issues: {[i['slug'] for i in empty_issues]}"
)
else:
print(f" ✅ No empty issues")
# Check centered verses
verse_count = 0
for issue in issues_data:
for p in issue["paragraphs"]:
if '' in p:
verse_count += 1
print(f" ℹ️ {verse_count} center-aligned verse paragraphs")
# ---------------------------------------------------------------------------
# Summary
# ---------------------------------------------------------------------------
print(f"\n{'=' * 60}")
print(f" EXTRACTION SUMMARY")
print(f"{'=' * 60}")
for vol_num in sorted(volume_groups.keys()):
vol_issues = volume_groups[vol_num]
vol_articles = sum(len(i["articles"]) for i in vol_issues)
vol_paras = sum(len(i["paragraphs"]) for i in vol_issues)
print(
f" Volume {vol_num}: {len(vol_issues)} issues, {vol_articles} articles, {vol_paras} paragraphs"
)
for issue in vol_issues:
print(
f" {issue['slug']}: {issue['title']} "
f"({len(issue['articles'])} articles, {len(issue['paragraphs'])} paras)"
)
print(f"\n Total: {total_issues} issues, {total_articles} articles")
print(f"{'=' * 60}")
print(f"\nDone! ✨")
if __name__ == "__main__":
main()