[\s]*)+$", "", html)
return html
def write_json(data: dict, path: str):
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w", encoding="utf-8") as f:
json.dump(data, f, indent=2, ensure_ascii=False)
f.write("\n")
# ---------------------------------------------------------------------------
# Parse issue info from p2 text
# ---------------------------------------------------------------------------
def parse_issue_info(text: str):
"""Parse issue info from the p2 bookmark text.
Handles various formats:
"Back to Godhead Magazine. Vol. 1, No. 01, October 23, 1966"
"Back to Godhead Magazine. Vol. 1, No. 06, January 20, 1967"
"Back to Godhead Magazine. [VOL II No 1.] No. 16, 1968"
"Back to Godhead Magazine VOL II. No 2. [#17], 1968"
"Back to Godhead Magazine no. 30, 1969"
"Back to Godhead Magazine No. 36, 1970"
Returns (issue_number, volume, display_title, date_str, slug) or None.
"""
text = text.strip()
# Remove "Back to Godhead Magazine" prefix variations
text_clean = re.sub(
r"^Back to Godhead Magazine\.?\s*", "", text, flags=re.IGNORECASE
).strip()
issue_num = None
volume = None
date_str = ""
# Try to extract issue number from various patterns
# Pattern 1: "Vol. X, No. YY, date" or "Vol. X No. YY, date"
m = re.search(
r"Vol\.?\s*(\d+)\.?\s*,?\s*No\.?\s*(\d+)\s*,?\s*(.*)", text_clean, re.IGNORECASE
)
if m:
volume = int(m.group(1))
issue_num = int(m.group(2))
date_str = m.group(3).strip().rstrip(",. ")
# Pattern 2: "[VOL II No 1.] No. 16, 1968" — use the absolute number
if issue_num is None:
m = re.search(r"No\.?\s*(\d+)\s*,?\s*(\d{4})", text_clean, re.IGNORECASE)
if m:
issue_num = int(m.group(1))
date_str = m.group(2).strip()
# Pattern 3: "VOL II. No 2. [#17], 1968"
if issue_num is None:
m = re.search(r"#(\d+)", text_clean)
if m:
issue_num = int(m.group(1))
# Pattern 4: "no. 30, 1969" or "no 31, 1969"
if issue_num is None:
m = re.search(r"no\.?\s*(\d+)\s*,?\s*(.*)", text_clean, re.IGNORECASE)
if m:
issue_num = int(m.group(1))
date_str = m.group(2).strip().rstrip(",. ")
# Try to extract volume from text if not found yet
if volume is None:
m = re.search(r"VOL\.?\s*(?:II|2)", text_clean, re.IGNORECASE)
if m:
volume = 2
else:
m = re.search(r"VOL\.?\s*(?:I|1)", text_clean, re.IGNORECASE)
if m:
volume = 1
if issue_num is None:
print(f" ⚠ Could not parse issue number from: {text}")
return None
# Extract year from date_str if present
year_match = re.search(r"(\d{4})", date_str)
year = int(year_match.group(1)) if year_match else None
# Build display title
if volume and issue_num <= 17:
display_title = f"Vol. {volume}, No. {issue_num:02d}"
if date_str:
display_title += f", {date_str}"
else:
display_title = f"No. {issue_num}"
if date_str:
display_title += f", {date_str}"
slug = f"no-{issue_num:02d}"
return issue_num, volume, display_title, date_str, year, slug
# ---------------------------------------------------------------------------
# Main extraction
# ---------------------------------------------------------------------------
def main():
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
html_path = os.path.join(base_dir, "public", "html", "btg-1978.html")
output_dir = os.path.join(base_dir, "lib", "jsons", "magazines", "btg-1978")
issues_dir = os.path.join(output_dir, "issues")
if not os.path.exists(html_path):
print(f"ERROR: {html_path} not found!")
sys.exit(1)
print(f"Reading {html_path}...")
with open(html_path, "r", encoding="utf-8") as f:
content = f.read()
soup = BeautifulSoup(content, "html.parser")
all_tags = soup.find_all("p")
print(f" Parsed {len(all_tags)} tags total")
# ---------------------------------------------------------------------------
# Phase 1: Find all issue boundaries
# ---------------------------------------------------------------------------
issue_boundaries = [] # List of (tag_index, issue_num, volume, display_title, date_str, year, slug)
for i, tag in enumerate(all_tags):
cls = get_css_class(tag)
if cls == "p2":
text = tag.get_text().strip()
if "Back to Godhead" in text:
parsed = parse_issue_info(text)
if parsed:
issue_num, volume, display_title, date_str, year, slug = parsed
issue_boundaries.append(
(i, issue_num, volume, display_title, date_str, year, slug)
)
print(f"\n Found {len(issue_boundaries)} issues")
for ib in issue_boundaries:
tag_idx, inum, vol, title, ds, yr, slug = ib
print(f" tag[{tag_idx}]: #{inum} (Vol {vol}) {title} → {slug}")
# ---------------------------------------------------------------------------
# Phase 2: Extract articles and paragraphs for each issue
# ---------------------------------------------------------------------------
# Known skip classes (bookmarks, empty lines)
SKIP_CLASSES = {"p2", "p11", "p28", "p24"}
issues_data = []
skipped_ads = 0
# Maximum paragraphs per issue — prevents mega-issues from absorbing
# unmarked content that belongs to missing intermediate issues.
MAX_PARAGRAPHS_PER_ISSUE = 500
for idx, (
start_tag_idx,
issue_num,
vol,
display_title,
date_str,
year,
slug,
) in enumerate(issue_boundaries):
# Determine end boundary
if idx + 1 < len(issue_boundaries):
end_tag_idx = issue_boundaries[idx + 1][0]
else:
end_tag_idx = len(all_tags)
issue_tags = all_tags[start_tag_idx:end_tag_idx]
paragraphs = []
articles = []
in_advertisement = False
for tag in issue_tags:
cls = get_css_class(tag)
text = tag.get_text().strip()
# Skip bookmark tags and empty lines
if cls in SKIP_CLASSES:
continue
if not text:
continue
# p8 = Article bookmark
if cls == "p8":
article_title = decode_scagoudy(text).strip()
# Check if this is an advertisement
if re.match(r"^Advertisements?", article_title, re.IGNORECASE):
in_advertisement = True
skipped_ads += 1
continue
else:
in_advertisement = False
articles.append({"title": article_title})
continue
# p122 and other classes can also be advertisement markers
if re.match(r"^Advertisements?\s*$", text, re.IGNORECASE):
in_advertisement = True
skipped_ads += 1
continue
# p9 = Article display title
if cls == "p9":
if in_advertisement:
continue
# Also check the p9 display title itself for advertisement
plain_title = decode_scagoudy(text).strip()
if re.match(r"^Advertisements?\s*$", plain_title, re.IGNORECASE):
in_advertisement = True
skipped_ads += 1
continue
html = inner_html(tag)
if html:
# Make it a bold article heading if not already
if not html.startswith(""):
html = f"{html}"
paragraphs.append(html)
continue
# Skip content within advertisement sections
if in_advertisement:
# Check if next p8 will reset it — for now just skip
continue
# p15 = Verse block — center align
if cls == "p15":
html = inner_html(tag)
if html:
paragraphs.append(f'
{html}
')
continue
# All other classes — extract as body text
html = inner_html(tag)
if html and html != "
":
# Check if this looks like a verse (entirely italic with
breaks)
stripped = html.strip()
if (
stripped.startswith("")
and "' not in stripped
):
plain = re.sub(r"", "", stripped)
lines = [
l.strip() for l in re.split(r"", plain) if l.strip()
]
if len(lines) >= 2:
html = f'{stripped}
'
paragraphs.append(html)
# Post-process: remove any ad-like paragraphs that slipped through.
# Two-pass approach:
# 1. Detect ad section start markers and skip until next real article
# 2. Filter individual ad-like paragraphs by content patterns
# --- Pass 1: Section-level ad detection ---
ad_section_patterns = [
r"^Advertisements?\s*$",
r"^CIRCULATION AND SUBSCRIPTIONS",
r"^Order Form",
r"^SPECIAL OFFER",
r"^Subscriptions?:",
r"^WORLDWIDE IMPORTS",
r"^THE\s+ORIENT\s+SHOP",
r"^PROTECT YOUR CHECKS",
]
ad_section_re = re.compile("|".join(ad_section_patterns), re.IGNORECASE)
cleaned_pass1 = []
skip_until_next_bold = False
for p in paragraphs:
plain = re.sub(r"<[^>]+>", "", p).strip()
# Detect ad section start
if ad_section_re.search(plain):
skip_until_next_bold = True
continue
if skip_until_next_bold:
# A bold article heading that's NOT an ad resets the skip
if p.startswith("") and len(plain) < 200:
if ad_section_re.search(plain):
continue
else:
skip_until_next_bold = False
cleaned_pass1.append(p)
continue
# If it looks like real content (long paragraph), stop skipping
if len(plain) > 150:
skip_until_next_bold = False
cleaned_pass1.append(p)
continue
# Short paragraphs in ad zone — skip
continue
cleaned_pass1.append(p)
# --- Pass 2: Individual paragraph ad detection ---
# These patterns catch standalone ad fragments that weren't part of
# a marked advertisement section.
ad_para_keywords = [
"subscription",
"per copy",
"enclosed is",
"please send me",
"order form",
"postage paid",
"money order",
"check or money order",
"zip code",
"state_____",
"address_____",
"city_____",
"available from iskcon",
"waterpipes",
"djellabas",
"panchos",
"orient shop",
"worldwide imports",
"sullivan st",
"bleecker",
"check protector",
"safeguards your checks",
"satisfaction guaranteed",
"import-export",
"davis & manswell",
"davis & manswell",
"bedford ave",
"brooklyn, n.y.",
"immediate delivery",
"free gift guide",
"write for free",
"free catalog",
"free catalogue",
"protect your checks",
"name of purchaser",
"such dresses at $",
"and gowns yet $",
"price per copy",
"one-year subscription",
"year subscription costs",
"back issues available",
"a subscription to",
"send check or money",
"your name & address",
"your name & address",
"674-9511",
]
# Also detect short paragraphs that look like addresses or phone numbers
address_re = re.compile(
r"^\d+\s+\w+\s+(st|street|ave|avenue|blvd|road|rd|lane|ln|dr|drive)\b",
re.IGNORECASE,
)
phone_re = re.compile(r"^\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}\s*$")
zipcode_re = re.compile(
r"(N\.?Y\.?\s+\d{5}|CA\s+\d{5}|Brooklyn|Manhattan)",
re.IGNORECASE,
)
cleaned = []
for p in cleaned_pass1:
plain = re.sub(r"<[^>]+>", "", p).strip()
plain_lower = plain.lower()
# Skip if matches any ad keyword
if any(kw in plain_lower for kw in ad_para_keywords):
continue
# Skip short paragraphs that look like addresses
if len(plain) < 80 and address_re.search(plain):
continue
# Skip standalone phone numbers
if phone_re.match(plain):
continue
# Skip short paragraphs with zip codes (likely address fragments)
if len(plain) < 60 and zipcode_re.search(plain):
continue
# Skip price-only paragraphs like "$1 plus 25c"
if re.match(
r"^\$?\d+[\.\d]*\s*(plus|postage|each|per)", plain, re.IGNORECASE
):
continue
cleaned.append(p)
paragraphs = cleaned
# Cap paragraphs to prevent mega-issues from absorbing unmarked
# intermediate issues (e.g., issues 18-29 have no p2 markers and
# would otherwise be absorbed into issue 17).
if len(paragraphs) > MAX_PARAGRAPHS_PER_ISSUE:
print(
f" ⚠ {slug}: Capped from {len(paragraphs)} to {MAX_PARAGRAPHS_PER_ISSUE} paragraphs "
f"(likely absorbed unmarked intermediate issues)"
)
paragraphs = paragraphs[:MAX_PARAGRAPHS_PER_ISSUE]
issue_data = {
"book": "btg-1978",
"volume": vol,
"part": issue_num,
"title": display_title,
"slug": slug,
"year": year,
"articles": articles,
"paragraphs": paragraphs,
}
issues_data.append(issue_data)
print(
f" {slug}: {display_title} — "
f"{len(articles)} articles, {len(paragraphs)} paragraphs"
)
print(f"\n Skipped {skipped_ads} advertisement sections")
# ---------------------------------------------------------------------------
# Phase 3: Build intro.json with hierarchical groups
# ---------------------------------------------------------------------------
# Group issues by era/volume
# Early issues (1-17) have Vol 1/Vol 2 designations
# Later issues (30+) just have issue numbers
group_early = [] # Issues 1-17 (1966-1968)
group_later = [] # Issues 30+ (1969-1973)
for issue in issues_data:
if issue["part"] <= 17:
group_early.append(issue)
else:
group_later.append(issue)
chapters = []
if group_early:
sub = []
for issue in group_early:
sub.append({"slug": issue["slug"], "title": issue["title"]})
chapters.append(
{
"title": "Issues #1–17 (1966–1968)",
"subChapters": sub,
}
)
if group_later:
sub = []
for issue in group_later:
sub.append({"slug": issue["slug"], "title": issue["title"]})
chapters.append(
{
"title": "Issues #30–54 (1969–1973)",
"subChapters": sub,
}
)
total_issues = len(issues_data)
total_articles = sum(len(i["articles"]) for i in issues_data)
# Determine year range
years = [i["year"] for i in issues_data if i["year"]]
min_year = min(years) if years else None
max_year = max(years) if years else None
intro_json = {
"book": "btg-1978",
"title": "Back to Godhead (1966–1973)",
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
"year": min_year,
"description": (
f"Back to Godhead magazine issues from the Western era ({min_year}–{max_year}). "
f"{total_issues} issues containing {total_articles} articles — lectures, essays, "
f"poems, and philosophical discussions on Kṛṣṇa consciousness."
),
"totalIssues": total_issues,
"totalArticles": total_articles,
"flag": "Pre 1978",
"chapters": chapters,
}
# ---------------------------------------------------------------------------
# Phase 4: Write JSON files
# ---------------------------------------------------------------------------
# Clean output directory
if os.path.exists(issues_dir):
for f in os.listdir(issues_dir):
os.remove(os.path.join(issues_dir, f))
# Write intro.json
write_json(intro_json, os.path.join(output_dir, "intro.json"))
print(f"\n Written: intro.json ({total_issues} issues, {total_articles} articles)")
# Write issue files
order = 100
for issue in issues_data:
issue_json = {
"book": "btg-1978",
"volume": issue["volume"],
"part": issue["part"],
"title": issue["title"],
"slug": issue["slug"],
"order": order,
"articles": issue["articles"],
"paragraphs": issue["paragraphs"],
}
filepath = os.path.join(issues_dir, f"{issue['slug']}.json")
write_json(issue_json, filepath)
order += 1
print(f" Written: {len(issues_data)} issue files")
# ---------------------------------------------------------------------------
# Phase 5: Quality checks
# ---------------------------------------------------------------------------
print(f"\n{'=' * 60}")
print(f" QUALITY CHECKS")
print(f"{'=' * 60}")
# Check for ScaGoudy remnants
sca_chars = set(SCAGOUDY_TO_IAST.keys())
sca_remnants = 0
for issue in issues_data:
for p in issue["paragraphs"]:
if any(c in p for c in ("å", "ë", "ö")):
sca_remnants += 1
break
if sca_remnants == 0:
print(f" ✅ No ScaGoudy remnants detected")
else:
print(f" ⚠ {sca_remnants} issues may have ScaGoudy remnants")
# Check for empty issues
empty_issues = [i for i in issues_data if len(i["paragraphs"]) == 0]
if empty_issues:
print(
f" ⚠ {len(empty_issues)} empty issues: {[i['slug'] for i in empty_issues]}"
)
else:
print(f" ✅ No empty issues")
# Check centered verses
verse_count = 0
for issue in issues_data:
for p in issue["paragraphs"]:
if '' in p:
verse_count += 1
print(f" ℹ️ {verse_count} center-aligned verse paragraphs")
# Check for remaining advertisement content
ad_check = 0
ad_keywords = ["advertisement", "subscribe", "subscription form"]
for issue in issues_data:
for p in issue["paragraphs"]:
lower_p = p.lower()
if any(kw in lower_p for kw in ad_keywords):
# Only flag if the paragraph is primarily an ad
if "advertisement" in lower_p and len(p) < 200:
ad_check += 1
if ad_check == 0:
print(f" ✅ No advertisement content detected in output")
else:
print(f" ⚠ {ad_check} paragraphs may contain advertisement content")
# ---------------------------------------------------------------------------
# Summary
# ---------------------------------------------------------------------------
print(f"\n{'=' * 60}")
print(f" EXTRACTION SUMMARY")
print(f"{'=' * 60}")
if group_early:
early_arts = sum(len(i["articles"]) for i in group_early)
early_paras = sum(len(i["paragraphs"]) for i in group_early)
print(
f" Issues #1–17 (1966–1968): {len(group_early)} issues, "
f"{early_arts} articles, {early_paras} paragraphs"
)
for issue in group_early:
print(
f" {issue['slug']}: {issue['title']} "
f"({len(issue['articles'])} articles, {len(issue['paragraphs'])} paras)"
)
if group_later:
later_arts = sum(len(i["articles"]) for i in group_later)
later_paras = sum(len(i["paragraphs"]) for i in group_later)
print(
f" Issues #30–54 (1969–1973): {len(group_later)} issues, "
f"{later_arts} articles, {later_paras} paragraphs"
)
for issue in group_later:
print(
f" {issue['slug']}: {issue['title']} "
f"({len(issue['articles'])} articles, {len(issue['paragraphs'])} paras)"
)
print(f"\n Total: {total_issues} issues, {total_articles} articles")
print(f" Skipped: {skipped_ads} advertisement sections")
print(f"{'=' * 60}")
print(f"\nDone! ✨")
if __name__ == "__main__":
main()