Files
spsays-training/scripts/sb1962_extract.py
T

618 lines
20 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Extract 1962 Śrīmad-Bhāgavatam First Canto verses from others-before-1977.html.
This is the original 1962-65 edition of Śrīmad-Bhāgavatam First Canto published
in India before Śrīla Prabhupāda went to the USA. It spans 3 volumes:
- Volume 1 (1962): Chapters 1–7 (first half)
- Volume 2 (1964): Chapter 7 (second half) through Chapter 12
- Volume 3 (1965): Chapters 13–19
Key differences from the standard SB extraction:
- No Devanagari or ScaGoudy encoding — transliteration is plain English (non-IAST)
- Different CSS class mappings
- Verse markers use "TEXT No. N" format (p18 class)
- Reference markers (p17) are sparse — only some verses have them
- Chapter boundaries use spelled-out ordinals: "FIRST CHAPTER", "SECOND CHAPTER", etc.
- Multi-verse entries use "TEXTS No.'s 3 AND 4" format
Output goes to app/lib/books/sb1962/
"""
import json
import os
import re
import sys
from bs4 import BeautifulSoup, Tag
# ---------------------------------------------------------------------------
# Chapter name → number mapping
# ---------------------------------------------------------------------------
ORDINAL_TO_NUM = {
"first": 1,
"second": 2,
"third": 3,
"fourth": 4,
"fifth": 5,
"sixth": 6,
"seventh": 7,
"eighth": 8,
"nineth": 9, # sic — typo in the original
"ninth": 9,
"tenth": 10,
"eleventh": 11,
"twelfth": 12,
"thirteenth": 13,
"fourteenth": 14,
"fifteenth": 15,
"sixteenth": 16,
"seventeenth": 17,
"eighteenth": 18,
"nineteenth": 19,
}
# Chapter titles (from the "Thus end" markers and chapter headers)
CHAPTER_TITLES = {
1: "Questions by the Sages",
2: "Divinity and Devotional Service",
3: "Description of Incarnations of Godhead",
4: "Appearance of Sri Narada",
5: "Sri Narada's Instructions on Srimad Bhagwatam for Vyasdeva",
6: "Conversation Between Narada and Vyasa",
7: "The Son of Drona Punished",
8: "Prayers of Queen Kunti and Parikshit Saved",
9: "Passing Away of Bhismadeva in the Presence of Lord Krishna",
10: "Departure of Lord Krishna for Dwarka",
11: "Lord Krishna's Entrance at Dwarka",
12: "Birth of Maharaj Parikshit",
13: "Dhritarastra Quits Home",
14: "Disappearance of Lord Krishna",
15: "The Pandavas Retire Timely",
16: "How Parikshit Received The Age of Kali",
17: "Punishment and Reward of Kali",
18: "Maharaj Parikshit Cursed by a Brahmin Boy",
19: "Appearance of Sri Sukdeva Goswami",
}
# ---------------------------------------------------------------------------
# HTML helpers
# ---------------------------------------------------------------------------
def get_css_class(tag: Tag) -> str:
"""Get the primary CSS class of a tag."""
classes = tag.get("class", [])
return classes[0] if classes else ""
def decode_html_content(tag: Tag) -> str:
"""Extract inner HTML preserving <i>/<b> formatting."""
inner = "".join(str(child) for child in tag.children)
# Remove span tags with CSS classes (RTF conversion artifacts)
inner = re.sub(r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner)
inner = re.sub(r"\s+", " ", inner).strip()
return inner
def decode_text(tag: Tag) -> str:
"""Extract plain text from a tag."""
return tag.get_text().strip()
# ---------------------------------------------------------------------------
# Synonym parser
# ---------------------------------------------------------------------------
def parse_synonyms(text: str) -> list:
"""Parse synonym text from the 1962 edition.
Format is similar to later editions but uses plain English:
"Om—Oh my Lord, namo—offering my obeisances, ..."
Words may be wrapped in <i> tags.
"""
# Strip HTML tags for parsing
plain = re.sub(r"<[^>]+>", "", text).strip()
if not plain:
return []
result = []
# Split on comma that's followed by an italic word (the next synonym)
# But also handle semicolons
# The pattern is: word—meaning, word—meaning, ...
# Split on ", " followed by what looks like a new word entry
# Actually simpler: split on the em-dash pattern
entries = re.split(r",\s*(?=[A-Z])", plain)
for entry in entries:
entry = entry.strip()
if not entry:
continue
if "—" in entry:
parts = entry.split("—", 1)
elif "–" in entry:
parts = entry.split("–", 1)
elif "- " in entry:
# Some entries use hyphen-space
parts = entry.split("- ", 1)
else:
continue
if len(parts) == 2:
word = parts[0].strip().strip("., ;:")
meaning = parts[1].strip().rstrip(".,")
if word and meaning:
result.append({"word": word, "meanings": [meaning]})
return result
# ---------------------------------------------------------------------------
# CSS class sets for the 1962 SB
# ---------------------------------------------------------------------------
# Verse reference bookmark (sparse — not all verses have these)
VERSE_REF_CLASS = "p17"
# TEXT No. N header
TEXT_HEADER_CLASS = "p18"
# "TEXT" label
TEXT_LABEL_CLASS = "p2"
# Transliteration pāda lines (italic, centered)
TRANSLITERATION_CLASSES = frozenset(["p19", "p87"])
# Section header class: "ENGLISH SYNONYMS", "TRANSLATION", "PURPORT"
SECTION_HEADER_CLASSES = frozenset(["p20", "p8"])
# Synonyms text class
SYNONYM_CLASSES = frozenset(["p88"])
# Translation text class
TRANSLATION_CLASSES = frozenset(["p89"])
# Purport paragraph classes
PURPORT_CLASSES = frozenset(["p74", "p6", "p48"])
# Chapter heading class
CHAPTER_HEADING_CLASS = "p3"
# Chapter bookmark class
CHAPTER_BOOKMARK_CLASS = "p11"
# Volume/section boundary classes — skip
SKIP_CLASSES = frozenset(
[
"p35", # volume title
"p8", # various front matter and banners (also used for some headers)
"p73", # "First Published in 1962"
"p105", # chapter subtitle in volume 2
"p106", # chapter subtitle in volume 3
"p71", # review quotes
"p72", # review quotes
]
)
# Line range in the HTML for the SB 1962 content
# From the first chapter marker to "END OF THE FIRST CANTO"
SB1962_START_LINE = 4726
SB1962_END_LINE = 15162
# ---------------------------------------------------------------------------
# Verse number parser
# ---------------------------------------------------------------------------
def parse_text_header(text: str):
"""Parse a TEXT header like 'TEXT No. 1' or 'TEXTS No.'s 3 AND 4'.
Returns (verse_num, verse_end) or None.
"""
text = text.strip()
# Multi-verse: "TEXTS No.'s 3 AND 4" or "TEXTS Nos. 26 AND 27"
m = re.match(
r"TEXTS?\s+No\.?'?s?\s*(\d+)\s+AND\s+(\d+)",
text,
re.IGNORECASE,
)
if m:
return (int(m.group(1)), int(m.group(2)))
# Single verse: "TEXT No. 1" or "TEXT No. 1."
m = re.match(r"TEXT\s+No\.?\s*(\d+)", text, re.IGNORECASE)
if m:
return (int(m.group(1)), None)
return None
def parse_chapter_heading(text: str):
"""Parse a chapter heading like 'FIRST CHAPTER' or 'SEVENTH CHAPTER'.
Returns chapter number or None.
"""
text = text.strip().upper()
# Remove bold/italic artifacts
text = re.sub(r"<[^>]+>", "", text).strip()
m = re.match(r"(\w+)\s+CHAPTER", text)
if m:
ordinal = m.group(1).lower()
return ORDINAL_TO_NUM.get(ordinal)
return None
def parse_ref_marker(text: str):
"""Parse a reference marker like 'SB. 1-1962:1.1.1' or 'SB. 1.13-1965:1.13.3-4'.
Returns (chapter, verse, verse_end) or None.
"""
# Match patterns like: SB. 1-1962:1.1.1 or SB. 1.7-1964:1.7.9 or SB. 1.13-1965:1.13.3-4
m = re.search(r"SB\.\s*[\d.]+-\d{4}:1\.(\d+)\.(\d+)(?:-(\d+))?", text)
if m:
chapter = int(m.group(1))
verse = int(m.group(2))
verse_end = int(m.group(3)) if m.group(3) else None
return (chapter, verse, verse_end)
return None
# ---------------------------------------------------------------------------
# State machine extraction
# ---------------------------------------------------------------------------
def _finalize_verse(verse: dict):
"""Clean up a verse before appending to results."""
# Join transliteration lines
padas = verse.pop("_padas", [])
verse["transliteration"] = "\n".join(padas)
# Clean up internal state
verse.pop("_state", None)
def extract_verses(soup: BeautifulSoup):
"""Extract all SB 1962 verses from the HTML.
Returns (verses, chapter_count).
"""
all_paragraphs = soup.find_all("p")
# Filter to only the paragraphs in the SB 1962 range
# We use the sourceline attribute if available, otherwise fall back to
# scanning for the start/end markers
paragraphs = []
in_range = False
for p in all_paragraphs:
cls = get_css_class(p)
text = decode_text(p)
# Start: first chapter marker
if cls == CHAPTER_BOOKMARK_CLASS and "SB. 1-1962:: First Chapter" in text:
in_range = True
if in_range:
paragraphs.append(p)
# End: "END OF THE FIRST CANTO"
if in_range and "END OF THE FIRST CANTO" in text:
break
if not paragraphs:
print("ERROR: Could not find SB 1962 content in the HTML!")
return [], 0
print(f" Found {len(paragraphs)} paragraphs in SB 1962 range")
verses = []
current_verse = None
state = "IDLE"
current_chapter = 0
chapters_seen = set()
for p in paragraphs:
cls = get_css_class(p)
text = decode_text(p)
# --- Chapter heading (p3): "FIRST CHAPTER", "SECOND CHAPTER", etc. ---
if cls == CHAPTER_HEADING_CLASS:
ch_num = parse_chapter_heading(text)
if ch_num is not None:
# Finalize previous verse if any
if current_verse:
_finalize_verse(current_verse)
verses.append(current_verse)
current_verse = None
state = "IDLE"
current_chapter = ch_num
chapters_seen.add(ch_num)
continue
# Skip non-chapter headings (APPENDIX, CONTENTS, PREFACE, etc.)
if any(
kw in text.upper()
for kw in ["APPENDIX", "CONTENTS", "PREFACE", "ERRATA"]
):
# We've left the chapter content — stop processing until next chapter
if current_verse:
_finalize_verse(current_verse)
verses.append(current_verse)
current_verse = None
state = "SKIP"
continue
# --- Skip mode (appendix, front matter between volumes) ---
if state == "SKIP":
# Resume when we hit a new chapter heading
if cls == CHAPTER_HEADING_CLASS:
ch_num = parse_chapter_heading(text)
if ch_num is not None:
current_chapter = ch_num
chapters_seen.add(ch_num)
state = "IDLE"
continue
# Also resume on a verse ref marker for the next volume
if cls == VERSE_REF_CLASS and "SB." in text:
ref = parse_ref_marker(text)
if ref:
state = "IDLE"
# Fall through to handle the ref below
else:
continue
else:
continue
# --- Chapter bookmark (p11) — might give us the chapter number ---
if cls == CHAPTER_BOOKMARK_CLASS:
# Could be "SB. 1-1962:: First Chapter" or "SB. 1.7-1964: ..."
# We already handle chapter number from the p3 heading
continue
# --- Volume/section boundaries — skip ---
if cls in SKIP_CLASSES:
continue
# --- Verse reference marker (p17) ---
if cls == VERSE_REF_CLASS:
ref = parse_ref_marker(text)
if ref:
# This gives us chapter/verse info, but the TEXT header is the
# primary verse trigger. Just note the chapter if different.
ch, vs, vs_end = ref
if ch != current_chapter and ch > 0:
current_chapter = ch
chapters_seen.add(ch)
continue
# --- TEXT No. N header (p18) — primary verse trigger ---
if cls == TEXT_HEADER_CLASS:
parsed = parse_text_header(text)
if parsed is None:
continue
verse_num, verse_end = parsed
# Finalize previous verse
if current_verse:
_finalize_verse(current_verse)
verses.append(current_verse)
if verse_end:
ref_str = f"SB 1.{current_chapter}.{verse_num}-{verse_end}"
else:
ref_str = f"SB 1.{current_chapter}.{verse_num}"
current_verse = {
"book_title": "Śrīmad-Bhāgavatam (1962 Edition)",
"chapter": current_chapter,
"verse": verse_num,
"reference": ref_str,
"transliteration": "",
"_padas": [],
"synonyms": [],
"translation": "",
"purport": [],
"_state": "VERSE_START",
}
if verse_end is not None:
current_verse["verse_end"] = verse_end
state = "VERSE_START"
continue
# --- TEXT label (p2) — just confirms we're in a verse ---
if cls == TEXT_LABEL_CLASS and current_verse:
continue
# --- Transliteration lines (p19, p87) ---
if (
cls in TRANSLITERATION_CLASSES
and current_verse
and state in ("VERSE_START", "TRANSLITERATION")
):
state = "TRANSLITERATION"
current_verse["_state"] = state
line = decode_text(p)
if line:
current_verse["_padas"].append(line)
continue
# --- Section headers (p20, p8): ENGLISH SYNONYMS, TRANSLATION, PURPORT ---
if cls in SECTION_HEADER_CLASSES and current_verse:
header_text = text.strip().upper()
if "SYNONYM" in header_text:
state = "SYNONYMS"
current_verse["_state"] = state
elif "TRANSLATION" in header_text:
state = "TRANSLATION"
current_verse["_state"] = state
elif "PURPORT" in header_text:
state = "PURPORT"
current_verse["_state"] = state
continue
# --- Synonyms text (p88) ---
if cls in SYNONYM_CLASSES and current_verse and state == "SYNONYMS":
html_content = decode_html_content(p)
current_verse["synonyms"] = parse_synonyms(html_content)
continue
# --- Translation text (p89) ---
if cls in TRANSLATION_CLASSES and current_verse and state == "TRANSLATION":
current_verse["translation"] = decode_html_content(p)
state = "POST_TRANSLATION"
current_verse["_state"] = state
continue
# --- Handle mislabeled headers (TRANSLATION header followed by
# translation in p89 while state is PURPORT) ---
if (
cls in TRANSLATION_CLASSES
and current_verse
and not current_verse.get("translation")
):
current_verse["translation"] = decode_html_content(p)
state = "POST_TRANSLATION"
current_verse["_state"] = state
continue
# --- Purport paragraphs (p74, p6, p48) ---
if state == "PURPORT" and current_verse and cls in PURPORT_CLASSES:
content = decode_html_content(p)
if content and "Thus end" not in text and "Thus ended" not in text:
current_verse["purport"].append(content)
continue
# --- Chapter end marker ("Thus end the Bhaktivedanta Purports") ---
if ("Thus end" in text or "Thus ended" in text) and current_verse:
_finalize_verse(current_verse)
verses.append(current_verse)
current_verse = None
state = "IDLE"
continue
# Don't forget the last verse
if current_verse:
_finalize_verse(current_verse)
verses.append(current_verse)
return verses, len(chapters_seen)
# ---------------------------------------------------------------------------
# JSON output
# ---------------------------------------------------------------------------
def write_json(data, filepath):
"""Write JSON with proper Unicode and formatting."""
os.makedirs(os.path.dirname(filepath), exist_ok=True)
with open(filepath, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False, indent=2)
print(f" Written: {filepath}")
def main():
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html")
output_dir = os.path.join(base_dir, "lib", "jsons", "books", "sb-1962")
if not os.path.exists(html_path):
print(f"ERROR: {html_path} not found!")
sys.exit(1)
print(f"Reading {html_path}...")
with open(html_path, "r", encoding="utf-8") as f:
soup = BeautifulSoup(f, "html.parser")
print("Extracting SB 1962 verses...")
verses, chapter_count = extract_verses(soup)
print(f"\nExtracted {len(verses)} verses across {chapter_count} chapters")
# Group by chapter
chapters_info = {}
for verse in verses:
ch = verse["chapter"]
if ch not in chapters_info:
chapters_info[ch] = 0
chapters_info[ch] += 1
# Build intro.json
chapters_list = []
for ch_num in sorted(chapters_info.keys()):
chapters_list.append(
{
"number": ch_num,
"title": CHAPTER_TITLES.get(ch_num, f"Chapter {ch_num}"),
"totalVerses": chapters_info[ch_num],
}
)
intro = {
"book": "sb1962",
"title": "Śrīmad-Bhāgavatam — First Canto (1962 Edition)",
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
"description": "The original 1962-65 Indian edition of the First Canto, published before Śrīla Prabhupāda's journey to the West. Three volumes covering 19 chapters.",
"totalChapters": len(chapters_list),
"chapters": chapters_list,
}
write_json(intro, os.path.join(output_dir, "intro.json"))
# Write verse JSON files
for verse in verses:
ch = str(verse["chapter"]).zfill(2)
vs = str(verse["verse"]).zfill(2)
filepath = os.path.join(output_dir, "chapters", ch, "verses", f"{vs}.json")
write_json(verse, filepath)
# Print summary
print("\n--- Verification ---")
for ch_num in sorted(chapters_info.keys()):
ch_verses = [v for v in verses if v["chapter"] == ch_num]
if ch_verses:
first = ch_verses[0]
last = ch_verses[-1]
title = CHAPTER_TITLES.get(ch_num, "")
print(
f" Ch {ch_num:>2}: {len(ch_verses):>3} verses "
f"({first['reference']} → {last['reference']}) {title}"
)
# Quick quality check
if verses:
print("\n--- Sample (first verse) ---")
v = verses[0]
print(f"Reference: {v['reference']}")
print(f"Transliteration:\n{v['transliteration']}")
print(f"Synonyms: {len(v['synonyms'])} entries")
if v["synonyms"]:
print(f" First: {v['synonyms'][0]}")
trans = v["translation"][:120] if v["translation"] else "(none)"
print(f"Translation: {trans}...")
print(f"Purport: {len(v['purport'])} paragraphs")
# Quality stats
print("\n--- Quality Stats ---")
empty_trans = sum(1 for v in verses if not v.get("transliteration", "").strip())
empty_translation = sum(1 for v in verses if not v.get("translation", "").strip())
empty_syn = sum(1 for v in verses if not v.get("synonyms"))
has_purport = sum(1 for v in verses if v.get("purport"))
print(f"Total verses: {len(verses)}")
print(f"Empty transliteration: {empty_trans}")
print(f"Empty translation: {empty_translation}")
print(f"Empty synonyms: {empty_syn}")
print(f"Verses with purport: {has_purport}")
print(f"Verses without purport: {len(verses) - has_purport}")
if __name__ == "__main__":
main()