feat: allow path params for verse navigation
This commit is contained in:
gosangam
Binary file not shown.
@@ -389,7 +389,7 @@ def write_json(data, filepath):
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "bg.html")
|
||||
output_dir = os.path.join(base_dir, "app", "lib", "books", "bg")
|
||||
output_dir = os.path.join(base_dir, "lib", "jsons", "books", "bg")
|
||||
|
||||
print(f"Reading {html_path}...")
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
|
||||
@@ -677,7 +677,7 @@ def write_json(data, filepath):
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "cc.html")
|
||||
output_dir = os.path.join(base_dir, "app", "lib", "books", "cc")
|
||||
output_dir = os.path.join(base_dir, "lib", "jsons", "books", "cc")
|
||||
|
||||
print(f"Reading {html_path}...")
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
|
||||
@@ -274,7 +274,7 @@ def build_letter_index(entries: list) -> dict:
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "glossary.html")
|
||||
output_path = os.path.join(base_dir, "app", "lib", "glossary", "index.json")
|
||||
output_path = os.path.join(base_dir, "lib", "jsons", "glossary", "index.json")
|
||||
|
||||
if not os.path.exists(html_path):
|
||||
print(f"ERROR: {html_path} not found!")
|
||||
|
||||
@@ -0,0 +1,263 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Match SB 1962 transliterations and synonym words with standard SB Canto 1.
|
||||
|
||||
For each verse in SB 1962 that has a matching verse in the standard SB
|
||||
(same chapter + verse number), this script:
|
||||
|
||||
1. Replaces the 1962 plain-English transliteration with the proper IAST version
|
||||
from the standard SB.
|
||||
2. Replaces each synonym "word" key with the IAST version from the standard SB,
|
||||
matching by position (index). If the 1962 edition has MORE synonyms than the
|
||||
standard SB, the extra ones are kept as-is. If the standard SB has more,
|
||||
the extras are ignored.
|
||||
3. Leaves the synonym "meanings" untouched (keeps the 1962 meanings).
|
||||
4. Leaves the translation and purport untouched (keeps the 1962 text).
|
||||
5. For verses that exist only in the 1962 edition (no standard SB match),
|
||||
everything is left as-is.
|
||||
|
||||
Usage:
|
||||
python3 scripts/match_sb1962.py
|
||||
|
||||
Prerequisites:
|
||||
- Standard SB must be extracted at lib/jsons/books/sb/cantos/01/chapters/*/verses/*.json
|
||||
- SB 1962 must be extracted at lib/jsons/books/sb-1962/chapters/*/verses/*.json
|
||||
"""
|
||||
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
|
||||
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
SB_DIR = os.path.join(BASE_DIR, "lib", "jsons", "books", "sb", "cantos", "01")
|
||||
SB62_DIR = os.path.join(BASE_DIR, "lib", "jsons", "books", "sb-1962")
|
||||
|
||||
# Known misaligned verses where the 1962 numbering diverges from the standard SB.
|
||||
# These are skipped during matching to avoid applying wrong IAST transliterations.
|
||||
# The 1962 edition splits/numbers some verses differently in chapter 15.
|
||||
SKIP_VERSES = {
|
||||
(15, 41),
|
||||
(15, 42),
|
||||
(15, 43),
|
||||
(15, 44),
|
||||
(15, 45),
|
||||
}
|
||||
|
||||
|
||||
def normalize_transliteration(text: str) -> str:
|
||||
"""Normalize transliteration for fuzzy comparison.
|
||||
|
||||
Strips diacritics, lowercases, removes punctuation/hyphens, collapses whitespace.
|
||||
This allows matching 'Udichim pravibesasam' with 'udīcīṁ praviveśāśāṁ'.
|
||||
"""
|
||||
import unicodedata
|
||||
|
||||
# NFD decompose then strip combining marks (diacritics)
|
||||
nfkd = unicodedata.normalize("NFD", text)
|
||||
stripped = "".join(c for c in nfkd if unicodedata.category(c) != "Mn")
|
||||
# Lowercase, remove hyphens, punctuation, collapse whitespace
|
||||
stripped = stripped.lower()
|
||||
stripped = stripped.replace("-", "").replace("'", "").replace("'", "")
|
||||
stripped = "".join(c for c in stripped if c.isalnum() or c.isspace())
|
||||
stripped = " ".join(stripped.split())
|
||||
return stripped
|
||||
|
||||
|
||||
def transliteration_similarity(a: str, b: str) -> float:
|
||||
"""Compute a simple word-overlap similarity between two transliterations.
|
||||
|
||||
Returns a ratio between 0.0 and 1.0.
|
||||
"""
|
||||
words_a = set(normalize_transliteration(a).split())
|
||||
words_b = set(normalize_transliteration(b).split())
|
||||
if not words_a or not words_b:
|
||||
return 0.0
|
||||
intersection = words_a & words_b
|
||||
union = words_a | words_b
|
||||
return len(intersection) / len(union)
|
||||
|
||||
|
||||
def load_sb_canto1():
|
||||
"""Load all standard SB Canto 1 verses, keyed by (chapter, verse).
|
||||
|
||||
Also returns a per-chapter list for fuzzy matching.
|
||||
"""
|
||||
verses = {}
|
||||
by_chapter: dict[int, list] = {}
|
||||
pattern = os.path.join(SB_DIR, "chapters", "*", "verses", "*.json")
|
||||
for f in sorted(glob.glob(pattern)):
|
||||
with open(f, "r", encoding="utf-8") as fh:
|
||||
d = json.load(fh)
|
||||
key = (d["chapter"], d["verse"])
|
||||
verses[key] = d
|
||||
|
||||
ch = d["chapter"]
|
||||
if ch not in by_chapter:
|
||||
by_chapter[ch] = []
|
||||
by_chapter[ch].append(d)
|
||||
|
||||
# For merged verses like 28-29, also index by the end verse
|
||||
if "verse_end" in d and d["verse_end"]:
|
||||
for v in range(d["verse"] + 1, d["verse_end"] + 1):
|
||||
verses[(d["chapter"], v)] = d
|
||||
return verses, by_chapter
|
||||
|
||||
|
||||
def load_sb1962():
|
||||
"""Load all SB 1962 verses, returning a list of (filepath, data) tuples."""
|
||||
results = []
|
||||
pattern = os.path.join(SB62_DIR, "chapters", "*", "verses", "*.json")
|
||||
for f in sorted(glob.glob(pattern)):
|
||||
with open(f, "r", encoding="utf-8") as fh:
|
||||
d = json.load(fh)
|
||||
results.append((f, d))
|
||||
return results
|
||||
|
||||
|
||||
def match_synonyms(sb62_synonyms, sb_synonyms):
|
||||
"""Match synonym word keys from standard SB into the 1962 synonyms.
|
||||
|
||||
Strategy: match by position index. For each synonym in the 1962 list,
|
||||
if there's a corresponding synonym at the same index in the standard SB,
|
||||
replace the word key with the IAST version. Keep the 1962 meaning.
|
||||
|
||||
If the 1962 has more synonyms than the standard, keep the extras as-is.
|
||||
"""
|
||||
result = []
|
||||
for i, syn62 in enumerate(sb62_synonyms):
|
||||
new_syn = dict(syn62) # copy
|
||||
if i < len(sb_synonyms):
|
||||
# Replace the word key with the IAST version
|
||||
new_syn["word"] = sb_synonyms[i]["word"]
|
||||
# Keep the original 1962 meaning regardless
|
||||
result.append(new_syn)
|
||||
return result
|
||||
|
||||
|
||||
def write_json(data, filepath):
|
||||
"""Write JSON with proper Unicode and formatting."""
|
||||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||||
with open(filepath, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def find_best_match(verse62: dict, sb_chapter_verses: list, threshold: float = 0.4):
|
||||
"""Find the best-matching standard SB verse by transliteration similarity.
|
||||
|
||||
Used as a fallback when direct verse-number matching fails or produces
|
||||
a low-confidence match. Returns the best-matching SB verse or None.
|
||||
"""
|
||||
trans_62 = verse62.get("transliteration", "")
|
||||
if not trans_62.strip():
|
||||
return None
|
||||
|
||||
best_score = 0.0
|
||||
best_match = None
|
||||
for sb_verse in sb_chapter_verses:
|
||||
sb_trans = sb_verse.get("transliteration", "")
|
||||
score = transliteration_similarity(trans_62, sb_trans)
|
||||
if score > best_score:
|
||||
best_score = score
|
||||
best_match = sb_verse
|
||||
|
||||
if best_score >= threshold:
|
||||
return best_match
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
print("Loading standard SB Canto 1 verses...")
|
||||
sb_verses, sb_by_chapter = load_sb_canto1()
|
||||
print(f" Loaded {len(sb_verses)} verse entries (including merged-verse aliases)")
|
||||
|
||||
print("Loading SB 1962 verses...")
|
||||
sb62_files = load_sb1962()
|
||||
print(f" Loaded {len(sb62_files)} verse files")
|
||||
|
||||
matched = 0
|
||||
unmatched = 0
|
||||
fuzzy_matched = 0
|
||||
trans_updated = 0
|
||||
syn_updated = 0
|
||||
|
||||
for filepath, verse62 in sb62_files:
|
||||
ch = verse62["chapter"]
|
||||
vs = verse62["verse"]
|
||||
key = (ch, vs)
|
||||
|
||||
# Skip known misaligned verses
|
||||
if key in SKIP_VERSES:
|
||||
unmatched += 1
|
||||
continue
|
||||
|
||||
sb_verse = sb_verses.get(key)
|
||||
|
||||
if sb_verse is None:
|
||||
# No match — keep as-is (e.g., 1962-only verse numbers)
|
||||
unmatched += 1
|
||||
continue
|
||||
|
||||
matched += 1
|
||||
changed = False
|
||||
|
||||
# 1. Replace transliteration with IAST version
|
||||
sb_trans = sb_verse.get("transliteration", "").strip()
|
||||
if sb_trans:
|
||||
old_trans = verse62.get("transliteration", "")
|
||||
verse62["transliteration"] = sb_trans
|
||||
if old_trans != sb_trans:
|
||||
trans_updated += 1
|
||||
changed = True
|
||||
|
||||
# 2. Match synonym word keys
|
||||
sb_syns = sb_verse.get("synonyms", [])
|
||||
v62_syns = verse62.get("synonyms", [])
|
||||
if sb_syns and v62_syns:
|
||||
new_syns = match_synonyms(v62_syns, sb_syns)
|
||||
if new_syns != v62_syns:
|
||||
verse62["synonyms"] = new_syns
|
||||
syn_updated += 1
|
||||
changed = True
|
||||
|
||||
# 3. Write back if changed
|
||||
if changed:
|
||||
write_json(verse62, filepath)
|
||||
|
||||
print(f"\n--- Results ---")
|
||||
print(f"Total 1962 verses: {len(sb62_files)}")
|
||||
print(
|
||||
f"Matched with standard SB: {matched} (direct: {matched - fuzzy_matched}, fuzzy: {fuzzy_matched})"
|
||||
)
|
||||
print(f"Unmatched (kept as-is): {unmatched}")
|
||||
print(f"Transliterations updated: {trans_updated}")
|
||||
print(f"Synonyms updated: {syn_updated}")
|
||||
|
||||
# Show a sample of matched verses
|
||||
print(f"\n--- Samples ---")
|
||||
samples = [(1, 1), (1, 2), (2, 1), (7, 1), (13, 1)]
|
||||
for ch, vs in samples:
|
||||
fpath = os.path.join(
|
||||
SB62_DIR, "chapters", str(ch).zfill(2), "verses", f"{str(vs).zfill(2)}.json"
|
||||
)
|
||||
if os.path.exists(fpath):
|
||||
with open(fpath, "r", encoding="utf-8") as f:
|
||||
d = json.load(f)
|
||||
print(f"\n {d['reference']}:")
|
||||
trans_lines = d["transliteration"].split("\n")
|
||||
print(f" Transliteration: {trans_lines[0]}")
|
||||
if len(trans_lines) > 1:
|
||||
print(f" {trans_lines[1]}")
|
||||
if d["synonyms"]:
|
||||
print(f" First synonym: {d['synonyms'][0]}")
|
||||
|
||||
# Show unmatched verses
|
||||
if unmatched > 0:
|
||||
print(f"\n--- Unmatched 1962 verses (kept as-is) ---")
|
||||
for filepath, verse62 in sb62_files:
|
||||
ch = verse62["chapter"]
|
||||
vs = verse62["verse"]
|
||||
if (ch, vs) not in sb_verses:
|
||||
print(f" {verse62['reference']}: {verse62['transliteration'][:50]}...")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,617 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract 1962 Śrīmad-Bhāgavatam First Canto verses from others-before-1977.html.
|
||||
|
||||
This is the original 1962-65 edition of Śrīmad-Bhāgavatam First Canto published
|
||||
in India before Śrīla Prabhupāda went to the USA. It spans 3 volumes:
|
||||
- Volume 1 (1962): Chapters 1–7 (first half)
|
||||
- Volume 2 (1964): Chapter 7 (second half) through Chapter 12
|
||||
- Volume 3 (1965): Chapters 13–19
|
||||
|
||||
Key differences from the standard SB extraction:
|
||||
- No Devanagari or ScaGoudy encoding — transliteration is plain English (non-IAST)
|
||||
- Different CSS class mappings
|
||||
- Verse markers use "TEXT No. N" format (p18 class)
|
||||
- Reference markers (p17) are sparse — only some verses have them
|
||||
- Chapter boundaries use spelled-out ordinals: "FIRST CHAPTER", "SECOND CHAPTER", etc.
|
||||
- Multi-verse entries use "TEXTS No.'s 3 AND 4" format
|
||||
|
||||
Output goes to app/lib/books/sb1962/
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Chapter name → number mapping
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
ORDINAL_TO_NUM = {
|
||||
"first": 1,
|
||||
"second": 2,
|
||||
"third": 3,
|
||||
"fourth": 4,
|
||||
"fifth": 5,
|
||||
"sixth": 6,
|
||||
"seventh": 7,
|
||||
"eighth": 8,
|
||||
"nineth": 9, # sic — typo in the original
|
||||
"ninth": 9,
|
||||
"tenth": 10,
|
||||
"eleventh": 11,
|
||||
"twelfth": 12,
|
||||
"thirteenth": 13,
|
||||
"fourteenth": 14,
|
||||
"fifteenth": 15,
|
||||
"sixteenth": 16,
|
||||
"seventeenth": 17,
|
||||
"eighteenth": 18,
|
||||
"nineteenth": 19,
|
||||
}
|
||||
|
||||
# Chapter titles (from the "Thus end" markers and chapter headers)
|
||||
CHAPTER_TITLES = {
|
||||
1: "Questions by the Sages",
|
||||
2: "Divinity and Devotional Service",
|
||||
3: "Description of Incarnations of Godhead",
|
||||
4: "Appearance of Sri Narada",
|
||||
5: "Sri Narada's Instructions on Srimad Bhagwatam for Vyasdeva",
|
||||
6: "Conversation Between Narada and Vyasa",
|
||||
7: "The Son of Drona Punished",
|
||||
8: "Prayers of Queen Kunti and Parikshit Saved",
|
||||
9: "Passing Away of Bhismadeva in the Presence of Lord Krishna",
|
||||
10: "Departure of Lord Krishna for Dwarka",
|
||||
11: "Lord Krishna's Entrance at Dwarka",
|
||||
12: "Birth of Maharaj Parikshit",
|
||||
13: "Dhritarastra Quits Home",
|
||||
14: "Disappearance of Lord Krishna",
|
||||
15: "The Pandavas Retire Timely",
|
||||
16: "How Parikshit Received The Age of Kali",
|
||||
17: "Punishment and Reward of Kali",
|
||||
18: "Maharaj Parikshit Cursed by a Brahmin Boy",
|
||||
19: "Appearance of Sri Sukdeva Goswami",
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HTML helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def get_css_class(tag: Tag) -> str:
|
||||
"""Get the primary CSS class of a tag."""
|
||||
classes = tag.get("class", [])
|
||||
return classes[0] if classes else ""
|
||||
|
||||
|
||||
def decode_html_content(tag: Tag) -> str:
|
||||
"""Extract inner HTML preserving <i>/<b> formatting."""
|
||||
inner = "".join(str(child) for child in tag.children)
|
||||
# Remove span tags with CSS classes (RTF conversion artifacts)
|
||||
inner = re.sub(r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner)
|
||||
inner = re.sub(r"\s+", " ", inner).strip()
|
||||
return inner
|
||||
|
||||
|
||||
def decode_text(tag: Tag) -> str:
|
||||
"""Extract plain text from a tag."""
|
||||
return tag.get_text().strip()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Synonym parser
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def parse_synonyms(text: str) -> list:
|
||||
"""Parse synonym text from the 1962 edition.
|
||||
|
||||
Format is similar to later editions but uses plain English:
|
||||
"Om—Oh my Lord, namo—offering my obeisances, ..."
|
||||
Words may be wrapped in <i> tags.
|
||||
"""
|
||||
# Strip HTML tags for parsing
|
||||
plain = re.sub(r"<[^>]+>", "", text).strip()
|
||||
if not plain:
|
||||
return []
|
||||
|
||||
result = []
|
||||
# Split on comma that's followed by an italic word (the next synonym)
|
||||
# But also handle semicolons
|
||||
# The pattern is: word—meaning, word—meaning, ...
|
||||
# Split on ", " followed by what looks like a new word entry
|
||||
# Actually simpler: split on the em-dash pattern
|
||||
entries = re.split(r",\s*(?=[A-Z])", plain)
|
||||
|
||||
for entry in entries:
|
||||
entry = entry.strip()
|
||||
if not entry:
|
||||
continue
|
||||
if "—" in entry:
|
||||
parts = entry.split("—", 1)
|
||||
elif "–" in entry:
|
||||
parts = entry.split("–", 1)
|
||||
elif "- " in entry:
|
||||
# Some entries use hyphen-space
|
||||
parts = entry.split("- ", 1)
|
||||
else:
|
||||
continue
|
||||
|
||||
if len(parts) == 2:
|
||||
word = parts[0].strip().strip("., ;:")
|
||||
meaning = parts[1].strip().rstrip(".,")
|
||||
if word and meaning:
|
||||
result.append({"word": word, "meanings": [meaning]})
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CSS class sets for the 1962 SB
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Verse reference bookmark (sparse — not all verses have these)
|
||||
VERSE_REF_CLASS = "p17"
|
||||
|
||||
# TEXT No. N header
|
||||
TEXT_HEADER_CLASS = "p18"
|
||||
|
||||
# "TEXT" label
|
||||
TEXT_LABEL_CLASS = "p2"
|
||||
|
||||
# Transliteration pāda lines (italic, centered)
|
||||
TRANSLITERATION_CLASSES = frozenset(["p19", "p87"])
|
||||
|
||||
# Section header class: "ENGLISH SYNONYMS", "TRANSLATION", "PURPORT"
|
||||
SECTION_HEADER_CLASSES = frozenset(["p20", "p8"])
|
||||
|
||||
# Synonyms text class
|
||||
SYNONYM_CLASSES = frozenset(["p88"])
|
||||
|
||||
# Translation text class
|
||||
TRANSLATION_CLASSES = frozenset(["p89"])
|
||||
|
||||
# Purport paragraph classes
|
||||
PURPORT_CLASSES = frozenset(["p74", "p6", "p48"])
|
||||
|
||||
# Chapter heading class
|
||||
CHAPTER_HEADING_CLASS = "p3"
|
||||
|
||||
# Chapter bookmark class
|
||||
CHAPTER_BOOKMARK_CLASS = "p11"
|
||||
|
||||
# Volume/section boundary classes — skip
|
||||
SKIP_CLASSES = frozenset(
|
||||
[
|
||||
"p35", # volume title
|
||||
"p8", # various front matter and banners (also used for some headers)
|
||||
"p73", # "First Published in 1962"
|
||||
"p105", # chapter subtitle in volume 2
|
||||
"p106", # chapter subtitle in volume 3
|
||||
"p71", # review quotes
|
||||
"p72", # review quotes
|
||||
]
|
||||
)
|
||||
|
||||
# Line range in the HTML for the SB 1962 content
|
||||
# From the first chapter marker to "END OF THE FIRST CANTO"
|
||||
SB1962_START_LINE = 4726
|
||||
SB1962_END_LINE = 15162
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Verse number parser
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def parse_text_header(text: str):
|
||||
"""Parse a TEXT header like 'TEXT No. 1' or 'TEXTS No.'s 3 AND 4'.
|
||||
|
||||
Returns (verse_num, verse_end) or None.
|
||||
"""
|
||||
text = text.strip()
|
||||
|
||||
# Multi-verse: "TEXTS No.'s 3 AND 4" or "TEXTS Nos. 26 AND 27"
|
||||
m = re.match(
|
||||
r"TEXTS?\s+No\.?'?s?\s*(\d+)\s+AND\s+(\d+)",
|
||||
text,
|
||||
re.IGNORECASE,
|
||||
)
|
||||
if m:
|
||||
return (int(m.group(1)), int(m.group(2)))
|
||||
|
||||
# Single verse: "TEXT No. 1" or "TEXT No. 1."
|
||||
m = re.match(r"TEXT\s+No\.?\s*(\d+)", text, re.IGNORECASE)
|
||||
if m:
|
||||
return (int(m.group(1)), None)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def parse_chapter_heading(text: str):
|
||||
"""Parse a chapter heading like 'FIRST CHAPTER' or 'SEVENTH CHAPTER'.
|
||||
|
||||
Returns chapter number or None.
|
||||
"""
|
||||
text = text.strip().upper()
|
||||
# Remove bold/italic artifacts
|
||||
text = re.sub(r"<[^>]+>", "", text).strip()
|
||||
|
||||
m = re.match(r"(\w+)\s+CHAPTER", text)
|
||||
if m:
|
||||
ordinal = m.group(1).lower()
|
||||
return ORDINAL_TO_NUM.get(ordinal)
|
||||
return None
|
||||
|
||||
|
||||
def parse_ref_marker(text: str):
|
||||
"""Parse a reference marker like 'SB. 1-1962:1.1.1' or 'SB. 1.13-1965:1.13.3-4'.
|
||||
|
||||
Returns (chapter, verse, verse_end) or None.
|
||||
"""
|
||||
# Match patterns like: SB. 1-1962:1.1.1 or SB. 1.7-1964:1.7.9 or SB. 1.13-1965:1.13.3-4
|
||||
m = re.search(r"SB\.\s*[\d.]+-\d{4}:1\.(\d+)\.(\d+)(?:-(\d+))?", text)
|
||||
if m:
|
||||
chapter = int(m.group(1))
|
||||
verse = int(m.group(2))
|
||||
verse_end = int(m.group(3)) if m.group(3) else None
|
||||
return (chapter, verse, verse_end)
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# State machine extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _finalize_verse(verse: dict):
|
||||
"""Clean up a verse before appending to results."""
|
||||
# Join transliteration lines
|
||||
padas = verse.pop("_padas", [])
|
||||
verse["transliteration"] = "\n".join(padas)
|
||||
|
||||
# Clean up internal state
|
||||
verse.pop("_state", None)
|
||||
|
||||
|
||||
def extract_verses(soup: BeautifulSoup):
|
||||
"""Extract all SB 1962 verses from the HTML.
|
||||
|
||||
Returns (verses, chapter_count).
|
||||
"""
|
||||
all_paragraphs = soup.find_all("p")
|
||||
|
||||
# Filter to only the paragraphs in the SB 1962 range
|
||||
# We use the sourceline attribute if available, otherwise fall back to
|
||||
# scanning for the start/end markers
|
||||
paragraphs = []
|
||||
in_range = False
|
||||
|
||||
for p in all_paragraphs:
|
||||
cls = get_css_class(p)
|
||||
text = decode_text(p)
|
||||
|
||||
# Start: first chapter marker
|
||||
if cls == CHAPTER_BOOKMARK_CLASS and "SB. 1-1962:: First Chapter" in text:
|
||||
in_range = True
|
||||
|
||||
if in_range:
|
||||
paragraphs.append(p)
|
||||
|
||||
# End: "END OF THE FIRST CANTO"
|
||||
if in_range and "END OF THE FIRST CANTO" in text:
|
||||
break
|
||||
|
||||
if not paragraphs:
|
||||
print("ERROR: Could not find SB 1962 content in the HTML!")
|
||||
return [], 0
|
||||
|
||||
print(f" Found {len(paragraphs)} paragraphs in SB 1962 range")
|
||||
|
||||
verses = []
|
||||
current_verse = None
|
||||
state = "IDLE"
|
||||
current_chapter = 0
|
||||
chapters_seen = set()
|
||||
|
||||
for p in paragraphs:
|
||||
cls = get_css_class(p)
|
||||
text = decode_text(p)
|
||||
|
||||
# --- Chapter heading (p3): "FIRST CHAPTER", "SECOND CHAPTER", etc. ---
|
||||
if cls == CHAPTER_HEADING_CLASS:
|
||||
ch_num = parse_chapter_heading(text)
|
||||
if ch_num is not None:
|
||||
# Finalize previous verse if any
|
||||
if current_verse:
|
||||
_finalize_verse(current_verse)
|
||||
verses.append(current_verse)
|
||||
current_verse = None
|
||||
state = "IDLE"
|
||||
|
||||
current_chapter = ch_num
|
||||
chapters_seen.add(ch_num)
|
||||
continue
|
||||
|
||||
# Skip non-chapter headings (APPENDIX, CONTENTS, PREFACE, etc.)
|
||||
if any(
|
||||
kw in text.upper()
|
||||
for kw in ["APPENDIX", "CONTENTS", "PREFACE", "ERRATA"]
|
||||
):
|
||||
# We've left the chapter content — stop processing until next chapter
|
||||
if current_verse:
|
||||
_finalize_verse(current_verse)
|
||||
verses.append(current_verse)
|
||||
current_verse = None
|
||||
state = "SKIP"
|
||||
continue
|
||||
|
||||
# --- Skip mode (appendix, front matter between volumes) ---
|
||||
if state == "SKIP":
|
||||
# Resume when we hit a new chapter heading
|
||||
if cls == CHAPTER_HEADING_CLASS:
|
||||
ch_num = parse_chapter_heading(text)
|
||||
if ch_num is not None:
|
||||
current_chapter = ch_num
|
||||
chapters_seen.add(ch_num)
|
||||
state = "IDLE"
|
||||
continue
|
||||
# Also resume on a verse ref marker for the next volume
|
||||
if cls == VERSE_REF_CLASS and "SB." in text:
|
||||
ref = parse_ref_marker(text)
|
||||
if ref:
|
||||
state = "IDLE"
|
||||
# Fall through to handle the ref below
|
||||
else:
|
||||
continue
|
||||
else:
|
||||
continue
|
||||
|
||||
# --- Chapter bookmark (p11) — might give us the chapter number ---
|
||||
if cls == CHAPTER_BOOKMARK_CLASS:
|
||||
# Could be "SB. 1-1962:: First Chapter" or "SB. 1.7-1964: ..."
|
||||
# We already handle chapter number from the p3 heading
|
||||
continue
|
||||
|
||||
# --- Volume/section boundaries — skip ---
|
||||
if cls in SKIP_CLASSES:
|
||||
continue
|
||||
|
||||
# --- Verse reference marker (p17) ---
|
||||
if cls == VERSE_REF_CLASS:
|
||||
ref = parse_ref_marker(text)
|
||||
if ref:
|
||||
# This gives us chapter/verse info, but the TEXT header is the
|
||||
# primary verse trigger. Just note the chapter if different.
|
||||
ch, vs, vs_end = ref
|
||||
if ch != current_chapter and ch > 0:
|
||||
current_chapter = ch
|
||||
chapters_seen.add(ch)
|
||||
continue
|
||||
|
||||
# --- TEXT No. N header (p18) — primary verse trigger ---
|
||||
if cls == TEXT_HEADER_CLASS:
|
||||
parsed = parse_text_header(text)
|
||||
if parsed is None:
|
||||
continue
|
||||
|
||||
verse_num, verse_end = parsed
|
||||
|
||||
# Finalize previous verse
|
||||
if current_verse:
|
||||
_finalize_verse(current_verse)
|
||||
verses.append(current_verse)
|
||||
|
||||
if verse_end:
|
||||
ref_str = f"SB 1.{current_chapter}.{verse_num}-{verse_end}"
|
||||
else:
|
||||
ref_str = f"SB 1.{current_chapter}.{verse_num}"
|
||||
|
||||
current_verse = {
|
||||
"book_title": "Śrīmad-Bhāgavatam (1962 Edition)",
|
||||
"chapter": current_chapter,
|
||||
"verse": verse_num,
|
||||
"reference": ref_str,
|
||||
"transliteration": "",
|
||||
"_padas": [],
|
||||
"synonyms": [],
|
||||
"translation": "",
|
||||
"purport": [],
|
||||
"_state": "VERSE_START",
|
||||
}
|
||||
if verse_end is not None:
|
||||
current_verse["verse_end"] = verse_end
|
||||
state = "VERSE_START"
|
||||
continue
|
||||
|
||||
# --- TEXT label (p2) — just confirms we're in a verse ---
|
||||
if cls == TEXT_LABEL_CLASS and current_verse:
|
||||
continue
|
||||
|
||||
# --- Transliteration lines (p19, p87) ---
|
||||
if (
|
||||
cls in TRANSLITERATION_CLASSES
|
||||
and current_verse
|
||||
and state in ("VERSE_START", "TRANSLITERATION")
|
||||
):
|
||||
state = "TRANSLITERATION"
|
||||
current_verse["_state"] = state
|
||||
line = decode_text(p)
|
||||
if line:
|
||||
current_verse["_padas"].append(line)
|
||||
continue
|
||||
|
||||
# --- Section headers (p20, p8): ENGLISH SYNONYMS, TRANSLATION, PURPORT ---
|
||||
if cls in SECTION_HEADER_CLASSES and current_verse:
|
||||
header_text = text.strip().upper()
|
||||
if "SYNONYM" in header_text:
|
||||
state = "SYNONYMS"
|
||||
current_verse["_state"] = state
|
||||
elif "TRANSLATION" in header_text:
|
||||
state = "TRANSLATION"
|
||||
current_verse["_state"] = state
|
||||
elif "PURPORT" in header_text:
|
||||
state = "PURPORT"
|
||||
current_verse["_state"] = state
|
||||
continue
|
||||
|
||||
# --- Synonyms text (p88) ---
|
||||
if cls in SYNONYM_CLASSES and current_verse and state == "SYNONYMS":
|
||||
html_content = decode_html_content(p)
|
||||
current_verse["synonyms"] = parse_synonyms(html_content)
|
||||
continue
|
||||
|
||||
# --- Translation text (p89) ---
|
||||
if cls in TRANSLATION_CLASSES and current_verse and state == "TRANSLATION":
|
||||
current_verse["translation"] = decode_html_content(p)
|
||||
state = "POST_TRANSLATION"
|
||||
current_verse["_state"] = state
|
||||
continue
|
||||
|
||||
# --- Handle mislabeled headers (TRANSLATION header followed by
|
||||
# translation in p89 while state is PURPORT) ---
|
||||
if (
|
||||
cls in TRANSLATION_CLASSES
|
||||
and current_verse
|
||||
and not current_verse.get("translation")
|
||||
):
|
||||
current_verse["translation"] = decode_html_content(p)
|
||||
state = "POST_TRANSLATION"
|
||||
current_verse["_state"] = state
|
||||
continue
|
||||
|
||||
# --- Purport paragraphs (p74, p6, p48) ---
|
||||
if state == "PURPORT" and current_verse and cls in PURPORT_CLASSES:
|
||||
content = decode_html_content(p)
|
||||
if content and "Thus end" not in text and "Thus ended" not in text:
|
||||
current_verse["purport"].append(content)
|
||||
continue
|
||||
|
||||
# --- Chapter end marker ("Thus end the Bhaktivedanta Purports") ---
|
||||
if ("Thus end" in text or "Thus ended" in text) and current_verse:
|
||||
_finalize_verse(current_verse)
|
||||
verses.append(current_verse)
|
||||
current_verse = None
|
||||
state = "IDLE"
|
||||
continue
|
||||
|
||||
# Don't forget the last verse
|
||||
if current_verse:
|
||||
_finalize_verse(current_verse)
|
||||
verses.append(current_verse)
|
||||
|
||||
return verses, len(chapters_seen)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# JSON output
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def write_json(data, filepath):
|
||||
"""Write JSON with proper Unicode and formatting."""
|
||||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||||
with open(filepath, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
print(f" Written: {filepath}")
|
||||
|
||||
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html")
|
||||
output_dir = os.path.join(base_dir, "lib", "jsons", "books", "sb-1962")
|
||||
|
||||
if not os.path.exists(html_path):
|
||||
print(f"ERROR: {html_path} not found!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Reading {html_path}...")
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
soup = BeautifulSoup(f, "html.parser")
|
||||
|
||||
print("Extracting SB 1962 verses...")
|
||||
verses, chapter_count = extract_verses(soup)
|
||||
|
||||
print(f"\nExtracted {len(verses)} verses across {chapter_count} chapters")
|
||||
|
||||
# Group by chapter
|
||||
chapters_info = {}
|
||||
for verse in verses:
|
||||
ch = verse["chapter"]
|
||||
if ch not in chapters_info:
|
||||
chapters_info[ch] = 0
|
||||
chapters_info[ch] += 1
|
||||
|
||||
# Build intro.json
|
||||
chapters_list = []
|
||||
for ch_num in sorted(chapters_info.keys()):
|
||||
chapters_list.append(
|
||||
{
|
||||
"number": ch_num,
|
||||
"title": CHAPTER_TITLES.get(ch_num, f"Chapter {ch_num}"),
|
||||
"totalVerses": chapters_info[ch_num],
|
||||
}
|
||||
)
|
||||
|
||||
intro = {
|
||||
"book": "sb1962",
|
||||
"title": "Śrīmad-Bhāgavatam — First Canto (1962 Edition)",
|
||||
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
||||
"description": "The original 1962-65 Indian edition of the First Canto, published before Śrīla Prabhupāda's journey to the West. Three volumes covering 19 chapters.",
|
||||
"totalChapters": len(chapters_list),
|
||||
"chapters": chapters_list,
|
||||
}
|
||||
write_json(intro, os.path.join(output_dir, "intro.json"))
|
||||
|
||||
# Write verse JSON files
|
||||
for verse in verses:
|
||||
ch = str(verse["chapter"]).zfill(2)
|
||||
vs = str(verse["verse"]).zfill(2)
|
||||
filepath = os.path.join(output_dir, "chapters", ch, "verses", f"{vs}.json")
|
||||
write_json(verse, filepath)
|
||||
|
||||
# Print summary
|
||||
print("\n--- Verification ---")
|
||||
for ch_num in sorted(chapters_info.keys()):
|
||||
ch_verses = [v for v in verses if v["chapter"] == ch_num]
|
||||
if ch_verses:
|
||||
first = ch_verses[0]
|
||||
last = ch_verses[-1]
|
||||
title = CHAPTER_TITLES.get(ch_num, "")
|
||||
print(
|
||||
f" Ch {ch_num:>2}: {len(ch_verses):>3} verses "
|
||||
f"({first['reference']} → {last['reference']}) {title}"
|
||||
)
|
||||
|
||||
# Quick quality check
|
||||
if verses:
|
||||
print("\n--- Sample (first verse) ---")
|
||||
v = verses[0]
|
||||
print(f"Reference: {v['reference']}")
|
||||
print(f"Transliteration:\n{v['transliteration']}")
|
||||
print(f"Synonyms: {len(v['synonyms'])} entries")
|
||||
if v["synonyms"]:
|
||||
print(f" First: {v['synonyms'][0]}")
|
||||
trans = v["translation"][:120] if v["translation"] else "(none)"
|
||||
print(f"Translation: {trans}...")
|
||||
print(f"Purport: {len(v['purport'])} paragraphs")
|
||||
|
||||
# Quality stats
|
||||
print("\n--- Quality Stats ---")
|
||||
empty_trans = sum(1 for v in verses if not v.get("transliteration", "").strip())
|
||||
empty_translation = sum(1 for v in verses if not v.get("translation", "").strip())
|
||||
empty_syn = sum(1 for v in verses if not v.get("synonyms"))
|
||||
has_purport = sum(1 for v in verses if v.get("purport"))
|
||||
print(f"Total verses: {len(verses)}")
|
||||
print(f"Empty transliteration: {empty_trans}")
|
||||
print(f"Empty translation: {empty_translation}")
|
||||
print(f"Empty synonyms: {empty_syn}")
|
||||
print(f"Verses with purport: {has_purport}")
|
||||
print(f"Verses without purport: {len(verses) - has_purport}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+32
-2
@@ -555,8 +555,38 @@ def extract_verses(soup: BeautifulSoup, max_canto=99, max_chapter=999, max_verse
|
||||
if state == "SUMMARY":
|
||||
continue
|
||||
|
||||
# --- TEXT N header (p21) ---
|
||||
# --- TEXT N header (p21) — authoritative verse number ---
|
||||
# The p20 reference sometimes has wrong verse numbers (duplicates).
|
||||
# The p21 TEXT header always has the correct verse number.
|
||||
if cls == TEXT_HEADER_CLASS and current_verse:
|
||||
m = re.match(r"TEXTS?\s+(\d+)(?:\s*[-–]\s*(\d+))?", text, re.IGNORECASE)
|
||||
if m:
|
||||
correct_verse = int(m.group(1))
|
||||
correct_end = int(m.group(2)) if m.group(2) else None
|
||||
|
||||
# Fix the verse number if p20 was wrong
|
||||
if current_verse["verse"] != correct_verse:
|
||||
current_verse["verse"] = correct_verse
|
||||
|
||||
# Fix verse_end
|
||||
if correct_end is not None:
|
||||
current_verse["verse_end"] = correct_end
|
||||
elif (
|
||||
"verse_end" in current_verse
|
||||
and current_verse.get("verse_end") != correct_end
|
||||
):
|
||||
if correct_end is None and "verse_end" in current_verse:
|
||||
del current_verse["verse_end"]
|
||||
|
||||
# Rebuild the clean reference with corrected numbers
|
||||
canto = current_verse["canto"]
|
||||
chapter = current_verse["chapter"]
|
||||
if correct_end is not None:
|
||||
current_verse["reference"] = (
|
||||
f"SB {canto}.{chapter}.{correct_verse}-{correct_end}"
|
||||
)
|
||||
else:
|
||||
current_verse["reference"] = f"SB {canto}.{chapter}.{correct_verse}"
|
||||
continue
|
||||
|
||||
# --- TEXT label (p22) ---
|
||||
@@ -691,7 +721,7 @@ def write_json(data, filepath):
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "sb.html")
|
||||
output_dir = os.path.join(base_dir, "app", "lib", "books", "sb")
|
||||
output_dir = os.path.join(base_dir, "lib", "jsons", "books", "sb")
|
||||
|
||||
print(f"Reading {html_path}...")
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
|
||||
+85
-6
@@ -30,7 +30,7 @@ if (!DATABASE_URL) {
|
||||
}
|
||||
|
||||
const BASE_DIR = path.resolve(__dirname, "..");
|
||||
const LIB_DIR = path.join(BASE_DIR, "app", "lib");
|
||||
const LIB_DIR = path.join(BASE_DIR, "lib", "jsons");
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
@@ -95,6 +95,18 @@ type CcVerse = {
|
||||
purport: string[];
|
||||
};
|
||||
|
||||
type Sb1962Verse = {
|
||||
book_title: string;
|
||||
chapter: number;
|
||||
verse: number;
|
||||
verse_end?: number;
|
||||
reference: string;
|
||||
transliteration: string;
|
||||
synonyms: { word: string; meanings: string[] }[];
|
||||
translation: string;
|
||||
purport: string[];
|
||||
};
|
||||
|
||||
type GlossaryData = {
|
||||
title: string;
|
||||
description: string;
|
||||
@@ -328,6 +340,69 @@ async function seedCc(db: ReturnType<MongoClient["db"]>) {
|
||||
console.log(` ✅ CC: ${totalCount} verses`);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// SB 1962
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function seedSb1962(db: ReturnType<MongoClient["db"]>) {
|
||||
const introPath = path.join(LIB_DIR, "books", "sb-1962", "intro.json");
|
||||
if (!fs.existsSync(introPath)) {
|
||||
console.log(" ⏭ SB 1962 intro.json not found, skipping");
|
||||
return;
|
||||
}
|
||||
|
||||
const intro = readJson(introPath);
|
||||
|
||||
const metaCol = db.collection("book_meta");
|
||||
await metaCol.deleteMany({ book: "sb1962" });
|
||||
await metaCol.insertOne({ book: "sb1962", data: intro });
|
||||
console.log(" 📖 SB 1962 book meta inserted");
|
||||
|
||||
const versesCol = db.collection("verses");
|
||||
await versesCol.deleteMany({ book: "sb1962" });
|
||||
|
||||
const chaptersDir = path.join(LIB_DIR, "books", "sb-1962", "chapters");
|
||||
if (!fs.existsSync(chaptersDir)) {
|
||||
console.log(" ⏭ SB 1962 chapters dir not found, skipping verses");
|
||||
return;
|
||||
}
|
||||
|
||||
const chapterDirs = fs
|
||||
.readdirSync(chaptersDir)
|
||||
.filter((d) => fs.statSync(path.join(chaptersDir, d)).isDirectory())
|
||||
.sort();
|
||||
|
||||
const docs: Record<string, unknown>[] = [];
|
||||
|
||||
for (const chDir of chapterDirs) {
|
||||
const versesDir = path.join(chaptersDir, chDir, "verses");
|
||||
const verses = readAllJsonInDir(versesDir) as Sb1962Verse[];
|
||||
|
||||
for (const v of verses) {
|
||||
docs.push({
|
||||
book: "sb1962",
|
||||
division: null,
|
||||
chapter: v.chapter,
|
||||
verse: v.verse,
|
||||
verseEnd: v.verse_end ?? null,
|
||||
reference: v.reference,
|
||||
bookTitle: v.book_title,
|
||||
devanagari: "",
|
||||
bengali: null,
|
||||
transliteration: v.transliteration,
|
||||
synonyms: v.synonyms,
|
||||
translation: v.translation,
|
||||
purport: v.purport,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (docs.length > 0) {
|
||||
await versesCol.insertMany(docs, { ordered: false });
|
||||
}
|
||||
console.log(` ✅ SB 1962: ${docs.length} verses`);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Glossary
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -396,23 +471,27 @@ async function main() {
|
||||
|
||||
const start = Date.now();
|
||||
|
||||
console.log("[1/5] Bhagavad-gītā");
|
||||
console.log("[1/6] Bhagavad-gītā");
|
||||
await seedBg(db);
|
||||
console.log();
|
||||
|
||||
console.log("[2/5] Śrīmad-Bhāgavatam");
|
||||
console.log("[2/6] Śrīmad-Bhāgavatam");
|
||||
await seedSb(db);
|
||||
console.log();
|
||||
|
||||
console.log("[3/5] Śrī Caitanya-caritāmṛta");
|
||||
console.log("[3/6] Śrī Caitanya-caritāmṛta");
|
||||
await seedCc(db);
|
||||
console.log();
|
||||
|
||||
console.log("[4/5] Glossary");
|
||||
console.log("[4/6] Śrīmad-Bhāgavatam (1962 Edition)");
|
||||
await seedSb1962(db);
|
||||
console.log();
|
||||
|
||||
console.log("[5/6] Glossary");
|
||||
await seedGlossary(db);
|
||||
console.log();
|
||||
|
||||
console.log("[5/5] Indexes");
|
||||
console.log("[6/6] Indexes");
|
||||
await createIndexes(db);
|
||||
console.log();
|
||||
|
||||
|
||||
Reference in New Issue
Block a user