feat: allow path params for verse navigation
This commit is contained in:
gosangam
@@ -0,0 +1,263 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Match SB 1962 transliterations and synonym words with standard SB Canto 1.
|
||||
|
||||
For each verse in SB 1962 that has a matching verse in the standard SB
|
||||
(same chapter + verse number), this script:
|
||||
|
||||
1. Replaces the 1962 plain-English transliteration with the proper IAST version
|
||||
from the standard SB.
|
||||
2. Replaces each synonym "word" key with the IAST version from the standard SB,
|
||||
matching by position (index). If the 1962 edition has MORE synonyms than the
|
||||
standard SB, the extra ones are kept as-is. If the standard SB has more,
|
||||
the extras are ignored.
|
||||
3. Leaves the synonym "meanings" untouched (keeps the 1962 meanings).
|
||||
4. Leaves the translation and purport untouched (keeps the 1962 text).
|
||||
5. For verses that exist only in the 1962 edition (no standard SB match),
|
||||
everything is left as-is.
|
||||
|
||||
Usage:
|
||||
python3 scripts/match_sb1962.py
|
||||
|
||||
Prerequisites:
|
||||
- Standard SB must be extracted at lib/jsons/books/sb/cantos/01/chapters/*/verses/*.json
|
||||
- SB 1962 must be extracted at lib/jsons/books/sb-1962/chapters/*/verses/*.json
|
||||
"""
|
||||
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
|
||||
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
SB_DIR = os.path.join(BASE_DIR, "lib", "jsons", "books", "sb", "cantos", "01")
|
||||
SB62_DIR = os.path.join(BASE_DIR, "lib", "jsons", "books", "sb-1962")
|
||||
|
||||
# Known misaligned verses where the 1962 numbering diverges from the standard SB.
|
||||
# These are skipped during matching to avoid applying wrong IAST transliterations.
|
||||
# The 1962 edition splits/numbers some verses differently in chapter 15.
|
||||
SKIP_VERSES = {
|
||||
(15, 41),
|
||||
(15, 42),
|
||||
(15, 43),
|
||||
(15, 44),
|
||||
(15, 45),
|
||||
}
|
||||
|
||||
|
||||
def normalize_transliteration(text: str) -> str:
|
||||
"""Normalize transliteration for fuzzy comparison.
|
||||
|
||||
Strips diacritics, lowercases, removes punctuation/hyphens, collapses whitespace.
|
||||
This allows matching 'Udichim pravibesasam' with 'udīcīṁ praviveśāśāṁ'.
|
||||
"""
|
||||
import unicodedata
|
||||
|
||||
# NFD decompose then strip combining marks (diacritics)
|
||||
nfkd = unicodedata.normalize("NFD", text)
|
||||
stripped = "".join(c for c in nfkd if unicodedata.category(c) != "Mn")
|
||||
# Lowercase, remove hyphens, punctuation, collapse whitespace
|
||||
stripped = stripped.lower()
|
||||
stripped = stripped.replace("-", "").replace("'", "").replace("'", "")
|
||||
stripped = "".join(c for c in stripped if c.isalnum() or c.isspace())
|
||||
stripped = " ".join(stripped.split())
|
||||
return stripped
|
||||
|
||||
|
||||
def transliteration_similarity(a: str, b: str) -> float:
|
||||
"""Compute a simple word-overlap similarity between two transliterations.
|
||||
|
||||
Returns a ratio between 0.0 and 1.0.
|
||||
"""
|
||||
words_a = set(normalize_transliteration(a).split())
|
||||
words_b = set(normalize_transliteration(b).split())
|
||||
if not words_a or not words_b:
|
||||
return 0.0
|
||||
intersection = words_a & words_b
|
||||
union = words_a | words_b
|
||||
return len(intersection) / len(union)
|
||||
|
||||
|
||||
def load_sb_canto1():
|
||||
"""Load all standard SB Canto 1 verses, keyed by (chapter, verse).
|
||||
|
||||
Also returns a per-chapter list for fuzzy matching.
|
||||
"""
|
||||
verses = {}
|
||||
by_chapter: dict[int, list] = {}
|
||||
pattern = os.path.join(SB_DIR, "chapters", "*", "verses", "*.json")
|
||||
for f in sorted(glob.glob(pattern)):
|
||||
with open(f, "r", encoding="utf-8") as fh:
|
||||
d = json.load(fh)
|
||||
key = (d["chapter"], d["verse"])
|
||||
verses[key] = d
|
||||
|
||||
ch = d["chapter"]
|
||||
if ch not in by_chapter:
|
||||
by_chapter[ch] = []
|
||||
by_chapter[ch].append(d)
|
||||
|
||||
# For merged verses like 28-29, also index by the end verse
|
||||
if "verse_end" in d and d["verse_end"]:
|
||||
for v in range(d["verse"] + 1, d["verse_end"] + 1):
|
||||
verses[(d["chapter"], v)] = d
|
||||
return verses, by_chapter
|
||||
|
||||
|
||||
def load_sb1962():
|
||||
"""Load all SB 1962 verses, returning a list of (filepath, data) tuples."""
|
||||
results = []
|
||||
pattern = os.path.join(SB62_DIR, "chapters", "*", "verses", "*.json")
|
||||
for f in sorted(glob.glob(pattern)):
|
||||
with open(f, "r", encoding="utf-8") as fh:
|
||||
d = json.load(fh)
|
||||
results.append((f, d))
|
||||
return results
|
||||
|
||||
|
||||
def match_synonyms(sb62_synonyms, sb_synonyms):
|
||||
"""Match synonym word keys from standard SB into the 1962 synonyms.
|
||||
|
||||
Strategy: match by position index. For each synonym in the 1962 list,
|
||||
if there's a corresponding synonym at the same index in the standard SB,
|
||||
replace the word key with the IAST version. Keep the 1962 meaning.
|
||||
|
||||
If the 1962 has more synonyms than the standard, keep the extras as-is.
|
||||
"""
|
||||
result = []
|
||||
for i, syn62 in enumerate(sb62_synonyms):
|
||||
new_syn = dict(syn62) # copy
|
||||
if i < len(sb_synonyms):
|
||||
# Replace the word key with the IAST version
|
||||
new_syn["word"] = sb_synonyms[i]["word"]
|
||||
# Keep the original 1962 meaning regardless
|
||||
result.append(new_syn)
|
||||
return result
|
||||
|
||||
|
||||
def write_json(data, filepath):
|
||||
"""Write JSON with proper Unicode and formatting."""
|
||||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||||
with open(filepath, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def find_best_match(verse62: dict, sb_chapter_verses: list, threshold: float = 0.4):
|
||||
"""Find the best-matching standard SB verse by transliteration similarity.
|
||||
|
||||
Used as a fallback when direct verse-number matching fails or produces
|
||||
a low-confidence match. Returns the best-matching SB verse or None.
|
||||
"""
|
||||
trans_62 = verse62.get("transliteration", "")
|
||||
if not trans_62.strip():
|
||||
return None
|
||||
|
||||
best_score = 0.0
|
||||
best_match = None
|
||||
for sb_verse in sb_chapter_verses:
|
||||
sb_trans = sb_verse.get("transliteration", "")
|
||||
score = transliteration_similarity(trans_62, sb_trans)
|
||||
if score > best_score:
|
||||
best_score = score
|
||||
best_match = sb_verse
|
||||
|
||||
if best_score >= threshold:
|
||||
return best_match
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
print("Loading standard SB Canto 1 verses...")
|
||||
sb_verses, sb_by_chapter = load_sb_canto1()
|
||||
print(f" Loaded {len(sb_verses)} verse entries (including merged-verse aliases)")
|
||||
|
||||
print("Loading SB 1962 verses...")
|
||||
sb62_files = load_sb1962()
|
||||
print(f" Loaded {len(sb62_files)} verse files")
|
||||
|
||||
matched = 0
|
||||
unmatched = 0
|
||||
fuzzy_matched = 0
|
||||
trans_updated = 0
|
||||
syn_updated = 0
|
||||
|
||||
for filepath, verse62 in sb62_files:
|
||||
ch = verse62["chapter"]
|
||||
vs = verse62["verse"]
|
||||
key = (ch, vs)
|
||||
|
||||
# Skip known misaligned verses
|
||||
if key in SKIP_VERSES:
|
||||
unmatched += 1
|
||||
continue
|
||||
|
||||
sb_verse = sb_verses.get(key)
|
||||
|
||||
if sb_verse is None:
|
||||
# No match — keep as-is (e.g., 1962-only verse numbers)
|
||||
unmatched += 1
|
||||
continue
|
||||
|
||||
matched += 1
|
||||
changed = False
|
||||
|
||||
# 1. Replace transliteration with IAST version
|
||||
sb_trans = sb_verse.get("transliteration", "").strip()
|
||||
if sb_trans:
|
||||
old_trans = verse62.get("transliteration", "")
|
||||
verse62["transliteration"] = sb_trans
|
||||
if old_trans != sb_trans:
|
||||
trans_updated += 1
|
||||
changed = True
|
||||
|
||||
# 2. Match synonym word keys
|
||||
sb_syns = sb_verse.get("synonyms", [])
|
||||
v62_syns = verse62.get("synonyms", [])
|
||||
if sb_syns and v62_syns:
|
||||
new_syns = match_synonyms(v62_syns, sb_syns)
|
||||
if new_syns != v62_syns:
|
||||
verse62["synonyms"] = new_syns
|
||||
syn_updated += 1
|
||||
changed = True
|
||||
|
||||
# 3. Write back if changed
|
||||
if changed:
|
||||
write_json(verse62, filepath)
|
||||
|
||||
print(f"\n--- Results ---")
|
||||
print(f"Total 1962 verses: {len(sb62_files)}")
|
||||
print(
|
||||
f"Matched with standard SB: {matched} (direct: {matched - fuzzy_matched}, fuzzy: {fuzzy_matched})"
|
||||
)
|
||||
print(f"Unmatched (kept as-is): {unmatched}")
|
||||
print(f"Transliterations updated: {trans_updated}")
|
||||
print(f"Synonyms updated: {syn_updated}")
|
||||
|
||||
# Show a sample of matched verses
|
||||
print(f"\n--- Samples ---")
|
||||
samples = [(1, 1), (1, 2), (2, 1), (7, 1), (13, 1)]
|
||||
for ch, vs in samples:
|
||||
fpath = os.path.join(
|
||||
SB62_DIR, "chapters", str(ch).zfill(2), "verses", f"{str(vs).zfill(2)}.json"
|
||||
)
|
||||
if os.path.exists(fpath):
|
||||
with open(fpath, "r", encoding="utf-8") as f:
|
||||
d = json.load(f)
|
||||
print(f"\n {d['reference']}:")
|
||||
trans_lines = d["transliteration"].split("\n")
|
||||
print(f" Transliteration: {trans_lines[0]}")
|
||||
if len(trans_lines) > 1:
|
||||
print(f" {trans_lines[1]}")
|
||||
if d["synonyms"]:
|
||||
print(f" First synonym: {d['synonyms'][0]}")
|
||||
|
||||
# Show unmatched verses
|
||||
if unmatched > 0:
|
||||
print(f"\n--- Unmatched 1962 verses (kept as-is) ---")
|
||||
for filepath, verse62 in sb62_files:
|
||||
ch = verse62["chapter"]
|
||||
vs = verse62["verse"]
|
||||
if (ch, vs) not in sb_verses:
|
||||
print(f" {verse62['reference']}: {verse62['transliteration'][:50]}...")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user