264 lines
9.0 KiB
Python
264 lines
9.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Match SB 1962 transliterations and synonym words with standard SB Canto 1.
|
|
|
|
For each verse in SB 1962 that has a matching verse in the standard SB
|
|
(same chapter + verse number), this script:
|
|
|
|
1. Replaces the 1962 plain-English transliteration with the proper IAST version
|
|
from the standard SB.
|
|
2. Replaces each synonym "word" key with the IAST version from the standard SB,
|
|
matching by position (index). If the 1962 edition has MORE synonyms than the
|
|
standard SB, the extra ones are kept as-is. If the standard SB has more,
|
|
the extras are ignored.
|
|
3. Leaves the synonym "meanings" untouched (keeps the 1962 meanings).
|
|
4. Leaves the translation and purport untouched (keeps the 1962 text).
|
|
5. For verses that exist only in the 1962 edition (no standard SB match),
|
|
everything is left as-is.
|
|
|
|
Usage:
|
|
python3 scripts/match_sb1962.py
|
|
|
|
Prerequisites:
|
|
- Standard SB must be extracted at lib/jsons/books/sb/cantos/01/chapters/*/verses/*.json
|
|
- SB 1962 must be extracted at lib/jsons/books/sb-1962/chapters/*/verses/*.json
|
|
"""
|
|
|
|
import glob
|
|
import json
|
|
import os
|
|
|
|
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
SB_DIR = os.path.join(BASE_DIR, "lib", "jsons", "books", "sb", "cantos", "01")
|
|
SB62_DIR = os.path.join(BASE_DIR, "lib", "jsons", "books", "sb-1962")
|
|
|
|
# Known misaligned verses where the 1962 numbering diverges from the standard SB.
|
|
# These are skipped during matching to avoid applying wrong IAST transliterations.
|
|
# The 1962 edition splits/numbers some verses differently in chapter 15.
|
|
SKIP_VERSES = {
|
|
(15, 41),
|
|
(15, 42),
|
|
(15, 43),
|
|
(15, 44),
|
|
(15, 45),
|
|
}
|
|
|
|
|
|
def normalize_transliteration(text: str) -> str:
|
|
"""Normalize transliteration for fuzzy comparison.
|
|
|
|
Strips diacritics, lowercases, removes punctuation/hyphens, collapses whitespace.
|
|
This allows matching 'Udichim pravibesasam' with 'udīcīṁ praviveśāśāṁ'.
|
|
"""
|
|
import unicodedata
|
|
|
|
# NFD decompose then strip combining marks (diacritics)
|
|
nfkd = unicodedata.normalize("NFD", text)
|
|
stripped = "".join(c for c in nfkd if unicodedata.category(c) != "Mn")
|
|
# Lowercase, remove hyphens, punctuation, collapse whitespace
|
|
stripped = stripped.lower()
|
|
stripped = stripped.replace("-", "").replace("'", "").replace("'", "")
|
|
stripped = "".join(c for c in stripped if c.isalnum() or c.isspace())
|
|
stripped = " ".join(stripped.split())
|
|
return stripped
|
|
|
|
|
|
def transliteration_similarity(a: str, b: str) -> float:
|
|
"""Compute a simple word-overlap similarity between two transliterations.
|
|
|
|
Returns a ratio between 0.0 and 1.0.
|
|
"""
|
|
words_a = set(normalize_transliteration(a).split())
|
|
words_b = set(normalize_transliteration(b).split())
|
|
if not words_a or not words_b:
|
|
return 0.0
|
|
intersection = words_a & words_b
|
|
union = words_a | words_b
|
|
return len(intersection) / len(union)
|
|
|
|
|
|
def load_sb_canto1():
|
|
"""Load all standard SB Canto 1 verses, keyed by (chapter, verse).
|
|
|
|
Also returns a per-chapter list for fuzzy matching.
|
|
"""
|
|
verses = {}
|
|
by_chapter: dict[int, list] = {}
|
|
pattern = os.path.join(SB_DIR, "chapters", "*", "verses", "*.json")
|
|
for f in sorted(glob.glob(pattern)):
|
|
with open(f, "r", encoding="utf-8") as fh:
|
|
d = json.load(fh)
|
|
key = (d["chapter"], d["verse"])
|
|
verses[key] = d
|
|
|
|
ch = d["chapter"]
|
|
if ch not in by_chapter:
|
|
by_chapter[ch] = []
|
|
by_chapter[ch].append(d)
|
|
|
|
# For merged verses like 28-29, also index by the end verse
|
|
if "verse_end" in d and d["verse_end"]:
|
|
for v in range(d["verse"] + 1, d["verse_end"] + 1):
|
|
verses[(d["chapter"], v)] = d
|
|
return verses, by_chapter
|
|
|
|
|
|
def load_sb1962():
|
|
"""Load all SB 1962 verses, returning a list of (filepath, data) tuples."""
|
|
results = []
|
|
pattern = os.path.join(SB62_DIR, "chapters", "*", "verses", "*.json")
|
|
for f in sorted(glob.glob(pattern)):
|
|
with open(f, "r", encoding="utf-8") as fh:
|
|
d = json.load(fh)
|
|
results.append((f, d))
|
|
return results
|
|
|
|
|
|
def match_synonyms(sb62_synonyms, sb_synonyms):
|
|
"""Match synonym word keys from standard SB into the 1962 synonyms.
|
|
|
|
Strategy: match by position index. For each synonym in the 1962 list,
|
|
if there's a corresponding synonym at the same index in the standard SB,
|
|
replace the word key with the IAST version. Keep the 1962 meaning.
|
|
|
|
If the 1962 has more synonyms than the standard, keep the extras as-is.
|
|
"""
|
|
result = []
|
|
for i, syn62 in enumerate(sb62_synonyms):
|
|
new_syn = dict(syn62) # copy
|
|
if i < len(sb_synonyms):
|
|
# Replace the word key with the IAST version
|
|
new_syn["word"] = sb_synonyms[i]["word"]
|
|
# Keep the original 1962 meaning regardless
|
|
result.append(new_syn)
|
|
return result
|
|
|
|
|
|
def write_json(data, filepath):
|
|
"""Write JSON with proper Unicode and formatting."""
|
|
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
|
with open(filepath, "w", encoding="utf-8") as f:
|
|
json.dump(data, f, ensure_ascii=False, indent=2)
|
|
|
|
|
|
def find_best_match(verse62: dict, sb_chapter_verses: list, threshold: float = 0.4):
|
|
"""Find the best-matching standard SB verse by transliteration similarity.
|
|
|
|
Used as a fallback when direct verse-number matching fails or produces
|
|
a low-confidence match. Returns the best-matching SB verse or None.
|
|
"""
|
|
trans_62 = verse62.get("transliteration", "")
|
|
if not trans_62.strip():
|
|
return None
|
|
|
|
best_score = 0.0
|
|
best_match = None
|
|
for sb_verse in sb_chapter_verses:
|
|
sb_trans = sb_verse.get("transliteration", "")
|
|
score = transliteration_similarity(trans_62, sb_trans)
|
|
if score > best_score:
|
|
best_score = score
|
|
best_match = sb_verse
|
|
|
|
if best_score >= threshold:
|
|
return best_match
|
|
return None
|
|
|
|
|
|
def main():
|
|
print("Loading standard SB Canto 1 verses...")
|
|
sb_verses, sb_by_chapter = load_sb_canto1()
|
|
print(f" Loaded {len(sb_verses)} verse entries (including merged-verse aliases)")
|
|
|
|
print("Loading SB 1962 verses...")
|
|
sb62_files = load_sb1962()
|
|
print(f" Loaded {len(sb62_files)} verse files")
|
|
|
|
matched = 0
|
|
unmatched = 0
|
|
fuzzy_matched = 0
|
|
trans_updated = 0
|
|
syn_updated = 0
|
|
|
|
for filepath, verse62 in sb62_files:
|
|
ch = verse62["chapter"]
|
|
vs = verse62["verse"]
|
|
key = (ch, vs)
|
|
|
|
# Skip known misaligned verses
|
|
if key in SKIP_VERSES:
|
|
unmatched += 1
|
|
continue
|
|
|
|
sb_verse = sb_verses.get(key)
|
|
|
|
if sb_verse is None:
|
|
# No match — keep as-is (e.g., 1962-only verse numbers)
|
|
unmatched += 1
|
|
continue
|
|
|
|
matched += 1
|
|
changed = False
|
|
|
|
# 1. Replace transliteration with IAST version
|
|
sb_trans = sb_verse.get("transliteration", "").strip()
|
|
if sb_trans:
|
|
old_trans = verse62.get("transliteration", "")
|
|
verse62["transliteration"] = sb_trans
|
|
if old_trans != sb_trans:
|
|
trans_updated += 1
|
|
changed = True
|
|
|
|
# 2. Match synonym word keys
|
|
sb_syns = sb_verse.get("synonyms", [])
|
|
v62_syns = verse62.get("synonyms", [])
|
|
if sb_syns and v62_syns:
|
|
new_syns = match_synonyms(v62_syns, sb_syns)
|
|
if new_syns != v62_syns:
|
|
verse62["synonyms"] = new_syns
|
|
syn_updated += 1
|
|
changed = True
|
|
|
|
# 3. Write back if changed
|
|
if changed:
|
|
write_json(verse62, filepath)
|
|
|
|
print(f"\n--- Results ---")
|
|
print(f"Total 1962 verses: {len(sb62_files)}")
|
|
print(
|
|
f"Matched with standard SB: {matched} (direct: {matched - fuzzy_matched}, fuzzy: {fuzzy_matched})"
|
|
)
|
|
print(f"Unmatched (kept as-is): {unmatched}")
|
|
print(f"Transliterations updated: {trans_updated}")
|
|
print(f"Synonyms updated: {syn_updated}")
|
|
|
|
# Show a sample of matched verses
|
|
print(f"\n--- Samples ---")
|
|
samples = [(1, 1), (1, 2), (2, 1), (7, 1), (13, 1)]
|
|
for ch, vs in samples:
|
|
fpath = os.path.join(
|
|
SB62_DIR, "chapters", str(ch).zfill(2), "verses", f"{str(vs).zfill(2)}.json"
|
|
)
|
|
if os.path.exists(fpath):
|
|
with open(fpath, "r", encoding="utf-8") as f:
|
|
d = json.load(f)
|
|
print(f"\n {d['reference']}:")
|
|
trans_lines = d["transliteration"].split("\n")
|
|
print(f" Transliteration: {trans_lines[0]}")
|
|
if len(trans_lines) > 1:
|
|
print(f" {trans_lines[1]}")
|
|
if d["synonyms"]:
|
|
print(f" First synonym: {d['synonyms'][0]}")
|
|
|
|
# Show unmatched verses
|
|
if unmatched > 0:
|
|
print(f"\n--- Unmatched 1962 verses (kept as-is) ---")
|
|
for filepath, verse62 in sb62_files:
|
|
ch = verse62["chapter"]
|
|
vs = verse62["verse"]
|
|
if (ch, vs) not in sb_verses:
|
|
print(f" {verse62['reference']}: {verse62['transliteration'][:50]}...")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|