1333 lines
43 KiB
Python
1333 lines
43 KiB
Python
#!/usr/bin/env python3
|
||
"""Extract Songs of the Vaiṣṇava Ācāryas (1972) from other.html.
|
||
|
||
Hierarchical sub-chapter extraction: each song within a part becomes its OWN
|
||
chapter JSON file with a parent-child relationship.
|
||
|
||
Parent chapters (Parts 1–4) are containers only — no content of their own.
|
||
|
||
Numbering: Part 1, Song 1 → slug "1-1", Part 2, Song 3 → slug "2-3", etc.
|
||
|
||
Song boundaries are delimited by p49 bookmarks with text like:
|
||
"SVA 1: Śrī Guru Praṇāma"
|
||
"SVA 2: Aruṇodaya-kīrtana"
|
||
|
||
Output:
|
||
lib/jsons/books/sva/intro.json — Book metadata with hierarchical chapters
|
||
lib/jsons/books/sva/intro/*.json — Intro segments (front-pages, foreword, introduction)
|
||
lib/jsons/books/sva/chapters/*.json — Sub-chapter files (1-1.json, 1-2.json, 2-1.json, ...)
|
||
|
||
Usage:
|
||
python3 scripts/sva_extract.py
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import re
|
||
import sys
|
||
from collections import defaultdict
|
||
|
||
from bs4 import BeautifulSoup, NavigableString, Tag
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# ScaGoudy → IAST Unicode mapping
|
||
# ---------------------------------------------------------------------------
|
||
SCAGOUDY_TO_IAST = {
|
||
"ä": "ā",
|
||
"Ä": "Ā",
|
||
"å": "ṛ",
|
||
"Å": "Ṛ",
|
||
"ñ": "ṣ",
|
||
"Ñ": "Ṣ",
|
||
"ö": "ṭ",
|
||
"Ö": "Ṭ",
|
||
"ë": "ṇ",
|
||
"Ë": "Ṇ",
|
||
"ò": "ḍ",
|
||
"Ò": "Ḍ",
|
||
"ç": "ś",
|
||
"Ç": "Ś",
|
||
"ì": "ṅ",
|
||
"Ì": "Ṅ",
|
||
"ï": "ñ",
|
||
"Ï": "Ñ",
|
||
"ü": "ū",
|
||
"Ü": "Ū",
|
||
"é": "ī",
|
||
"É": "Ī",
|
||
"à": "ṁ",
|
||
"À": "Ṁ",
|
||
"ù": "ḥ",
|
||
"Ù": "Ḥ",
|
||
"è": "ṝ",
|
||
"û": "ṁl",
|
||
"ÿ": "ḷ",
|
||
}
|
||
|
||
|
||
def decode_scagoudy(text: str) -> str:
|
||
"""Replace ScaGoudy-encoded characters with proper IAST Unicode."""
|
||
for src, dst in SCAGOUDY_TO_IAST.items():
|
||
text = text.replace(src, dst)
|
||
return text
|
||
|
||
|
||
def decode_html_content(tag: Tag) -> str:
|
||
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
|
||
inner = "".join(str(child) for child in tag.children)
|
||
inner = decode_scagoudy(inner)
|
||
# Remove span tags with CSS classes (RTF conversion artifacts)
|
||
inner = re.sub(
|
||
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
|
||
)
|
||
# Clean up Apple-tab-span leftovers (tab characters)
|
||
inner = inner.replace("\t", " ")
|
||
# Force valid UTF-8 by round-tripping through encode/decode with replacement
|
||
inner = inner.encode("utf-8", errors="replace").decode("utf-8", errors="replace")
|
||
# Strip null bytes and other control characters that break BSON
|
||
inner = re.sub(r"[\x00-\x08\x0b\x0c\x0e-\x1f]", "", inner)
|
||
# Remove lone surrogates (U+D800–U+DFFF) that are invalid in BSON/MongoDB
|
||
inner = re.sub(r"[\ud800-\udfff]", "", inner)
|
||
# Merge adjacent bold tags: </b><b> → (nothing)
|
||
for _ in range(5):
|
||
inner = re.sub(r"</b>\s*<b>", "", inner)
|
||
# Remove empty bold and italic tags
|
||
inner = re.sub(r"<b>\s*</b>", "", inner)
|
||
inner = re.sub(r"<i>\s*</i>", "", inner)
|
||
# Normalize whitespace
|
||
inner = re.sub(r"\s+", " ", inner).strip()
|
||
return inner
|
||
|
||
|
||
def decode_text(tag: Tag) -> str:
|
||
"""Get plain text from a tag with ScaGoudy decoding."""
|
||
return decode_scagoudy(tag.get_text().strip())
|
||
|
||
|
||
def get_css_class(tag: Tag) -> str:
|
||
"""Get the first CSS class of a tag, or empty string."""
|
||
classes = tag.get("class", [])
|
||
return classes[0] if classes else ""
|
||
|
||
|
||
def is_empty(tag: Tag) -> bool:
|
||
"""Check if a tag is empty or whitespace-only."""
|
||
text = tag.get_text().strip()
|
||
return not text or text == "\xa0"
|
||
|
||
|
||
def is_dash_line(text: str) -> bool:
|
||
"""Return True if text is only dashes, spaces, and/or whitespace."""
|
||
cleaned = text.strip()
|
||
if not cleaned:
|
||
return False
|
||
return all(c in "-—–\u2014\u2013 \t" for c in cleaned)
|
||
|
||
|
||
def is_inside_table(tag: Tag) -> bool:
|
||
"""Check if a tag is nested inside a <table> element."""
|
||
parent = tag.parent
|
||
while parent:
|
||
if parent.name == "table":
|
||
return True
|
||
parent = parent.parent
|
||
return False
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# CSS class categorization for SVA
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# Book title header (bold, p12)
|
||
BOOK_HEADER_CLASS = "p12"
|
||
|
||
# Bookmark class — section boundaries
|
||
BOOKMARK_CLASS = "p13"
|
||
|
||
# Song bookmark / section bookmark within a part
|
||
SONG_BOOKMARK_CLASS = "p49"
|
||
|
||
# Front pages text
|
||
FRONT_PAGES_CLASS = "p18"
|
||
|
||
# Body paragraph classes
|
||
BODY_CLASSES = frozenset(["p6", "p48", "p140", "p204", "p44"])
|
||
|
||
# Section headings (bold)
|
||
SECTION_HEADING_CLASS = "p31"
|
||
|
||
# Song title headings (bold)
|
||
SONG_TITLE_CLASS = "p27"
|
||
|
||
# Song title variant (bold italic with author)
|
||
SONG_TITLE_VARIANT_CLASS = "p169"
|
||
|
||
# Verse line classes (italic, centered)
|
||
VERSE_PADA_CLASSES = frozenset(["p5", "p53", "p24", "p66", "p255", "p134"])
|
||
|
||
# Verse number classes (e.g. "3)", "4)", "[1]") — centered but NOT verse text
|
||
VERSE_NUMBER_CLASSES = frozenset(["p248", "p30"])
|
||
|
||
# Translation variant classes (bold, centered) — in addition to p56
|
||
TRANSLATION_VARIANT_CLASSES = frozenset(["p86", "p240", "p242"])
|
||
|
||
# Reference/purport note classes that should be centered
|
||
CENTERED_NOTE_CLASSES = frozenset(["p241", "p22"])
|
||
|
||
# Synonyms text
|
||
SYNONYMS_CLASS = "p54"
|
||
|
||
# Translation text (bold)
|
||
TRANSLATION_CLASS = "p56"
|
||
|
||
# "SYNONYMS" header
|
||
SYNONYMS_HEADER_CLASS = "p165"
|
||
|
||
# "TRANSLATION" header
|
||
TRANSLATION_HEADER_CLASS = "p55"
|
||
|
||
# Mantra text
|
||
MANTRA_CLASS = "p19"
|
||
|
||
# Note text
|
||
NOTE_CLASS = "p204"
|
||
|
||
# Attribution
|
||
ATTRIBUTION_CLASS = "p129"
|
||
|
||
# Date/location
|
||
DATE_CLASS = "p67"
|
||
|
||
# Signature block
|
||
SIGNATURE_CLASS = "p222"
|
||
|
||
# Bold section header
|
||
BOLD_HEADER_CLASS = "p227"
|
||
|
||
# Bracketed notes
|
||
BRACKET_NOTE_CLASS = "p228"
|
||
|
||
# Disciplic succession cell content
|
||
DISC_SUCC_CLASS = "p229"
|
||
|
||
# Reference notes
|
||
REFERENCE_CLASSES = frozenset(
|
||
[
|
||
"p187",
|
||
"p231",
|
||
"p233",
|
||
"p237",
|
||
"p32",
|
||
"p251",
|
||
"p203",
|
||
"p246",
|
||
"p249",
|
||
"p254",
|
||
"p241",
|
||
]
|
||
)
|
||
|
||
# Bold text classes
|
||
BOLD_TEXT_CLASSES = frozenset(["p236", "p238"])
|
||
|
||
# Synonym notes
|
||
SYNONYM_NOTES_CLASS = "p247"
|
||
|
||
# Bold headings
|
||
BOLD_HEADING_CLASSES = frozenset(["p224", "p223"])
|
||
|
||
# Part heading (bold, for song subsections)
|
||
PART_HEADING_CLASS = "p22"
|
||
|
||
# Sub-translation / inline bold classes
|
||
INLINE_BOLD_CLASSES = frozenset(["p86", "p240", "p242", "p243", "p245", "p258"])
|
||
|
||
# Verse number marker
|
||
VERSE_NUMBER_CLASS = "p30"
|
||
|
||
# Song commentary / purport headings
|
||
PURPORT_HEADING_CLASS = "p96"
|
||
|
||
# Purport body text
|
||
PURPORT_BODY_CLASS = "p140"
|
||
|
||
# Additional verse line classes in tables
|
||
TABLE_VERSE_CLASSES = frozenset(["p248", "p255", "p5", "p6"])
|
||
|
||
# Verse numbering markers
|
||
VERSE_MARKER_CLASSES = frozenset(["p252", "p253"])
|
||
|
||
# Classes to skip entirely (empty/break/decorative)
|
||
SKIP_CLASSES = frozenset(
|
||
[
|
||
"p15",
|
||
"p120",
|
||
"p225",
|
||
"p226",
|
||
"p219",
|
||
"p230",
|
||
"p232",
|
||
"p234",
|
||
"p135",
|
||
"p183",
|
||
"p172",
|
||
"p179",
|
||
"p260",
|
||
]
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Clean parent titles (pre-decoded IAST)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
CLEAN_PARENT_TITLES = {
|
||
1: "Prayers and Praṇāmas",
|
||
2: "Songs of Śrīla Bhaktivinoda Ṭhākura",
|
||
3: "Songs of Śrīla Narottama dāsa Ṭhākura",
|
||
4: "Songs by other Ācāryas",
|
||
}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Extraction helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def find_sva_boundaries(all_elements: list) -> tuple:
|
||
"""Find the start and end element indices of the SVA content.
|
||
|
||
Start: <p class="p12"> containing "Songs of the Vaiñëava Äcäryas"
|
||
End: <p class="p12"> containing "The Hare" (Cook Book)
|
||
"""
|
||
start_idx = None
|
||
end_idx = len(all_elements)
|
||
|
||
for i, elem in enumerate(all_elements):
|
||
if elem.name != "p":
|
||
continue
|
||
cls = get_css_class(elem)
|
||
text = decode_text(elem)
|
||
|
||
# Find start
|
||
if start_idx is None:
|
||
if cls == BOOK_HEADER_CLASS and "Songs of the Vai" in text:
|
||
start_idx = i
|
||
continue
|
||
|
||
# Find end: the next p12 header (Hare Kṛṣṇa Cook Book)
|
||
if cls == BOOK_HEADER_CLASS:
|
||
end_idx = i
|
||
break
|
||
|
||
if start_idx is None:
|
||
print("ERROR: Could not find 'Songs of the Vaiṣṇava Ācāryas' header!")
|
||
sys.exit(1)
|
||
|
||
return start_idx, end_idx
|
||
|
||
|
||
def find_sva_bookmarks(all_elements: list, start_idx: int, end_idx: int) -> list:
|
||
"""Find all SVA bookmark/section tags in the SVA range.
|
||
|
||
Looks for p13 class with "SVA" text, and p49 class with "SVA" text.
|
||
Returns list of (element_index, bookmark_text, bookmark_type).
|
||
bookmark_type is "p13" or "p49".
|
||
"""
|
||
bookmarks = []
|
||
for i in range(start_idx, end_idx):
|
||
elem = all_elements[i]
|
||
if elem.name != "p":
|
||
continue
|
||
cls = get_css_class(elem)
|
||
if cls not in (BOOKMARK_CLASS, SONG_BOOKMARK_CLASS):
|
||
continue
|
||
text = decode_text(elem).strip()
|
||
if text.startswith("SVA"):
|
||
bookmarks.append((i, text, cls))
|
||
return bookmarks
|
||
|
||
|
||
def process_table_t3(table: Tag) -> list:
|
||
"""Process a t3 table (side-by-side Bengali verse + English translation).
|
||
|
||
Returns a list of HTML paragraph strings.
|
||
"""
|
||
result = []
|
||
for tr in table.find_all("tr"):
|
||
tds = tr.find_all("td")
|
||
if len(tds) < 2:
|
||
# Single-cell row — just extract content
|
||
for td in tds:
|
||
for p in td.find_all("p"):
|
||
html = decode_html_content(p)
|
||
if html and html.strip() and html.strip() != "\xa0":
|
||
result.append(html)
|
||
continue
|
||
|
||
# Left cell: Bengali verse lines
|
||
left_ps = tds[0].find_all("p")
|
||
verse_lines = []
|
||
for p in left_ps:
|
||
html = decode_html_content(p)
|
||
if html and html.strip() and html.strip() != "\xa0":
|
||
# Strip <i> tags — we'll re-wrap
|
||
stripped = re.sub(r"</?i>", "", html)
|
||
if stripped.strip():
|
||
verse_lines.append(stripped.strip())
|
||
|
||
if verse_lines:
|
||
joined = "<br>".join(verse_lines)
|
||
result.append(f'<div style="text-align:center"><i>{joined}</i></div>')
|
||
|
||
# Right cell: English translation — centered
|
||
right_ps = tds[1].find_all("p")
|
||
for p in right_ps:
|
||
html = decode_html_content(p)
|
||
if html and html.strip() and html.strip() != "\xa0":
|
||
result.append(f'<div style="text-align:center">{html}</div>')
|
||
|
||
return result
|
||
|
||
|
||
def process_table_t2(table: Tag) -> list:
|
||
"""Process a t2 table (disciplic succession table).
|
||
|
||
Extract cell content as regular paragraphs.
|
||
"""
|
||
result = []
|
||
for tr in table.find_all("tr"):
|
||
for td in tr.find_all("td"):
|
||
for p in td.find_all("p"):
|
||
html = decode_html_content(p)
|
||
if html and html.strip() and html.strip() != "\xa0":
|
||
result.append(html)
|
||
return result
|
||
|
||
|
||
def extract_paragraphs(
|
||
all_elements: list,
|
||
start_idx: int,
|
||
end_idx: int,
|
||
) -> list:
|
||
"""Extract paragraphs for a section (song, intro segment).
|
||
|
||
Handles all CSS classes within the SVA content.
|
||
Handles <table class="t3"> and <table class="t2"> elements.
|
||
Groups consecutive verse pāda classes into centered italic divs.
|
||
"""
|
||
result = []
|
||
verse_accumulator = []
|
||
|
||
# Track which <p> tags are inside tables we've already processed
|
||
processed_table_ps = set()
|
||
|
||
def flush_verse():
|
||
if verse_accumulator:
|
||
joined = "<br>".join(verse_accumulator)
|
||
result.append(f'<div style="text-align:center"><i>{joined}</i></div>')
|
||
verse_accumulator.clear()
|
||
|
||
for i in range(start_idx, end_idx):
|
||
elem = all_elements[i]
|
||
|
||
# Handle tables
|
||
if elem.name == "table":
|
||
flush_verse()
|
||
table_classes = elem.get("class", [])
|
||
|
||
# Mark all <p> inside this table as processed
|
||
for p in elem.find_all("p"):
|
||
processed_table_ps.add(id(p))
|
||
|
||
if "t3" in table_classes:
|
||
table_paras = process_table_t3(elem)
|
||
result.extend(table_paras)
|
||
elif "t2" in table_classes:
|
||
table_paras = process_table_t2(elem)
|
||
result.extend(table_paras)
|
||
else:
|
||
# Generic table — extract all paragraph content
|
||
for p in elem.find_all("p"):
|
||
html = decode_html_content(p)
|
||
if html and html.strip():
|
||
result.append(html)
|
||
continue
|
||
|
||
if elem.name != "p":
|
||
continue
|
||
|
||
# Skip if this <p> was already processed as part of a table
|
||
if id(elem) in processed_table_ps:
|
||
continue
|
||
|
||
# Skip if inside a table (safety check)
|
||
if is_inside_table(elem):
|
||
continue
|
||
|
||
cls = get_css_class(elem)
|
||
|
||
# Skip empty paragraphs
|
||
if is_empty(elem):
|
||
continue
|
||
|
||
# Skip noise classes
|
||
if cls in SKIP_CLASSES:
|
||
continue
|
||
|
||
# Skip bookmark tags (p13, p49) — these are navigation markers
|
||
if cls == BOOKMARK_CLASS:
|
||
continue
|
||
if cls == SONG_BOOKMARK_CLASS:
|
||
continue
|
||
|
||
# Skip book title header (p12)
|
||
if cls == BOOK_HEADER_CLASS:
|
||
continue
|
||
|
||
text = decode_text(elem)
|
||
|
||
# Check for dash lines — skip them
|
||
if is_dash_line(text):
|
||
continue
|
||
|
||
# Sanskrit verse pādas (p5, p53, p24, p248, p255) — accumulate
|
||
if cls in VERSE_PADA_CLASSES:
|
||
html = decode_html_content(elem)
|
||
if html:
|
||
stripped = re.sub(r"</?i>", "", html)
|
||
if stripped.strip():
|
||
verse_accumulator.append(stripped.strip())
|
||
continue
|
||
|
||
# Non-verse paragraph — flush any accumulated verse first
|
||
flush_verse()
|
||
|
||
# Short p6 content (verse numbers like "1)", "2)") should be centered
|
||
# Long p6 content (body paragraphs) stays left-aligned
|
||
if cls == "p6":
|
||
html = decode_html_content(elem)
|
||
if html:
|
||
plain = re.sub(r"<[^>]+>", "", html).strip()
|
||
# Short content that looks like a verse number or bracketed ref
|
||
if len(plain) <= 10 and re.match(r"^[\[\(]?\d+[\]\)]?\.?$", plain):
|
||
result.append(f'<div style="text-align:center">{html}</div>')
|
||
else:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Front pages text (p18)
|
||
if cls == FRONT_PAGES_CLASS:
|
||
html = decode_html_content(elem)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Section heading (p31) — bold, centered
|
||
if cls == SECTION_HEADING_CLASS:
|
||
html = decode_html_content(elem)
|
||
if html:
|
||
if not html.strip().startswith("<b>"):
|
||
html = f"<b>{html}</b>"
|
||
result.append(f'<div style="text-align:center">{html}</div>')
|
||
continue
|
||
|
||
# Song title (p27) — bold, centered
|
||
if cls == SONG_TITLE_CLASS:
|
||
html = decode_html_content(elem)
|
||
if html:
|
||
if not html.strip().startswith("<b>"):
|
||
html = f"<b>{html}</b>"
|
||
result.append(f'<div style="text-align:center">{html}</div>')
|
||
continue
|
||
|
||
# Song title variant (p169) — bold italic with author, centered
|
||
if cls == SONG_TITLE_VARIANT_CLASS:
|
||
html = decode_html_content(elem)
|
||
if html:
|
||
result.append(f'<div style="text-align:center">{html}</div>')
|
||
continue
|
||
|
||
# Synonyms (p54)
|
||
if cls == SYNONYMS_CLASS:
|
||
html = decode_html_content(elem)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Verse number heading (p248, p30) — e.g. "3)", "4)", "[1]" — center it
|
||
if cls in VERSE_NUMBER_CLASSES:
|
||
flush_verse()
|
||
html = decode_html_content(elem)
|
||
if html:
|
||
result.append(f'<div style="text-align:center">{html}</div>')
|
||
continue
|
||
|
||
# Translation variant classes (p86, p240, p242) — bold, centered
|
||
if cls in TRANSLATION_VARIANT_CLASSES:
|
||
flush_verse()
|
||
html = decode_html_content(elem)
|
||
if html:
|
||
result.append(f'<div style="text-align:center">{html}</div>')
|
||
continue
|
||
|
||
# Centered note classes (p241)
|
||
if cls in CENTERED_NOTE_CLASSES:
|
||
flush_verse()
|
||
html = decode_html_content(elem)
|
||
if html:
|
||
result.append(f'<div style="text-align:center">{html}</div>')
|
||
continue
|
||
|
||
# Translation (p56) — bold, centered
|
||
if cls == TRANSLATION_CLASS:
|
||
html = decode_html_content(elem)
|
||
if html:
|
||
result.append(f'<div style="text-align:center">{html}</div>')
|
||
continue
|
||
|
||
# SYNONYMS header (p165) — centered
|
||
if cls == SYNONYMS_HEADER_CLASS:
|
||
html = decode_html_content(elem)
|
||
if html:
|
||
result.append(f'<div style="text-align:center">{html}</div>')
|
||
continue
|
||
|
||
# TRANSLATION header (p55) — centered
|
||
if cls == TRANSLATION_HEADER_CLASS:
|
||
html = decode_html_content(elem)
|
||
if html:
|
||
result.append(f'<div style="text-align:center">{html}</div>')
|
||
continue
|
||
|
||
# Mantra text (p19)
|
||
if cls == MANTRA_CLASS:
|
||
html = decode_html_content(elem)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Attribution (p129)
|
||
if cls == ATTRIBUTION_CLASS:
|
||
html = decode_html_content(elem)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Date/location (p67)
|
||
if cls == DATE_CLASS:
|
||
html = decode_html_content(elem)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Signature block (p222)
|
||
if cls == SIGNATURE_CLASS:
|
||
html = decode_html_content(elem)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Bold section header (p227)
|
||
if cls == BOLD_HEADER_CLASS:
|
||
html = decode_html_content(elem)
|
||
if html:
|
||
if not html.strip().startswith("<b>"):
|
||
html = f"<b>{html}</b>"
|
||
result.append(html)
|
||
continue
|
||
|
||
# Bracketed notes (p228)
|
||
if cls == BRACKET_NOTE_CLASS:
|
||
html = decode_html_content(elem)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Disciplic succession cell content (p229)
|
||
if cls == DISC_SUCC_CLASS:
|
||
html = decode_html_content(elem)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Reference notes (p187, p231, p233, p237, p32, p251) — centered
|
||
if cls in REFERENCE_CLASSES:
|
||
html = decode_html_content(elem)
|
||
if html:
|
||
result.append(f'<div style="text-align:center">{html}</div>')
|
||
continue
|
||
|
||
# Bold text classes
|
||
if cls in BOLD_TEXT_CLASSES:
|
||
html = decode_html_content(elem)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Synonym note (p247) — centered
|
||
if cls == SYNONYM_NOTES_CLASS:
|
||
html = decode_html_content(elem)
|
||
if html:
|
||
result.append(f'<div style="text-align:center">{html}</div>')
|
||
continue
|
||
|
||
# Bold headings (p224, p223)
|
||
if cls in BOLD_HEADING_CLASSES:
|
||
html = decode_html_content(elem)
|
||
if html:
|
||
if not html.strip().startswith("<b>"):
|
||
html = f"<b>{html}</b>"
|
||
result.append(html)
|
||
continue
|
||
|
||
# Part heading (p22) — e.g. "part 1", "part 2"
|
||
if cls == PART_HEADING_CLASS:
|
||
html = decode_html_content(elem)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Inline bold classes (p86, p240, p242, p243, p245, p258)
|
||
if cls in INLINE_BOLD_CLASSES:
|
||
html = decode_html_content(elem)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Verse number (p30)
|
||
if cls == VERSE_NUMBER_CLASS:
|
||
html = decode_html_content(elem)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Purport heading (p96)
|
||
if cls == PURPORT_HEADING_CLASS:
|
||
html = decode_html_content(elem)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Purport body (p140)
|
||
if cls == PURPORT_BODY_CLASS:
|
||
html = decode_html_content(elem)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Verse markers (p252, p253)
|
||
if cls in VERSE_MARKER_CLASSES:
|
||
html = decode_html_content(elem)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Body paragraphs (p6, p48, p140, p204, p44)
|
||
if cls in BODY_CLASSES:
|
||
html = decode_html_content(elem)
|
||
if html:
|
||
# Detect verses disguised as body: starts with <i> and has <br>
|
||
stripped_html = html.strip()
|
||
if stripped_html.startswith("<i>") and "<br" in stripped_html:
|
||
inner = re.sub(r"</?i>", "", stripped_html)
|
||
result.append(
|
||
f'<div style="text-align:center"><i>{inner}</i></div>'
|
||
)
|
||
else:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Any other class — try to extract (be permissive)
|
||
html = decode_html_content(elem)
|
||
if html and not is_dash_line(text):
|
||
result.append(html)
|
||
|
||
# Flush any trailing verse
|
||
flush_verse()
|
||
|
||
return result
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Main extraction logic
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def extract_sva(soup: BeautifulSoup) -> dict:
|
||
"""Extract the entire Songs of the Vaiṣṇava Ācāryas from the parsed HTML.
|
||
|
||
Returns a dict with:
|
||
"intro_segments": list of intro segment dicts
|
||
"sub_chapters": list of sub-chapter dicts (one per song)
|
||
"parent_chapters": list of parent chapter info for intro.json
|
||
"""
|
||
# Get all top-level <p> and <table> elements
|
||
all_elements = list(soup.find_all(["p", "table"]))
|
||
|
||
# Step 1: Find SVA boundaries
|
||
sva_start, sva_end = find_sva_boundaries(all_elements)
|
||
print(f" SVA content: elements {sva_start} to {sva_end}")
|
||
print(f" Total elements in range: {sva_end - sva_start}")
|
||
|
||
# Step 2: Find all SVA bookmarks
|
||
bookmarks = find_sva_bookmarks(all_elements, sva_start, sva_end)
|
||
print(f" Found {len(bookmarks)} SVA bookmarks")
|
||
for pos, text, cls in bookmarks[:30]:
|
||
print(f" Elem ~{pos} [{cls}]: {text[:80]}")
|
||
if len(bookmarks) > 30:
|
||
print(f" ... and {len(bookmarks) - 30} more")
|
||
|
||
# Step 3: Categorize bookmarks
|
||
# p13 bookmarks define major sections:
|
||
# "SVA: Foreword" → foreword
|
||
# "SVA: Introduction" → introduction
|
||
# "SVA 2: ..." → Part 2 start
|
||
# "SVA 3: ..." → Part 3 start
|
||
# "SVA 4: ..." → Part 4 start
|
||
# p49 bookmarks define songs within parts:
|
||
# "SVA 1: ..." → songs in Part 1
|
||
# "SVA 2: ..." → songs in Part 2
|
||
# etc.
|
||
|
||
foreword_pos = None
|
||
introduction_pos = None
|
||
part2_pos = None
|
||
part3_pos = None
|
||
part4_pos = None
|
||
|
||
first_part1_pos = None
|
||
|
||
for pos, text, cls in bookmarks:
|
||
if cls == BOOKMARK_CLASS: # p13
|
||
if text.strip() == "SVA: Foreword":
|
||
foreword_pos = pos
|
||
elif text.strip() == "SVA: Introduction":
|
||
introduction_pos = pos
|
||
elif text.strip().startswith("SVA 2:"):
|
||
part2_pos = pos
|
||
elif text.strip().startswith("SVA 3:"):
|
||
part3_pos = pos
|
||
elif text.strip().startswith("SVA 4:"):
|
||
part4_pos = pos
|
||
|
||
if cls == SONG_BOOKMARK_CLASS: # p49
|
||
if text.strip().startswith("SVA 1:") and first_part1_pos is None:
|
||
first_part1_pos = pos
|
||
|
||
print(f"\n Section boundaries:")
|
||
print(f" Book start: elem {sva_start}")
|
||
print(
|
||
f" Foreword: {'elem ' + str(foreword_pos) if foreword_pos else 'NOT FOUND'}"
|
||
)
|
||
print(
|
||
f" Introduction: {'elem ' + str(introduction_pos) if introduction_pos else 'NOT FOUND'}"
|
||
)
|
||
print(
|
||
f" Part 1 start: {'elem ' + str(first_part1_pos) if first_part1_pos else 'NOT FOUND'}"
|
||
)
|
||
print(
|
||
f" Part 2 start: {'elem ' + str(part2_pos) if part2_pos else 'NOT FOUND'}"
|
||
)
|
||
print(
|
||
f" Part 3 start: {'elem ' + str(part3_pos) if part3_pos else 'NOT FOUND'}"
|
||
)
|
||
print(
|
||
f" Part 4 start: {'elem ' + str(part4_pos) if part4_pos else 'NOT FOUND'}"
|
||
)
|
||
print(f" Book end: elem {sva_end}")
|
||
|
||
# Step 4: Extract intro segments
|
||
|
||
# Front pages: from book header to Foreword bookmark
|
||
front_pages_start = sva_start
|
||
front_pages_end = (
|
||
foreword_pos
|
||
if foreword_pos
|
||
else (introduction_pos or first_part1_pos or sva_end)
|
||
)
|
||
fp_paras = extract_paragraphs(all_elements, front_pages_start, front_pages_end)
|
||
front_pages = {
|
||
"slug": "front-pages",
|
||
"title": "Front Pages",
|
||
"type": "intro-segment",
|
||
"book": "sva",
|
||
"order": 1,
|
||
"paragraphs": fp_paras,
|
||
}
|
||
|
||
# Foreword: from Foreword bookmark to Introduction bookmark
|
||
foreword = None
|
||
if foreword_pos is not None:
|
||
fw_start = foreword_pos + 1
|
||
fw_end = introduction_pos if introduction_pos else (first_part1_pos or sva_end)
|
||
fw_paras = extract_paragraphs(all_elements, fw_start, fw_end)
|
||
foreword = {
|
||
"slug": "foreword",
|
||
"title": "Foreword",
|
||
"type": "intro-segment",
|
||
"book": "sva",
|
||
"order": 2,
|
||
"paragraphs": fw_paras,
|
||
}
|
||
|
||
# Introduction: from Introduction bookmark to Part 1 start
|
||
introduction = None
|
||
if introduction_pos is not None:
|
||
intro_start = introduction_pos + 1
|
||
intro_end = first_part1_pos if first_part1_pos else (part2_pos or sva_end)
|
||
intro_paras = extract_paragraphs(all_elements, intro_start, intro_end)
|
||
introduction = {
|
||
"slug": "introduction",
|
||
"title": "Introduction",
|
||
"type": "intro-segment",
|
||
"book": "sva",
|
||
"order": 3,
|
||
"paragraphs": intro_paras,
|
||
}
|
||
|
||
intro_segments = [front_pages]
|
||
if foreword:
|
||
intro_segments.append(foreword)
|
||
if introduction:
|
||
intro_segments.append(introduction)
|
||
|
||
# Step 5: Collect all p49 song bookmarks grouped by part number
|
||
song_bookmarks_by_part = defaultdict(list)
|
||
|
||
for pos, text, cls in bookmarks:
|
||
if cls != SONG_BOOKMARK_CLASS:
|
||
continue
|
||
# Parse: "SVA 1: Śrī Guru Praṇāma" → part_num=1, title="Śrī Guru Praṇāma"
|
||
m = re.match(r"SVA\s+(\d+):\s+(.+)", text.strip())
|
||
if m:
|
||
part_num = int(m.group(1))
|
||
song_title = m.group(2).strip()
|
||
song_bookmarks_by_part[part_num].append(
|
||
{
|
||
"pos": pos,
|
||
"title": song_title,
|
||
"raw_text": text.strip(),
|
||
}
|
||
)
|
||
|
||
# Step 6: Define part boundaries
|
||
part_boundaries = []
|
||
|
||
if first_part1_pos is not None:
|
||
ch1_end = part2_pos if part2_pos else sva_end
|
||
part_boundaries.append((1, first_part1_pos, ch1_end))
|
||
|
||
if part2_pos is not None:
|
||
ch2_end = part3_pos if part3_pos else sva_end
|
||
part_boundaries.append((2, part2_pos, ch2_end))
|
||
|
||
if part3_pos is not None:
|
||
ch3_end = part4_pos if part4_pos else sva_end
|
||
part_boundaries.append((3, part3_pos, ch3_end))
|
||
|
||
if part4_pos is not None:
|
||
part_boundaries.append((4, part4_pos, sva_end))
|
||
|
||
# Step 7: Extract sub-chapters (songs) for each part
|
||
sub_chapters = []
|
||
parent_chapters = []
|
||
|
||
for part_num, part_start, part_end in part_boundaries:
|
||
parent_title = CLEAN_PARENT_TITLES.get(part_num, f"Part {part_num}")
|
||
|
||
songs = song_bookmarks_by_part.get(part_num, [])
|
||
# Sort by position
|
||
songs.sort(key=lambda s: s["pos"])
|
||
|
||
sub_entries = []
|
||
song_counter = 0
|
||
|
||
for si, song in enumerate(songs):
|
||
song_title = decode_scagoudy(song["title"])
|
||
|
||
# Detect "Purport to..." bookmarks — merge into preceding song
|
||
is_purport = song_title.lower().startswith("purport to")
|
||
if (
|
||
is_purport
|
||
and sub_chapters
|
||
and sub_chapters[-1]["parentChapter"] == part_num
|
||
):
|
||
# Merge this purport's content into the last song
|
||
content_start = song["pos"] + 1
|
||
if si + 1 < len(songs):
|
||
content_end = songs[si + 1]["pos"]
|
||
else:
|
||
content_end = part_end
|
||
|
||
purport_paragraphs = extract_paragraphs(
|
||
all_elements, content_start, content_end
|
||
)
|
||
sub_chapters[-1]["paragraphs"].extend(purport_paragraphs)
|
||
continue
|
||
|
||
song_counter += 1
|
||
slug = f"{part_num}-{song_counter}"
|
||
|
||
# Content range: from this bookmark to next song bookmark (or part end)
|
||
# We start AFTER the bookmark element itself
|
||
content_start = song["pos"] + 1
|
||
if si + 1 < len(songs):
|
||
content_end = songs[si + 1]["pos"]
|
||
else:
|
||
content_end = part_end
|
||
|
||
paragraphs = extract_paragraphs(all_elements, content_start, content_end)
|
||
|
||
sub_chapters.append(
|
||
{
|
||
"book": "sva",
|
||
"slug": slug,
|
||
"parentChapter": part_num,
|
||
"subChapter": song_counter,
|
||
"title": song_title,
|
||
"parentTitle": parent_title,
|
||
"type": "sub-chapter",
|
||
"paragraphs": paragraphs,
|
||
}
|
||
)
|
||
sub_entries.append({"slug": slug, "title": song_title})
|
||
|
||
parent_chapters.append(
|
||
{
|
||
"number": part_num,
|
||
"title": parent_title,
|
||
"subChapters": sub_entries,
|
||
}
|
||
)
|
||
|
||
print(f"\n Part {part_num}: {parent_title} → {len(sub_entries)} songs")
|
||
for entry in sub_entries:
|
||
matching = [sc for sc in sub_chapters if sc["slug"] == entry["slug"]]
|
||
para_count = len(matching[0]["paragraphs"]) if matching else 0
|
||
print(
|
||
f" {entry['slug']:>5}: {para_count:>3} paragraphs — {entry['title']}"
|
||
)
|
||
|
||
return {
|
||
"intro_segments": intro_segments,
|
||
"sub_chapters": sub_chapters,
|
||
"parent_chapters": parent_chapters,
|
||
}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Output
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def write_json(data, filepath):
|
||
"""Write data as formatted JSON to filepath, creating directories as needed."""
|
||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||
with open(filepath, "w", encoding="utf-8") as f:
|
||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||
|
||
|
||
def main():
|
||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
html_path = os.path.join(base_dir, "public", "html", "other.html")
|
||
output_base = os.path.join(base_dir, "lib", "jsons", "books", "sva")
|
||
|
||
if not os.path.exists(html_path):
|
||
print(f"ERROR: {html_path} not found!")
|
||
sys.exit(1)
|
||
|
||
print(f"Reading {html_path}...")
|
||
with open(html_path, "r", encoding="utf-8") as f:
|
||
soup = BeautifulSoup(f, "html.parser")
|
||
|
||
print("Extracting Songs of the Vaiṣṇava Ācāryas (1972)...\n")
|
||
result = extract_sva(soup)
|
||
|
||
intro_segments = result["intro_segments"]
|
||
sub_chapters = result["sub_chapters"]
|
||
parent_chapters = result["parent_chapters"]
|
||
|
||
# --- Write intro.json ---
|
||
intro_meta = {
|
||
"book": "sva",
|
||
"title": "Songs of the Vaiṣṇava Ācāryas",
|
||
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
||
"year": 1972,
|
||
"description": "Bengali devotional songs by the great Vaiṣṇava teachers, with English translations and commentary.",
|
||
"totalChapters": 4,
|
||
"chapters": parent_chapters,
|
||
}
|
||
intro_path = os.path.join(output_base, "intro.json")
|
||
write_json(intro_meta, intro_path)
|
||
print(f"\n Written: {intro_path}")
|
||
|
||
# --- Write intro segments ---
|
||
print(f"\n--- Intro Segments ---")
|
||
for seg in intro_segments:
|
||
filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
|
||
write_json(seg, filepath)
|
||
print(
|
||
f" {seg['order']:>2}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}"
|
||
)
|
||
|
||
# --- Write sub-chapter files (songs) ---
|
||
print(f"\n--- Songs (Sub-Chapters) ---")
|
||
for sc in sub_chapters:
|
||
slug = sc["slug"]
|
||
filepath = os.path.join(output_base, "chapters", f"{slug}.json")
|
||
write_json(sc, filepath)
|
||
print(
|
||
f" {slug:>5}: {len(sc['paragraphs']):>3} paragraphs — "
|
||
f"[Part {sc['parentChapter']}] {sc['title']}"
|
||
)
|
||
|
||
# --- Summary ---
|
||
total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments)
|
||
total_chapter_paras = sum(len(sc["paragraphs"]) for sc in sub_chapters)
|
||
total_paras = total_intro_paras + total_chapter_paras
|
||
|
||
print(f"\n{'=' * 60}")
|
||
print(f" EXTRACTION SUMMARY")
|
||
print(f"{'=' * 60}")
|
||
print(f" Intro segments: {len(intro_segments)}")
|
||
print(f" Parent chapters: {len(parent_chapters)}")
|
||
print(f" Songs (sub-ch): {len(sub_chapters)}")
|
||
print(f" Total paragraphs: {total_paras}")
|
||
print(f" Intro: {total_intro_paras}")
|
||
print(f" Songs: {total_chapter_paras}")
|
||
if sub_chapters:
|
||
print(f" Avg paras/song: {total_chapter_paras / len(sub_chapters):.1f}")
|
||
print(f" Output directory: {output_base}/")
|
||
|
||
# --- Per-parent breakdown ---
|
||
print(f"\n--- Part Breakdown ---")
|
||
for pc in parent_chapters:
|
||
part_num = pc["number"]
|
||
sub_count = len(pc["subChapters"])
|
||
part_paras = sum(
|
||
len(sc["paragraphs"])
|
||
for sc in sub_chapters
|
||
if sc["parentChapter"] == part_num
|
||
)
|
||
print(
|
||
f" Part {part_num}: {sub_count:>2} songs, {part_paras:>4} paragraphs — {pc['title']}"
|
||
)
|
||
|
||
# --- Samples ---
|
||
print(f"\n--- Sample: Front Pages ---")
|
||
if intro_segments:
|
||
front = intro_segments[0]
|
||
for i, p in enumerate(front["paragraphs"][:5]):
|
||
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
||
print(f" [{i}] {preview}...")
|
||
|
||
if sub_chapters:
|
||
first_sc = sub_chapters[0]
|
||
print(f"\n--- Sample: {first_sc['slug']} ({first_sc['title']}) ---")
|
||
for i, p in enumerate(first_sc["paragraphs"][:5]):
|
||
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
||
print(f" [{i}] {preview}...")
|
||
|
||
if len(sub_chapters) > 1:
|
||
last_sc = sub_chapters[-1]
|
||
print(f"\n--- Sample: {last_sc['slug']} ({last_sc['title']}) ---")
|
||
for i, p in enumerate(last_sc["paragraphs"][:5]):
|
||
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
||
print(f" [{i}] {preview}...")
|
||
|
||
# --- Quality Checks ---
|
||
print(f"\n--- Quality Checks ---")
|
||
|
||
# Check all 4 parts present
|
||
expected_parents = set(range(1, 5))
|
||
actual_parents = {pc["number"] for pc in parent_chapters}
|
||
missing = expected_parents - actual_parents
|
||
extra = actual_parents - expected_parents
|
||
if missing:
|
||
print(f" ⚠ Missing parts: {sorted(missing)}")
|
||
if extra:
|
||
print(f" ⚠ Extra parts: {sorted(extra)}")
|
||
if not missing and not extra:
|
||
print(f" ✅ All 4 parts present")
|
||
|
||
# Check all 3 intro segments present
|
||
expected_intros = {"front-pages", "foreword", "introduction"}
|
||
actual_intros = {s["slug"] for s in intro_segments}
|
||
missing_intros = expected_intros - actual_intros
|
||
if missing_intros:
|
||
print(f" ⚠ Missing intro segments: {missing_intros}")
|
||
else:
|
||
print(f" ✅ All 3 intro segments present")
|
||
|
||
# Check empty sub-chapters
|
||
empty_scs = [sc["slug"] for sc in sub_chapters if len(sc["paragraphs"]) == 0]
|
||
if empty_scs:
|
||
print(f" ⚠ Empty songs (0 paragraphs): {empty_scs}")
|
||
else:
|
||
print(f" ✅ No empty songs")
|
||
|
||
# Check empty intro segments
|
||
empty_intros = [s["slug"] for s in intro_segments if len(s["paragraphs"]) == 0]
|
||
if empty_intros:
|
||
print(f" ⚠ Empty intro segments: {empty_intros}")
|
||
else:
|
||
print(f" ✅ No empty intro segments")
|
||
|
||
# Check ScaGoudy decoding
|
||
encoded_chars = set("äåöëòçüéàùè")
|
||
suspect_count = 0
|
||
all_sections = sub_chapters + intro_segments
|
||
for section in all_sections:
|
||
for p in section["paragraphs"]:
|
||
plain = re.sub(r"<[^>]+>", "", p)
|
||
for c in encoded_chars:
|
||
if c in plain:
|
||
suspect_count += 1
|
||
break
|
||
if suspect_count == 0:
|
||
print(f" ✅ No suspected undecoded ScaGoudy characters found")
|
||
else:
|
||
print(
|
||
f" ℹ {suspect_count} paragraphs contain characters that may be undecoded ScaGoudy"
|
||
)
|
||
|
||
# Check verse blocks
|
||
verse_block_count = 0
|
||
for sc in sub_chapters:
|
||
for p in sc["paragraphs"]:
|
||
if '<div style="text-align:center"><i>' in p:
|
||
verse_block_count += 1
|
||
print(f" ℹ {verse_block_count} verse blocks in songs")
|
||
|
||
# Check table-derived content
|
||
table_verse_count = 0
|
||
for sc in sub_chapters:
|
||
for p in sc["paragraphs"]:
|
||
if '<div style="text-align:center"><i>' in p and "<br>" in p:
|
||
table_verse_count += 1
|
||
print(f" ℹ {table_verse_count} multi-line verse blocks (likely from tables)")
|
||
|
||
# ScaGoudy decoding spot check
|
||
print(f"\n--- ScaGoudy Decoding Spot Check ---")
|
||
if intro_segments:
|
||
fw_segs = [s for s in intro_segments if s["slug"] == "foreword"]
|
||
if fw_segs:
|
||
fw_text = " ".join(fw_segs[0]["paragraphs"][:10])
|
||
if "Prabhupāda" in fw_text:
|
||
print(f" ✅ 'Prabhupāda' correctly decoded in Foreword")
|
||
elif "Prabhupäda" in fw_text:
|
||
print(f" ⚠ ScaGoudy decoding incomplete for 'Prabhupäda'")
|
||
if "Kṛṣṇa" in fw_text:
|
||
print(f" ✅ 'Kṛṣṇa' correctly decoded in Foreword")
|
||
elif "Kåñëa" in fw_text:
|
||
print(f" ⚠ 'Kåñëa' still encoded in Foreword")
|
||
|
||
# Sample verse blocks
|
||
print(f"\n--- Sample Verse Blocks ---")
|
||
shown = 0
|
||
for sc in sub_chapters:
|
||
for p in sc["paragraphs"]:
|
||
if '<div style="text-align:center"><i>' in p:
|
||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
||
print(f" {sc['slug']} ({sc['title']}): {preview}...")
|
||
shown += 1
|
||
if shown >= 5:
|
||
break
|
||
if shown >= 5:
|
||
break
|
||
|
||
# Song title check
|
||
print(f"\n--- Song Titles ---")
|
||
for pc in parent_chapters:
|
||
print(f" Part {pc['number']}: {pc['title']}")
|
||
for sub in pc["subChapters"]:
|
||
print(f" {sub['slug']:>5}: {sub['title']}")
|
||
|
||
# Verify slug numbering continuity within each parent
|
||
print(f"\n--- Slug Numbering Check ---")
|
||
for pc in parent_chapters:
|
||
part_num = pc["number"]
|
||
slugs = [s["slug"] for s in pc["subChapters"]]
|
||
expected_slugs = [f"{part_num}-{i + 1}" for i in range(len(slugs))]
|
||
if slugs == expected_slugs:
|
||
print(
|
||
f" ✅ Part {part_num}: {len(slugs)} songs, slugs {slugs[0]}..{slugs[-1]}"
|
||
)
|
||
else:
|
||
print(
|
||
f" ⚠ Part {part_num}: slug mismatch! expected {expected_slugs}, got {slugs}"
|
||
)
|
||
|
||
# Sub-chapter size check
|
||
print(f"\n--- Song Size Check ---")
|
||
very_small = []
|
||
very_large = []
|
||
for sc in sub_chapters:
|
||
count = len(sc["paragraphs"])
|
||
if count < 1:
|
||
very_small.append((sc["slug"], count, sc["title"]))
|
||
elif count > 200:
|
||
very_large.append((sc["slug"], count, sc["title"]))
|
||
if very_small:
|
||
print(f" ⚠ Very small songs (0 paragraphs):")
|
||
for slug, count, title in very_small:
|
||
print(f" {slug}: {count} paragraphs — {title}")
|
||
else:
|
||
print(f" ✅ All songs have ≥ 1 paragraph")
|
||
if very_large:
|
||
print(f" ⚠ Very large songs (> 200 paragraphs):")
|
||
for slug, count, title in very_large:
|
||
print(f" {slug}: {count} paragraphs — {title}")
|
||
else:
|
||
print(f" ✅ All songs have ≤ 200 paragraphs")
|
||
|
||
# Sub-chapter title decoding check
|
||
print(f"\n--- Song Title Decoding Check ---")
|
||
bad_titles = 0
|
||
for sc in sub_chapters:
|
||
title = sc["title"]
|
||
if "å" in title or "ë" in title or "ñ" in title or "ä" in title:
|
||
print(f" ⚠ {sc['slug']} title has un-decoded ScaGoudy: {title}")
|
||
bad_titles += 1
|
||
if bad_titles == 0:
|
||
print(f" ✅ All {len(sub_chapters)} song titles properly decoded")
|
||
|
||
# Check for residual span tags
|
||
span_count = 0
|
||
for sc in sub_chapters:
|
||
for p in sc["paragraphs"]:
|
||
if 'class="' in p and "<span" in p:
|
||
span_count += 1
|
||
for seg in intro_segments:
|
||
for p in seg["paragraphs"]:
|
||
if 'class="' in p and "<span" in p:
|
||
span_count += 1
|
||
if span_count == 0:
|
||
print(f" ✅ No residual span class artifacts found")
|
||
else:
|
||
print(f" ⚠ {span_count} paragraphs still contain span class artifacts")
|
||
|
||
# Check for dash-line paragraphs
|
||
dash_count = 0
|
||
for sc in sub_chapters:
|
||
for p in sc["paragraphs"]:
|
||
plain = re.sub(r"<[^>]+>", "", p)
|
||
if is_dash_line(plain):
|
||
dash_count += 1
|
||
for seg in intro_segments:
|
||
for p in seg["paragraphs"]:
|
||
plain = re.sub(r"<[^>]+>", "", p)
|
||
if is_dash_line(plain):
|
||
dash_count += 1
|
||
if dash_count == 0:
|
||
print(f" ✅ No dashed-line paragraphs in output")
|
||
else:
|
||
print(f" ⚠ {dash_count} dashed-line paragraphs found in output")
|
||
|
||
print(f"\nDone! ✨")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|