fix: SVA chapters
This commit is contained in:
gosangam
+386
-169
@@ -1,13 +1,21 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract Songs of the Vaiṣṇava Ācāryas (1972) from other.html.
|
||||
|
||||
The book has 3 intro segments (front-pages, foreword, introduction),
|
||||
and 4 chapters (Parts 1–4).
|
||||
Hierarchical sub-chapter extraction: each song within a part becomes its OWN
|
||||
chapter JSON file with a parent-child relationship.
|
||||
|
||||
Parent chapters (Parts 1–4) are containers only — no content of their own.
|
||||
|
||||
Numbering: Part 1, Song 1 → slug "1-1", Part 2, Song 3 → slug "2-3", etc.
|
||||
|
||||
Song boundaries are delimited by p49 bookmarks with text like:
|
||||
"SVA 1: Śrī Guru Praṇāma"
|
||||
"SVA 2: Aruṇodaya-kīrtana"
|
||||
|
||||
Output:
|
||||
lib/jsons/books/sva/intro.json — Book metadata
|
||||
lib/jsons/books/sva/intro/*.json — Intro segments
|
||||
lib/jsons/books/sva/chapters/*.json — Chapters (01–04)
|
||||
lib/jsons/books/sva/intro.json — Book metadata with hierarchical chapters
|
||||
lib/jsons/books/sva/intro/*.json — Intro segments (front-pages, foreword, introduction)
|
||||
lib/jsons/books/sva/chapters/*.json — Sub-chapter files (1-1.json, 1-2.json, 2-1.json, ...)
|
||||
|
||||
Usage:
|
||||
python3 scripts/sva_extract.py
|
||||
@@ -17,6 +25,7 @@ import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
|
||||
from bs4 import BeautifulSoup, NavigableString, Tag
|
||||
|
||||
@@ -39,13 +48,17 @@ SCAGOUDY_TO_IAST = {
|
||||
"ç": "ś",
|
||||
"Ç": "Ś",
|
||||
"ì": "ṅ",
|
||||
"Ì": "Ṅ",
|
||||
"ï": "ñ",
|
||||
"Ï": "Ñ",
|
||||
"ü": "ū",
|
||||
"Ü": "Ū",
|
||||
"é": "ī",
|
||||
"É": "Ī",
|
||||
"à": "ṁ",
|
||||
"À": "Ṁ",
|
||||
"ù": "ḥ",
|
||||
"Ù": "Ḥ",
|
||||
"è": "ṝ",
|
||||
"û": "ṁl",
|
||||
"ÿ": "ḷ",
|
||||
@@ -69,6 +82,18 @@ def decode_html_content(tag: Tag) -> str:
|
||||
)
|
||||
# Clean up Apple-tab-span leftovers (tab characters)
|
||||
inner = inner.replace("\t", " ")
|
||||
# Force valid UTF-8 by round-tripping through encode/decode with replacement
|
||||
inner = inner.encode("utf-8", errors="replace").decode("utf-8", errors="replace")
|
||||
# Strip null bytes and other control characters that break BSON
|
||||
inner = re.sub(r"[\x00-\x08\x0b\x0c\x0e-\x1f]", "", inner)
|
||||
# Remove lone surrogates (U+D800–U+DFFF) that are invalid in BSON/MongoDB
|
||||
inner = re.sub(r"[\ud800-\udfff]", "", inner)
|
||||
# Merge adjacent bold tags: </b><b> → (nothing)
|
||||
for _ in range(5):
|
||||
inner = re.sub(r"</b>\s*<b>", "", inner)
|
||||
# Remove empty bold and italic tags
|
||||
inner = re.sub(r"<b>\s*</b>", "", inner)
|
||||
inner = re.sub(r"<i>\s*</i>", "", inner)
|
||||
# Normalize whitespace
|
||||
inner = re.sub(r"\s+", " ", inner).strip()
|
||||
return inner
|
||||
@@ -138,10 +163,16 @@ SONG_TITLE_CLASS = "p27"
|
||||
SONG_TITLE_VARIANT_CLASS = "p169"
|
||||
|
||||
# Verse line classes (italic, centered)
|
||||
VERSE_PADA_CLASSES = frozenset(["p5", "p53", "p24", "p66", "p255"])
|
||||
VERSE_PADA_CLASSES = frozenset(["p5", "p53", "p24", "p66", "p255", "p134"])
|
||||
|
||||
# Verse number classes (e.g. "3)", "4)", "[1]") — centered but NOT verse text
|
||||
VERSE_NUMBER_CLASS = "p248"
|
||||
VERSE_NUMBER_CLASSES = frozenset(["p248", "p30"])
|
||||
|
||||
# Translation variant classes (bold, centered) — in addition to p56
|
||||
TRANSLATION_VARIANT_CLASSES = frozenset(["p86", "p240", "p242"])
|
||||
|
||||
# Reference/purport note classes that should be centered
|
||||
CENTERED_NOTE_CLASSES = frozenset(["p241", "p22"])
|
||||
|
||||
# Synonyms text
|
||||
SYNONYMS_CLASS = "p54"
|
||||
@@ -246,6 +277,18 @@ SKIP_CLASSES = frozenset(
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Clean parent titles (pre-decoded IAST)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
CLEAN_PARENT_TITLES = {
|
||||
1: "Prayers and Praṇāmas",
|
||||
2: "Songs of Śrīla Bhaktivinoda Ṭhākura",
|
||||
3: "Songs of Śrīla Narottama dāsa Ṭhākura",
|
||||
4: "Songs by other Ācāryas",
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Extraction helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -337,12 +380,12 @@ def process_table_t3(table: Tag) -> list:
|
||||
joined = "<br>".join(verse_lines)
|
||||
result.append(f'<div style="text-align:center"><i>{joined}</i></div>')
|
||||
|
||||
# Right cell: English translation
|
||||
# Right cell: English translation — centered
|
||||
right_ps = tds[1].find_all("p")
|
||||
for p in right_ps:
|
||||
html = decode_html_content(p)
|
||||
if html and html.strip() and html.strip() != "\xa0":
|
||||
result.append(html)
|
||||
result.append(f'<div style="text-align:center">{html}</div>')
|
||||
|
||||
return result
|
||||
|
||||
@@ -367,7 +410,7 @@ def extract_paragraphs(
|
||||
start_idx: int,
|
||||
end_idx: int,
|
||||
) -> list:
|
||||
"""Extract paragraphs for a section (chapter, intro segment).
|
||||
"""Extract paragraphs for a section (song, intro segment).
|
||||
|
||||
Handles all CSS classes within the SVA content.
|
||||
Handles <table class="t3"> and <table class="t2"> elements.
|
||||
@@ -460,6 +503,19 @@ def extract_paragraphs(
|
||||
# Non-verse paragraph — flush any accumulated verse first
|
||||
flush_verse()
|
||||
|
||||
# Short p6 content (verse numbers like "1)", "2)") should be centered
|
||||
# Long p6 content (body paragraphs) stays left-aligned
|
||||
if cls == "p6":
|
||||
html = decode_html_content(elem)
|
||||
if html:
|
||||
plain = re.sub(r"<[^>]+>", "", html).strip()
|
||||
# Short content that looks like a verse number or bracketed ref
|
||||
if len(plain) <= 10 and re.match(r"^[\[\(]?\d+[\]\)]?\.?$", plain):
|
||||
result.append(f'<div style="text-align:center">{html}</div>')
|
||||
else:
|
||||
result.append(html)
|
||||
continue
|
||||
|
||||
# Front pages text (p18)
|
||||
if cls == FRONT_PAGES_CLASS:
|
||||
html = decode_html_content(elem)
|
||||
@@ -467,29 +523,29 @@ def extract_paragraphs(
|
||||
result.append(html)
|
||||
continue
|
||||
|
||||
# Section heading (p31) — bold
|
||||
# Section heading (p31) — bold, centered
|
||||
if cls == SECTION_HEADING_CLASS:
|
||||
html = decode_html_content(elem)
|
||||
if html:
|
||||
if not html.strip().startswith("<b>"):
|
||||
html = f"<b>{html}</b>"
|
||||
result.append(html)
|
||||
result.append(f'<div style="text-align:center">{html}</div>')
|
||||
continue
|
||||
|
||||
# Song title (p27) — bold
|
||||
# Song title (p27) — bold, centered
|
||||
if cls == SONG_TITLE_CLASS:
|
||||
html = decode_html_content(elem)
|
||||
if html:
|
||||
if not html.strip().startswith("<b>"):
|
||||
html = f"<b>{html}</b>"
|
||||
result.append(html)
|
||||
result.append(f'<div style="text-align:center">{html}</div>')
|
||||
continue
|
||||
|
||||
# Song title variant (p169) — bold italic with author
|
||||
# Song title variant (p169) — bold italic with author, centered
|
||||
if cls == SONG_TITLE_VARIANT_CLASS:
|
||||
html = decode_html_content(elem)
|
||||
if html:
|
||||
result.append(html)
|
||||
result.append(f'<div style="text-align:center">{html}</div>')
|
||||
continue
|
||||
|
||||
# Synonyms (p54)
|
||||
@@ -499,8 +555,25 @@ def extract_paragraphs(
|
||||
result.append(html)
|
||||
continue
|
||||
|
||||
# Verse number heading (p248) — e.g. "3)", "4)", "[1]" — center it
|
||||
if cls == VERSE_NUMBER_CLASS:
|
||||
# Verse number heading (p248, p30) — e.g. "3)", "4)", "[1]" — center it
|
||||
if cls in VERSE_NUMBER_CLASSES:
|
||||
flush_verse()
|
||||
html = decode_html_content(elem)
|
||||
if html:
|
||||
result.append(f'<div style="text-align:center">{html}</div>')
|
||||
continue
|
||||
|
||||
# Translation variant classes (p86, p240, p242) — bold, centered
|
||||
if cls in TRANSLATION_VARIANT_CLASSES:
|
||||
flush_verse()
|
||||
html = decode_html_content(elem)
|
||||
if html:
|
||||
result.append(f'<div style="text-align:center">{html}</div>')
|
||||
continue
|
||||
|
||||
# Centered note classes (p241)
|
||||
if cls in CENTERED_NOTE_CLASSES:
|
||||
flush_verse()
|
||||
html = decode_html_content(elem)
|
||||
if html:
|
||||
result.append(f'<div style="text-align:center">{html}</div>')
|
||||
@@ -513,18 +586,18 @@ def extract_paragraphs(
|
||||
result.append(f'<div style="text-align:center">{html}</div>')
|
||||
continue
|
||||
|
||||
# SYNONYMS header (p165)
|
||||
# SYNONYMS header (p165) — centered
|
||||
if cls == SYNONYMS_HEADER_CLASS:
|
||||
html = decode_html_content(elem)
|
||||
if html:
|
||||
result.append(html)
|
||||
result.append(f'<div style="text-align:center">{html}</div>')
|
||||
continue
|
||||
|
||||
# TRANSLATION header (p55)
|
||||
# TRANSLATION header (p55) — centered
|
||||
if cls == TRANSLATION_HEADER_CLASS:
|
||||
html = decode_html_content(elem)
|
||||
if html:
|
||||
result.append(html)
|
||||
result.append(f'<div style="text-align:center">{html}</div>')
|
||||
continue
|
||||
|
||||
# Mantra text (p19)
|
||||
@@ -578,11 +651,11 @@ def extract_paragraphs(
|
||||
result.append(html)
|
||||
continue
|
||||
|
||||
# Reference notes
|
||||
# Reference notes (p187, p231, p233, p237, p32, p251) — centered
|
||||
if cls in REFERENCE_CLASSES:
|
||||
html = decode_html_content(elem)
|
||||
if html:
|
||||
result.append(html)
|
||||
result.append(f'<div style="text-align:center">{html}</div>')
|
||||
continue
|
||||
|
||||
# Bold text classes
|
||||
@@ -592,11 +665,11 @@ def extract_paragraphs(
|
||||
result.append(html)
|
||||
continue
|
||||
|
||||
# Synonym notes (p247)
|
||||
# Synonym note (p247) — centered
|
||||
if cls == SYNONYM_NOTES_CLASS:
|
||||
html = decode_html_content(elem)
|
||||
if html:
|
||||
result.append(html)
|
||||
result.append(f'<div style="text-align:center">{html}</div>')
|
||||
continue
|
||||
|
||||
# Bold headings (p224, p223)
|
||||
@@ -686,8 +759,8 @@ def extract_sva(soup: BeautifulSoup) -> dict:
|
||||
|
||||
Returns a dict with:
|
||||
"intro_segments": list of intro segment dicts
|
||||
"chapters": list of chapter dicts
|
||||
"chapter_listing": list of {number, title} for intro.json
|
||||
"sub_chapters": list of sub-chapter dicts (one per song)
|
||||
"parent_chapters": list of parent chapter info for intro.json
|
||||
"""
|
||||
# Get all top-level <p> and <table> elements
|
||||
all_elements = list(soup.find_all(["p", "table"]))
|
||||
@@ -700,18 +773,22 @@ def extract_sva(soup: BeautifulSoup) -> dict:
|
||||
# Step 2: Find all SVA bookmarks
|
||||
bookmarks = find_sva_bookmarks(all_elements, sva_start, sva_end)
|
||||
print(f" Found {len(bookmarks)} SVA bookmarks")
|
||||
for pos, text, cls in bookmarks[:20]:
|
||||
for pos, text, cls in bookmarks[:30]:
|
||||
print(f" Elem ~{pos} [{cls}]: {text[:80]}")
|
||||
if len(bookmarks) > 20:
|
||||
print(f" ... and {len(bookmarks) - 20} more")
|
||||
if len(bookmarks) > 30:
|
||||
print(f" ... and {len(bookmarks) - 30} more")
|
||||
|
||||
# Step 3: Categorize bookmarks into sections
|
||||
# Step 3: Categorize bookmarks
|
||||
# p13 bookmarks define major sections:
|
||||
# "SVA: Foreword" → foreword
|
||||
# "SVA: Introduction" → introduction
|
||||
# "SVA 2: ..." → Part 2 start
|
||||
# "SVA 3: ..." → Part 3 start
|
||||
# "SVA 4: ..." → Part 4 start
|
||||
# p49 bookmarks define songs within parts:
|
||||
# "SVA 1: ..." → songs in Part 1
|
||||
# "SVA 2: ..." → songs in Part 2
|
||||
# etc.
|
||||
|
||||
foreword_pos = None
|
||||
introduction_pos = None
|
||||
@@ -719,18 +796,8 @@ def extract_sva(soup: BeautifulSoup) -> dict:
|
||||
part3_pos = None
|
||||
part4_pos = None
|
||||
|
||||
# p49 bookmarks define songs within parts:
|
||||
# "SVA 1: ..." → songs in Part 1
|
||||
# "SVA 2: ..." → songs in Part 2
|
||||
# "SVA 3: ..." → songs in Part 3
|
||||
# "SVA 4: ..." → songs in Part 4
|
||||
|
||||
# Also find the first SVA 1: bookmark (start of Part 1)
|
||||
first_part1_pos = None
|
||||
|
||||
# Find "SVA: A Glimpse..." bookmark in intro
|
||||
glimpse_pos = None
|
||||
|
||||
for pos, text, cls in bookmarks:
|
||||
if cls == BOOKMARK_CLASS: # p13
|
||||
if text.strip() == "SVA: Foreword":
|
||||
@@ -747,8 +814,6 @@ def extract_sva(soup: BeautifulSoup) -> dict:
|
||||
if cls == SONG_BOOKMARK_CLASS: # p49
|
||||
if text.strip().startswith("SVA 1:") and first_part1_pos is None:
|
||||
first_part1_pos = pos
|
||||
if "Glimpse" in text or "Life of" in text:
|
||||
glimpse_pos = pos
|
||||
|
||||
print(f"\n Section boundaries:")
|
||||
print(f" Book start: elem {sva_start}")
|
||||
@@ -821,63 +886,133 @@ def extract_sva(soup: BeautifulSoup) -> dict:
|
||||
"paragraphs": intro_paras,
|
||||
}
|
||||
|
||||
# Step 5: Extract chapters (Parts 1–4)
|
||||
chapter_boundaries = []
|
||||
|
||||
if first_part1_pos is not None:
|
||||
ch1_end = part2_pos if part2_pos else sva_end
|
||||
chapter_boundaries.append((1, "Prayers and Praṇāmas", first_part1_pos, ch1_end))
|
||||
|
||||
if part2_pos is not None:
|
||||
ch2_end = part3_pos if part3_pos else sva_end
|
||||
chapter_boundaries.append(
|
||||
(2, "Songs of Śrīla Bhaktivinoda Ṭhākura", part2_pos, ch2_end)
|
||||
)
|
||||
|
||||
if part3_pos is not None:
|
||||
ch3_end = part4_pos if part4_pos else sva_end
|
||||
chapter_boundaries.append(
|
||||
(3, "Songs of Śrīla Narottama dāsa Ṭhākura", part3_pos, ch3_end)
|
||||
)
|
||||
|
||||
if part4_pos is not None:
|
||||
chapter_boundaries.append((4, "Songs by other Ācāryas", part4_pos, sva_end))
|
||||
|
||||
chapters = []
|
||||
for ch_num, ch_title, ch_start, ch_end in chapter_boundaries:
|
||||
# Start extraction from the element after the p13 bookmark (for parts 2-4)
|
||||
# or from the p49 bookmark itself for part 1
|
||||
content_start = ch_start
|
||||
if ch_num >= 2:
|
||||
# For parts 2-4, skip the p13 bookmark and the p31 heading that follows
|
||||
content_start = ch_start + 1
|
||||
|
||||
paragraphs = extract_paragraphs(all_elements, content_start, ch_end)
|
||||
|
||||
ch = {
|
||||
"book": "sva",
|
||||
"chapter": ch_num,
|
||||
"title": ch_title,
|
||||
"type": "chapter",
|
||||
"paragraphs": paragraphs,
|
||||
}
|
||||
chapters.append(ch)
|
||||
|
||||
# Step 6: Build results
|
||||
intro_segments = [front_pages]
|
||||
if foreword:
|
||||
intro_segments.append(foreword)
|
||||
if introduction:
|
||||
intro_segments.append(introduction)
|
||||
|
||||
chapter_listing = [
|
||||
{"number": ch["chapter"], "title": ch["title"]} for ch in chapters
|
||||
]
|
||||
# Step 5: Collect all p49 song bookmarks grouped by part number
|
||||
song_bookmarks_by_part = defaultdict(list)
|
||||
|
||||
for pos, text, cls in bookmarks:
|
||||
if cls != SONG_BOOKMARK_CLASS:
|
||||
continue
|
||||
# Parse: "SVA 1: Śrī Guru Praṇāma" → part_num=1, title="Śrī Guru Praṇāma"
|
||||
m = re.match(r"SVA\s+(\d+):\s+(.+)", text.strip())
|
||||
if m:
|
||||
part_num = int(m.group(1))
|
||||
song_title = m.group(2).strip()
|
||||
song_bookmarks_by_part[part_num].append(
|
||||
{
|
||||
"pos": pos,
|
||||
"title": song_title,
|
||||
"raw_text": text.strip(),
|
||||
}
|
||||
)
|
||||
|
||||
# Step 6: Define part boundaries
|
||||
part_boundaries = []
|
||||
|
||||
if first_part1_pos is not None:
|
||||
ch1_end = part2_pos if part2_pos else sva_end
|
||||
part_boundaries.append((1, first_part1_pos, ch1_end))
|
||||
|
||||
if part2_pos is not None:
|
||||
ch2_end = part3_pos if part3_pos else sva_end
|
||||
part_boundaries.append((2, part2_pos, ch2_end))
|
||||
|
||||
if part3_pos is not None:
|
||||
ch3_end = part4_pos if part4_pos else sva_end
|
||||
part_boundaries.append((3, part3_pos, ch3_end))
|
||||
|
||||
if part4_pos is not None:
|
||||
part_boundaries.append((4, part4_pos, sva_end))
|
||||
|
||||
# Step 7: Extract sub-chapters (songs) for each part
|
||||
sub_chapters = []
|
||||
parent_chapters = []
|
||||
|
||||
for part_num, part_start, part_end in part_boundaries:
|
||||
parent_title = CLEAN_PARENT_TITLES.get(part_num, f"Part {part_num}")
|
||||
|
||||
songs = song_bookmarks_by_part.get(part_num, [])
|
||||
# Sort by position
|
||||
songs.sort(key=lambda s: s["pos"])
|
||||
|
||||
sub_entries = []
|
||||
song_counter = 0
|
||||
|
||||
for si, song in enumerate(songs):
|
||||
song_title = decode_scagoudy(song["title"])
|
||||
|
||||
# Detect "Purport to..." bookmarks — merge into preceding song
|
||||
is_purport = song_title.lower().startswith("purport to")
|
||||
if (
|
||||
is_purport
|
||||
and sub_chapters
|
||||
and sub_chapters[-1]["parentChapter"] == part_num
|
||||
):
|
||||
# Merge this purport's content into the last song
|
||||
content_start = song["pos"] + 1
|
||||
if si + 1 < len(songs):
|
||||
content_end = songs[si + 1]["pos"]
|
||||
else:
|
||||
content_end = part_end
|
||||
|
||||
purport_paragraphs = extract_paragraphs(
|
||||
all_elements, content_start, content_end
|
||||
)
|
||||
sub_chapters[-1]["paragraphs"].extend(purport_paragraphs)
|
||||
continue
|
||||
|
||||
song_counter += 1
|
||||
slug = f"{part_num}-{song_counter}"
|
||||
|
||||
# Content range: from this bookmark to next song bookmark (or part end)
|
||||
# We start AFTER the bookmark element itself
|
||||
content_start = song["pos"] + 1
|
||||
if si + 1 < len(songs):
|
||||
content_end = songs[si + 1]["pos"]
|
||||
else:
|
||||
content_end = part_end
|
||||
|
||||
paragraphs = extract_paragraphs(all_elements, content_start, content_end)
|
||||
|
||||
sub_chapters.append(
|
||||
{
|
||||
"book": "sva",
|
||||
"slug": slug,
|
||||
"parentChapter": part_num,
|
||||
"subChapter": song_counter,
|
||||
"title": song_title,
|
||||
"parentTitle": parent_title,
|
||||
"type": "sub-chapter",
|
||||
"paragraphs": paragraphs,
|
||||
}
|
||||
)
|
||||
sub_entries.append({"slug": slug, "title": song_title})
|
||||
|
||||
parent_chapters.append(
|
||||
{
|
||||
"number": part_num,
|
||||
"title": parent_title,
|
||||
"subChapters": sub_entries,
|
||||
}
|
||||
)
|
||||
|
||||
print(f"\n Part {part_num}: {parent_title} → {len(sub_entries)} songs")
|
||||
for entry in sub_entries:
|
||||
matching = [sc for sc in sub_chapters if sc["slug"] == entry["slug"]]
|
||||
para_count = len(matching[0]["paragraphs"]) if matching else 0
|
||||
print(
|
||||
f" {entry['slug']:>5}: {para_count:>3} paragraphs — {entry['title']}"
|
||||
)
|
||||
|
||||
return {
|
||||
"intro_segments": intro_segments,
|
||||
"chapters": chapters,
|
||||
"chapter_listing": chapter_listing,
|
||||
"sub_chapters": sub_chapters,
|
||||
"parent_chapters": parent_chapters,
|
||||
}
|
||||
|
||||
|
||||
@@ -910,8 +1045,8 @@ def main():
|
||||
result = extract_sva(soup)
|
||||
|
||||
intro_segments = result["intro_segments"]
|
||||
chapters = result["chapters"]
|
||||
chapter_listing = result["chapter_listing"]
|
||||
sub_chapters = result["sub_chapters"]
|
||||
parent_chapters = result["parent_chapters"]
|
||||
|
||||
# --- Write intro.json ---
|
||||
intro_meta = {
|
||||
@@ -921,7 +1056,7 @@ def main():
|
||||
"year": 1972,
|
||||
"description": "Bengali devotional songs by the great Vaiṣṇava teachers, with English translations and commentary.",
|
||||
"totalChapters": 4,
|
||||
"chapters": chapter_listing,
|
||||
"chapters": parent_chapters,
|
||||
}
|
||||
intro_path = os.path.join(output_base, "intro.json")
|
||||
write_json(intro_meta, intro_path)
|
||||
@@ -933,75 +1068,82 @@ def main():
|
||||
filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
|
||||
write_json(seg, filepath)
|
||||
print(
|
||||
f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}"
|
||||
f" {seg['order']:>2}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}"
|
||||
)
|
||||
|
||||
# --- Write chapters ---
|
||||
print(f"\n--- Chapters ---")
|
||||
for ch in chapters:
|
||||
ch_num_str = str(ch["chapter"]).zfill(2)
|
||||
filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
|
||||
write_json(ch, filepath)
|
||||
# --- Write sub-chapter files (songs) ---
|
||||
print(f"\n--- Songs (Sub-Chapters) ---")
|
||||
for sc in sub_chapters:
|
||||
slug = sc["slug"]
|
||||
filepath = os.path.join(output_base, "chapters", f"{slug}.json")
|
||||
write_json(sc, filepath)
|
||||
print(
|
||||
f" Part {ch['chapter']}: {len(ch['paragraphs']):>3} paragraphs — "
|
||||
f"{ch['title']}"
|
||||
f" {slug:>5}: {len(sc['paragraphs']):>3} paragraphs — "
|
||||
f"[Part {sc['parentChapter']}] {sc['title']}"
|
||||
)
|
||||
|
||||
# --- Summary ---
|
||||
total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments)
|
||||
total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
|
||||
total_chapter_paras = sum(len(sc["paragraphs"]) for sc in sub_chapters)
|
||||
total_paras = total_intro_paras + total_chapter_paras
|
||||
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f" EXTRACTION SUMMARY")
|
||||
print(f"{'=' * 60}")
|
||||
print(f" Intro segments: {len(intro_segments)}")
|
||||
for seg in intro_segments:
|
||||
print(
|
||||
f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs"
|
||||
)
|
||||
print(f" Chapters (Parts): {len(chapters)}")
|
||||
for ch in chapters:
|
||||
print(
|
||||
f" Part {ch['chapter']}: {len(ch['paragraphs'])} paragraphs — {ch['title']}"
|
||||
)
|
||||
print(f" Parent chapters: {len(parent_chapters)}")
|
||||
print(f" Songs (sub-ch): {len(sub_chapters)}")
|
||||
print(f" Total paragraphs: {total_paras}")
|
||||
print(f" Intro: {total_intro_paras}")
|
||||
print(f" Chapters: {total_chapter_paras}")
|
||||
if chapters:
|
||||
print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
|
||||
print(f" Songs: {total_chapter_paras}")
|
||||
if sub_chapters:
|
||||
print(f" Avg paras/song: {total_chapter_paras / len(sub_chapters):.1f}")
|
||||
print(f" Output directory: {output_base}/")
|
||||
|
||||
# --- Per-parent breakdown ---
|
||||
print(f"\n--- Part Breakdown ---")
|
||||
for pc in parent_chapters:
|
||||
part_num = pc["number"]
|
||||
sub_count = len(pc["subChapters"])
|
||||
part_paras = sum(
|
||||
len(sc["paragraphs"])
|
||||
for sc in sub_chapters
|
||||
if sc["parentChapter"] == part_num
|
||||
)
|
||||
print(
|
||||
f" Part {part_num}: {sub_count:>2} songs, {part_paras:>4} paragraphs — {pc['title']}"
|
||||
)
|
||||
|
||||
# --- Samples ---
|
||||
print(f"\n--- Sample: Part 1 (Prayers and Praṇāmas) ---")
|
||||
if chapters:
|
||||
ch1 = chapters[0]
|
||||
for i, p in enumerate(ch1["paragraphs"][:5]):
|
||||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
||||
print(f"\n--- Sample: Front Pages ---")
|
||||
if intro_segments:
|
||||
front = intro_segments[0]
|
||||
for i, p in enumerate(front["paragraphs"][:5]):
|
||||
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
||||
print(f" [{i}] {preview}...")
|
||||
|
||||
if len(chapters) >= 2:
|
||||
print(f"\n--- Sample: Part 2 (Songs of Bhaktivinoda Ṭhākura) ---")
|
||||
ch2 = chapters[1]
|
||||
for i, p in enumerate(ch2["paragraphs"][:5]):
|
||||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
||||
if sub_chapters:
|
||||
first_sc = sub_chapters[0]
|
||||
print(f"\n--- Sample: {first_sc['slug']} ({first_sc['title']}) ---")
|
||||
for i, p in enumerate(first_sc["paragraphs"][:5]):
|
||||
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
||||
print(f" [{i}] {preview}...")
|
||||
|
||||
if len(chapters) >= 4:
|
||||
print(f"\n--- Sample: Part 4 (Songs by other Ācāryas) ---")
|
||||
ch4 = chapters[3]
|
||||
for i, p in enumerate(ch4["paragraphs"][:5]):
|
||||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
||||
if len(sub_chapters) > 1:
|
||||
last_sc = sub_chapters[-1]
|
||||
print(f"\n--- Sample: {last_sc['slug']} ({last_sc['title']}) ---")
|
||||
for i, p in enumerate(last_sc["paragraphs"][:5]):
|
||||
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
||||
print(f" [{i}] {preview}...")
|
||||
|
||||
# --- Quality Checks ---
|
||||
print(f"\n--- Quality Checks ---")
|
||||
|
||||
# Check all 4 parts present
|
||||
expected_chapters = set(range(1, 5))
|
||||
actual_chapters = set(ch["chapter"] for ch in chapters)
|
||||
missing = expected_chapters - actual_chapters
|
||||
extra = actual_chapters - expected_chapters
|
||||
expected_parents = set(range(1, 5))
|
||||
actual_parents = {pc["number"] for pc in parent_chapters}
|
||||
missing = expected_parents - actual_parents
|
||||
extra = actual_parents - expected_parents
|
||||
if missing:
|
||||
print(f" ⚠ Missing parts: {sorted(missing)}")
|
||||
if extra:
|
||||
@@ -1009,7 +1151,7 @@ def main():
|
||||
if not missing and not extra:
|
||||
print(f" ✅ All 4 parts present")
|
||||
|
||||
# Check intro segments
|
||||
# Check all 3 intro segments present
|
||||
expected_intros = {"front-pages", "foreword", "introduction"}
|
||||
actual_intros = {s["slug"] for s in intro_segments}
|
||||
missing_intros = expected_intros - actual_intros
|
||||
@@ -1018,17 +1160,24 @@ def main():
|
||||
else:
|
||||
print(f" ✅ All 3 intro segments present")
|
||||
|
||||
# Check empty chapters
|
||||
empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
|
||||
if empty_chapters:
|
||||
print(f" ⚠ Empty parts (0 paragraphs): {empty_chapters}")
|
||||
# Check empty sub-chapters
|
||||
empty_scs = [sc["slug"] for sc in sub_chapters if len(sc["paragraphs"]) == 0]
|
||||
if empty_scs:
|
||||
print(f" ⚠ Empty songs (0 paragraphs): {empty_scs}")
|
||||
else:
|
||||
print(f" ✅ No empty parts")
|
||||
print(f" ✅ No empty songs")
|
||||
|
||||
# Check empty intro segments
|
||||
empty_intros = [s["slug"] for s in intro_segments if len(s["paragraphs"]) == 0]
|
||||
if empty_intros:
|
||||
print(f" ⚠ Empty intro segments: {empty_intros}")
|
||||
else:
|
||||
print(f" ✅ No empty intro segments")
|
||||
|
||||
# Check ScaGoudy decoding
|
||||
encoded_chars = set("äåöëòçüéàùè")
|
||||
suspect_count = 0
|
||||
all_sections = chapters + intro_segments
|
||||
all_sections = sub_chapters + intro_segments
|
||||
for section in all_sections:
|
||||
for p in section["paragraphs"]:
|
||||
plain = re.sub(r"<[^>]+>", "", p)
|
||||
@@ -1045,16 +1194,16 @@ def main():
|
||||
|
||||
# Check verse blocks
|
||||
verse_block_count = 0
|
||||
for ch in chapters:
|
||||
for p in ch["paragraphs"]:
|
||||
for sc in sub_chapters:
|
||||
for p in sc["paragraphs"]:
|
||||
if '<div style="text-align:center"><i>' in p:
|
||||
verse_block_count += 1
|
||||
print(f" ℹ {verse_block_count} verse blocks in chapters")
|
||||
print(f" ℹ {verse_block_count} verse blocks in songs")
|
||||
|
||||
# Check table-derived content
|
||||
table_verse_count = 0
|
||||
for ch in chapters:
|
||||
for p in ch["paragraphs"]:
|
||||
for sc in sub_chapters:
|
||||
for p in sc["paragraphs"]:
|
||||
if '<div style="text-align:center"><i>' in p and "<br>" in p:
|
||||
table_verse_count += 1
|
||||
print(f" ℹ {table_verse_count} multi-line verse blocks (likely from tables)")
|
||||
@@ -1077,11 +1226,11 @@ def main():
|
||||
# Sample verse blocks
|
||||
print(f"\n--- Sample Verse Blocks ---")
|
||||
shown = 0
|
||||
for ch in chapters:
|
||||
for p in ch["paragraphs"]:
|
||||
for sc in sub_chapters:
|
||||
for p in sc["paragraphs"]:
|
||||
if '<div style="text-align:center"><i>' in p:
|
||||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
||||
print(f" Part {ch['chapter']}: {preview}...")
|
||||
print(f" {sc['slug']} ({sc['title']}): {preview}...")
|
||||
shown += 1
|
||||
if shown >= 5:
|
||||
break
|
||||
@@ -1089,24 +1238,92 @@ def main():
|
||||
break
|
||||
|
||||
# Song title check
|
||||
print(f"\n--- Song Titles in Chapters ---")
|
||||
song_count = 0
|
||||
sample_songs = []
|
||||
for ch in chapters:
|
||||
ch_songs = 0
|
||||
for p in ch["paragraphs"]:
|
||||
# Bold italic song titles
|
||||
if "<b><i>" in p and "</i></b>" in p:
|
||||
plain = re.sub(r"<[^>]+>", "", p).strip()
|
||||
if len(plain) < 100:
|
||||
ch_songs += 1
|
||||
if len(sample_songs) < 8:
|
||||
sample_songs.append(f" Part {ch['chapter']}: {plain}")
|
||||
song_count += ch_songs
|
||||
print(f" Part {ch['chapter']}: ~{ch_songs} song titles")
|
||||
print(f" Total song titles: ~{song_count}")
|
||||
for s in sample_songs:
|
||||
print(s)
|
||||
print(f"\n--- Song Titles ---")
|
||||
for pc in parent_chapters:
|
||||
print(f" Part {pc['number']}: {pc['title']}")
|
||||
for sub in pc["subChapters"]:
|
||||
print(f" {sub['slug']:>5}: {sub['title']}")
|
||||
|
||||
# Verify slug numbering continuity within each parent
|
||||
print(f"\n--- Slug Numbering Check ---")
|
||||
for pc in parent_chapters:
|
||||
part_num = pc["number"]
|
||||
slugs = [s["slug"] for s in pc["subChapters"]]
|
||||
expected_slugs = [f"{part_num}-{i + 1}" for i in range(len(slugs))]
|
||||
if slugs == expected_slugs:
|
||||
print(
|
||||
f" ✅ Part {part_num}: {len(slugs)} songs, slugs {slugs[0]}..{slugs[-1]}"
|
||||
)
|
||||
else:
|
||||
print(
|
||||
f" ⚠ Part {part_num}: slug mismatch! expected {expected_slugs}, got {slugs}"
|
||||
)
|
||||
|
||||
# Sub-chapter size check
|
||||
print(f"\n--- Song Size Check ---")
|
||||
very_small = []
|
||||
very_large = []
|
||||
for sc in sub_chapters:
|
||||
count = len(sc["paragraphs"])
|
||||
if count < 1:
|
||||
very_small.append((sc["slug"], count, sc["title"]))
|
||||
elif count > 200:
|
||||
very_large.append((sc["slug"], count, sc["title"]))
|
||||
if very_small:
|
||||
print(f" ⚠ Very small songs (0 paragraphs):")
|
||||
for slug, count, title in very_small:
|
||||
print(f" {slug}: {count} paragraphs — {title}")
|
||||
else:
|
||||
print(f" ✅ All songs have ≥ 1 paragraph")
|
||||
if very_large:
|
||||
print(f" ⚠ Very large songs (> 200 paragraphs):")
|
||||
for slug, count, title in very_large:
|
||||
print(f" {slug}: {count} paragraphs — {title}")
|
||||
else:
|
||||
print(f" ✅ All songs have ≤ 200 paragraphs")
|
||||
|
||||
# Sub-chapter title decoding check
|
||||
print(f"\n--- Song Title Decoding Check ---")
|
||||
bad_titles = 0
|
||||
for sc in sub_chapters:
|
||||
title = sc["title"]
|
||||
if "å" in title or "ë" in title or "ñ" in title or "ä" in title:
|
||||
print(f" ⚠ {sc['slug']} title has un-decoded ScaGoudy: {title}")
|
||||
bad_titles += 1
|
||||
if bad_titles == 0:
|
||||
print(f" ✅ All {len(sub_chapters)} song titles properly decoded")
|
||||
|
||||
# Check for residual span tags
|
||||
span_count = 0
|
||||
for sc in sub_chapters:
|
||||
for p in sc["paragraphs"]:
|
||||
if 'class="' in p and "<span" in p:
|
||||
span_count += 1
|
||||
for seg in intro_segments:
|
||||
for p in seg["paragraphs"]:
|
||||
if 'class="' in p and "<span" in p:
|
||||
span_count += 1
|
||||
if span_count == 0:
|
||||
print(f" ✅ No residual span class artifacts found")
|
||||
else:
|
||||
print(f" ⚠ {span_count} paragraphs still contain span class artifacts")
|
||||
|
||||
# Check for dash-line paragraphs
|
||||
dash_count = 0
|
||||
for sc in sub_chapters:
|
||||
for p in sc["paragraphs"]:
|
||||
plain = re.sub(r"<[^>]+>", "", p)
|
||||
if is_dash_line(plain):
|
||||
dash_count += 1
|
||||
for seg in intro_segments:
|
||||
for p in seg["paragraphs"]:
|
||||
plain = re.sub(r"<[^>]+>", "", p)
|
||||
if is_dash_line(plain):
|
||||
dash_count += 1
|
||||
if dash_count == 0:
|
||||
print(f" ✅ No dashed-line paragraphs in output")
|
||||
else:
|
||||
print(f" ⚠ {dash_count} dashed-line paragraphs found in output")
|
||||
|
||||
print(f"\nDone! ✨")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user