846 lines
27 KiB
Python
846 lines
27 KiB
Python
#!/usr/bin/env python3
|
||
"""Extract Gītār Gān (GG) from other.html.
|
||
|
||
Gītār Gān is a Bengali poetic rendering of Bhagavad-gītā As It Is.
|
||
It has 1 intro segment (introduction) and 18 chapters.
|
||
|
||
The content is primarily Bengali verse text (p24, italic with <br> line breaks)
|
||
interleaved with text number headings (p224, bold).
|
||
|
||
Output:
|
||
lib/jsons/books/gg/intro.json — Book metadata
|
||
lib/jsons/books/gg/intro/*.json — Intro segments
|
||
lib/jsons/books/gg/chapters/*.json — Chapters (01–18)
|
||
|
||
Usage:
|
||
python3 scripts/gg_extract.py
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import re
|
||
import sys
|
||
|
||
from bs4 import BeautifulSoup, Tag
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# ScaGoudy → IAST Unicode mapping
|
||
# ---------------------------------------------------------------------------
|
||
SCAGOUDY_TO_IAST = {
|
||
"ä": "ā",
|
||
"Ä": "Ā",
|
||
"å": "ṛ",
|
||
"Å": "Ṛ",
|
||
"ñ": "ṣ",
|
||
"Ñ": "Ṣ",
|
||
"ö": "ṭ",
|
||
"Ö": "Ṭ",
|
||
"ë": "ṇ",
|
||
"Ë": "Ṇ",
|
||
"ò": "ḍ",
|
||
"Ò": "Ḍ",
|
||
"ç": "ś",
|
||
"Ç": "Ś",
|
||
"ì": "ṅ",
|
||
"ï": "ñ",
|
||
"ü": "ū",
|
||
"Ü": "Ū",
|
||
"é": "ī",
|
||
"É": "Ī",
|
||
"à": "ṁ",
|
||
"ù": "ḥ",
|
||
"è": "ṝ",
|
||
"û": "ṁl",
|
||
"ÿ": "ḷ",
|
||
}
|
||
|
||
|
||
def decode_scagoudy(text: str) -> str:
|
||
"""Replace ScaGoudy-encoded characters with proper IAST Unicode."""
|
||
for src, dst in SCAGOUDY_TO_IAST.items():
|
||
text = text.replace(src, dst)
|
||
return text
|
||
|
||
|
||
def decode_html_content(tag: Tag) -> str:
|
||
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
|
||
inner = "".join(str(child) for child in tag.children)
|
||
inner = decode_scagoudy(inner)
|
||
# Remove span tags with CSS classes (RTF conversion artifacts)
|
||
inner = re.sub(
|
||
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
|
||
)
|
||
# Clean up Apple-tab-span leftovers (tab characters)
|
||
inner = inner.replace("\t", " ")
|
||
# Normalize whitespace (but preserve <br> tags)
|
||
# First, protect <br> tags
|
||
inner = re.sub(r"<br\s*/?>", "<br/>", inner)
|
||
# Normalize spaces around text (not across <br>)
|
||
parts = inner.split("<br/>")
|
||
parts = [re.sub(r"\s+", " ", p).strip() for p in parts]
|
||
inner = "<br/>".join(parts)
|
||
return inner.strip()
|
||
|
||
|
||
def decode_text(tag: Tag) -> str:
|
||
"""Get plain text from a tag with ScaGoudy decoding."""
|
||
return decode_scagoudy(tag.get_text().strip())
|
||
|
||
|
||
def get_css_class(tag: Tag) -> str:
|
||
"""Get the first CSS class of a tag, or empty string."""
|
||
classes = tag.get("class", [])
|
||
return classes[0] if classes else ""
|
||
|
||
|
||
def is_empty(tag: Tag) -> bool:
|
||
"""Check if a tag is empty or whitespace-only."""
|
||
text = tag.get_text().strip()
|
||
return not text or text == "\xa0"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# CSS class categorization for GG
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# Book title header class
|
||
BOOK_HEADER_CLASS = "p12"
|
||
|
||
# Subtitle class
|
||
SUBTITLE_CLASS = "p31"
|
||
|
||
# Bookmark class
|
||
BOOKMARK_CLASS = "p13"
|
||
|
||
# Introduction heading / Chapter heading
|
||
HEADING_CLASS = "p27"
|
||
|
||
# Introduction body text
|
||
BODY_CLASS = "p6"
|
||
|
||
# Signature block
|
||
SIGNATURE_CLASS = "p222"
|
||
|
||
# Chapter title heading (contains Chapter N + Title)
|
||
CHAPTER_TITLE_CLASS = "p223"
|
||
|
||
# Bengali verse text (italic, with <br> line breaks — THE MAIN content)
|
||
VERSE_CLASS = "p24"
|
||
|
||
# Text number headings (bold)
|
||
TEXT_HEADING_CLASS = "p224"
|
||
|
||
# Empty/break classes to skip
|
||
SKIP_CLASSES = frozenset(["p225", "p226", "p15"])
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Chapter titles mapping (from bookmarks)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
CHAPTER_TITLES = {
|
||
1: "Observing the Armies on the Battlefield of Kurukṣetra",
|
||
2: "Contents of the Gītā Summarized",
|
||
3: "Karma-yoga",
|
||
4: "Transcendental Knowledge",
|
||
5: "Karma-yoga—Action in Kṛṣṇa Consciousness",
|
||
6: "Dhyāna-yoga",
|
||
7: "Knowledge of the Absolute",
|
||
8: "Attaining the Supreme",
|
||
9: "The Most Confidential Knowledge",
|
||
10: "The Opulence of the Absolute",
|
||
11: "The Universal Form",
|
||
12: "Devotional Service",
|
||
13: "Nature, the Enjoyer, and Consciousness",
|
||
14: "The Three Modes of Material Nature",
|
||
15: "The Yoga of the Supreme Person",
|
||
16: "The Divine and Demoniac Natures",
|
||
17: "The Divisions of Faith",
|
||
18: "Conclusion—The Perfection of Renunciation",
|
||
}
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Extraction helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def find_gg_boundaries(all_paragraphs: list) -> tuple:
|
||
"""Find the start and end paragraph indices of the GG content.
|
||
|
||
Start: <p class="p12"><b>Gétär Gän</b></p> (or decoded: Gītār Gān)
|
||
End: Next <p class="p12"> after GG content (Songs of the Vaiṣṇava Ācāryas)
|
||
"""
|
||
start_idx = None
|
||
end_idx = len(all_paragraphs)
|
||
|
||
for i, p in enumerate(all_paragraphs):
|
||
cls = get_css_class(p)
|
||
text = p.get_text().strip()
|
||
decoded = decode_scagoudy(text)
|
||
|
||
if cls == BOOK_HEADER_CLASS:
|
||
if start_idx is None:
|
||
# Look for "Gétär Gän" or decoded "Gītār Gān"
|
||
if (
|
||
"Gétär Gän" in text
|
||
or "Gītār Gān" in decoded
|
||
or "G\u00e9t\u00e4r G\u00e4n" in text
|
||
):
|
||
start_idx = i
|
||
continue
|
||
else:
|
||
# Next p12 after start marks the end
|
||
end_idx = i
|
||
break
|
||
|
||
if start_idx is None:
|
||
print("ERROR: Could not find 'Gītār Gān' header in other.html!")
|
||
sys.exit(1)
|
||
|
||
return start_idx, end_idx
|
||
|
||
|
||
def find_all_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
|
||
"""Find all GG bookmark tags in the GG range.
|
||
|
||
Bookmarks are p13 class with text starting with "GG:".
|
||
Returns list of (paragraph_index, bookmark_text).
|
||
"""
|
||
bookmarks = []
|
||
for i in range(start_idx, end_idx):
|
||
p = all_paragraphs[i]
|
||
cls = get_css_class(p)
|
||
if cls != BOOKMARK_CLASS:
|
||
continue
|
||
text = decode_text(p).strip()
|
||
if text.startswith("GG:"):
|
||
bookmarks.append((i, text))
|
||
return bookmarks
|
||
|
||
|
||
def parse_chapter_bookmark(text: str):
|
||
"""Parse a chapter bookmark like 'GG: Chapter 1'.
|
||
|
||
Returns chapter_number (int) or None if not a chapter bookmark.
|
||
"""
|
||
m = re.match(r"GG:\s+Chapter\s+(\d+)", text.strip())
|
||
if m:
|
||
return int(m.group(1))
|
||
return None
|
||
|
||
|
||
def extract_chapter_title(tag: Tag) -> tuple:
|
||
"""Extract chapter number and title from a p223 element.
|
||
|
||
The p223 contains something like:
|
||
<span class="s41"><b>Chapter 1</b></span><b>Observing the Armies...</b>
|
||
|
||
Returns (chapter_number, title_text) or None.
|
||
"""
|
||
text = decode_text(tag)
|
||
# Try to parse "Chapter N\nTitle" or "Chapter N Title"
|
||
m = re.match(r"Chapter\s+(\d+)\s*(.*)", text, re.DOTALL)
|
||
if m:
|
||
ch_num = int(m.group(1))
|
||
title = m.group(2).strip()
|
||
# Clean up any leading/trailing whitespace or newlines
|
||
title = re.sub(r"\s+", " ", title).strip()
|
||
return ch_num, title
|
||
return None
|
||
|
||
|
||
def extract_verse_content(tag: Tag) -> str:
|
||
"""Extract Bengali verse content from a p24 tag.
|
||
|
||
The content is italic with <br> line breaks. We wrap it as a centered
|
||
italic div: <div style="text-align:center"><i>content</i></div>
|
||
|
||
ScaGoudy decoding is applied.
|
||
"""
|
||
html = decode_html_content(tag)
|
||
if not html:
|
||
return ""
|
||
|
||
# Strip outer <i> tags if present — we'll re-wrap
|
||
stripped = re.sub(r"^<i>(.*)</i>$", r"\1", html, flags=re.DOTALL)
|
||
# Also remove any remaining <i>/<\/i> pairs within
|
||
stripped = re.sub(r"</?i>", "", stripped)
|
||
|
||
if stripped.strip():
|
||
return f'<div style="text-align:center"><i>{stripped.strip()}</i></div>'
|
||
return ""
|
||
|
||
|
||
def extract_text_heading(tag: Tag) -> tuple:
|
||
"""Extract text heading from a p224 tag.
|
||
|
||
Content is bold, e.g. <b>Text 1</b> or <b>Texts 16-18</b>
|
||
or closing: <b>Thus Bhaktivedānta sings...</b>
|
||
|
||
Some headings have the translation merged in, like:
|
||
<b>Text 30: All these performers who know...</b>
|
||
These are split: the text number is returned as the heading,
|
||
and the translation is returned separately as a "deferred" paragraph
|
||
to be emitted AFTER the next verse block.
|
||
|
||
Returns (heading_html, deferred_translation_html_or_None).
|
||
"""
|
||
html = decode_html_content(tag)
|
||
if not html:
|
||
return ("", None)
|
||
|
||
stripped = html.strip()
|
||
|
||
# Remove outer <b> to inspect inner text
|
||
inner = stripped
|
||
if inner.startswith("<b>") and inner.endswith("</b>"):
|
||
inner = inner[3:-4].strip()
|
||
|
||
# Check if this is a merged "Text N: translation..." heading
|
||
m = re.match(r"^(Texts?\s+[\d\-–]+)\s*[:\.]\s*(.+)$", inner, re.DOTALL)
|
||
if m:
|
||
text_num = m.group(1).strip()
|
||
translation = m.group(2).strip()
|
||
heading = f'<div style="text-align:center"><b>{text_num}</b></div>'
|
||
deferred = f'<div style="text-align:center"><b>{translation}</b></div>'
|
||
return (heading, deferred)
|
||
|
||
# Plain text heading (e.g. "Text 1") or closing statement — center it
|
||
if not stripped.startswith("<b>"):
|
||
stripped = f"<b>{stripped}</b>"
|
||
return (f'<div style="text-align:center">{stripped}</div>', None)
|
||
|
||
|
||
def extract_chapter_heading(tag: Tag) -> str:
|
||
"""Extract chapter heading from a p223 tag as a bold paragraph.
|
||
|
||
Returns something like: <b>Chapter 1<br/>Observing the Armies...</b>
|
||
"""
|
||
html = decode_html_content(tag)
|
||
if not html:
|
||
return ""
|
||
# Clean up the HTML — remove inner bold tags and re-wrap the whole thing
|
||
inner = re.sub(r"</?b>", "", html)
|
||
inner = inner.strip()
|
||
# Try to add a <br/> between "Chapter N" and the title if not already there
|
||
inner = re.sub(r"(Chapter\s+\d+)\s+", r"\1<br/>", inner, count=1)
|
||
return f"<b>{inner}</b>"
|
||
|
||
|
||
def extract_paragraphs_for_intro(
|
||
all_paragraphs: list,
|
||
start_idx: int,
|
||
end_idx: int,
|
||
) -> list:
|
||
"""Extract paragraphs for the introduction section.
|
||
|
||
Handles p27 (heading), p6 (body text), p222 (signature block).
|
||
"""
|
||
result = []
|
||
|
||
for i in range(start_idx, end_idx):
|
||
p = all_paragraphs[i]
|
||
cls = get_css_class(p)
|
||
|
||
# Skip empty paragraphs
|
||
if is_empty(p):
|
||
continue
|
||
|
||
# Skip noise classes
|
||
if cls in SKIP_CLASSES:
|
||
continue
|
||
|
||
# Skip bookmark tags
|
||
if cls == BOOKMARK_CLASS:
|
||
continue
|
||
|
||
# Skip book header (p12)
|
||
if cls == BOOK_HEADER_CLASS:
|
||
continue
|
||
|
||
# Skip subtitle (p31)
|
||
if cls == SUBTITLE_CLASS:
|
||
continue
|
||
|
||
# Introduction heading (p27)
|
||
if cls == HEADING_CLASS:
|
||
html = decode_html_content(p)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Body text (p6)
|
||
if cls == BODY_CLASS:
|
||
html = decode_html_content(p)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Signature block (p222)
|
||
if cls == SIGNATURE_CLASS:
|
||
html = decode_html_content(p)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Any other class in intro range — try to include
|
||
html = decode_html_content(p)
|
||
if html:
|
||
result.append(html)
|
||
|
||
return result
|
||
|
||
|
||
def extract_paragraphs_for_chapter(
|
||
all_paragraphs: list,
|
||
start_idx: int,
|
||
end_idx: int,
|
||
) -> list:
|
||
"""Extract paragraphs for a chapter.
|
||
|
||
Handles:
|
||
p223 — Chapter heading (bold paragraph)
|
||
p24 — Bengali verse text (centered italic div)
|
||
p224 — Text number heading (bold paragraph)
|
||
p225, p226, p15 — Skip (empty/break)
|
||
"""
|
||
result = []
|
||
deferred_translation = None
|
||
|
||
for i in range(start_idx, end_idx):
|
||
p = all_paragraphs[i]
|
||
cls = get_css_class(p)
|
||
|
||
# Skip empty paragraphs
|
||
if is_empty(p):
|
||
continue
|
||
|
||
# Skip noise classes
|
||
if cls in SKIP_CLASSES:
|
||
continue
|
||
|
||
# Skip bookmark tags
|
||
if cls == BOOKMARK_CLASS:
|
||
continue
|
||
|
||
# Chapter title heading (p223)
|
||
if cls == CHAPTER_TITLE_CLASS:
|
||
heading = extract_chapter_heading(p)
|
||
if heading:
|
||
result.append(heading)
|
||
continue
|
||
|
||
# Bengali verse text (p24)
|
||
if cls == VERSE_CLASS:
|
||
verse = extract_verse_content(p)
|
||
if verse:
|
||
result.append(verse)
|
||
# Emit deferred translation AFTER the verse
|
||
if deferred_translation:
|
||
result.append(deferred_translation)
|
||
deferred_translation = None
|
||
continue
|
||
|
||
# Text number heading (p224)
|
||
if cls == TEXT_HEADING_CLASS:
|
||
# If there's a deferred translation from a previous heading,
|
||
# emit it now (before this new heading)
|
||
if deferred_translation:
|
||
result.append(deferred_translation)
|
||
deferred_translation = None
|
||
heading, deferred = extract_text_heading(p)
|
||
if heading:
|
||
result.append(heading)
|
||
if deferred:
|
||
# Hold the translation — it will be emitted after the next verse block
|
||
deferred_translation = deferred
|
||
continue
|
||
|
||
# Heading class (p27) — may appear within chapters
|
||
if cls == HEADING_CLASS:
|
||
html = decode_html_content(p)
|
||
if html:
|
||
if not html.strip().startswith("<b>"):
|
||
html = f"<b>{html}</b>"
|
||
result.append(html)
|
||
continue
|
||
|
||
# Body text (p6) — may appear within chapters
|
||
if cls == BODY_CLASS:
|
||
html = decode_html_content(p)
|
||
if html:
|
||
result.append(html)
|
||
continue
|
||
|
||
# Any other class — try to extract content (be permissive)
|
||
html = decode_html_content(p)
|
||
if html:
|
||
result.append(html)
|
||
|
||
# Flush any remaining deferred translation at end of chapter
|
||
if deferred_translation:
|
||
result.append(deferred_translation)
|
||
deferred_translation = None
|
||
|
||
return result
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Main extraction logic
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def extract_gg(soup: BeautifulSoup) -> dict:
|
||
"""Extract the entire Gītār Gān from the parsed HTML.
|
||
|
||
Returns a dict with:
|
||
"intro_segments": list of intro segment dicts
|
||
"chapters": list of chapter dicts
|
||
"chapter_listing": list of {number, title} for intro.json
|
||
"""
|
||
all_paragraphs = list(soup.find_all("p"))
|
||
|
||
# Step 1: Find GG boundaries
|
||
gg_start, gg_end = find_gg_boundaries(all_paragraphs)
|
||
print(f" GG content: paragraphs {gg_start} to {gg_end}")
|
||
print(f" Total paragraphs in range: {gg_end - gg_start}")
|
||
|
||
# Step 2: Find all bookmarks
|
||
bookmarks = find_all_bookmarks(all_paragraphs, gg_start, gg_end)
|
||
print(f" Found {len(bookmarks)} GG bookmarks")
|
||
for pos, text in bookmarks:
|
||
print(f" Para ~{pos}: {text}")
|
||
|
||
# Step 3: Categorize bookmarks
|
||
chapter_bookmarks = [] # (para_idx, chapter_num)
|
||
|
||
for pos, text in bookmarks:
|
||
ch_num = parse_chapter_bookmark(text)
|
||
if ch_num is not None:
|
||
chapter_bookmarks.append((pos, ch_num))
|
||
|
||
print(f" Chapter bookmarks: {len(chapter_bookmarks)}")
|
||
|
||
# Sort chapter bookmarks by position
|
||
chapter_bookmarks.sort(key=lambda x: x[0])
|
||
|
||
# Build an ordered list of all section start positions for boundary computation
|
||
all_section_starts = sorted([pos for pos, _ in chapter_bookmarks])
|
||
|
||
def next_section_after(pos):
|
||
"""Return the next section start after `pos`, or gg_end."""
|
||
for s in all_section_starts:
|
||
if s > pos:
|
||
return s
|
||
return gg_end
|
||
|
||
# Step 4: Extract Introduction
|
||
# The introduction is everything between the book header and the first chapter bookmark
|
||
introduction = None
|
||
if chapter_bookmarks:
|
||
first_chapter_pos = chapter_bookmarks[0][0]
|
||
# Introduction starts after the book header (gg_start + 1) and ends at first bookmark
|
||
intro_content_start = gg_start + 1
|
||
intro_content_end = first_chapter_pos
|
||
intro_paras = extract_paragraphs_for_intro(
|
||
all_paragraphs, intro_content_start, intro_content_end
|
||
)
|
||
if intro_paras:
|
||
introduction = {
|
||
"slug": "introduction",
|
||
"title": "Introduction",
|
||
"type": "intro-segment",
|
||
"book": "gg",
|
||
"order": 1,
|
||
"paragraphs": intro_paras,
|
||
}
|
||
|
||
# Step 5: Extract chapters 1–18
|
||
chapters = []
|
||
for idx, (ch_pos, ch_num) in enumerate(chapter_bookmarks):
|
||
content_start = ch_pos + 1
|
||
content_end = next_section_after(ch_pos)
|
||
|
||
paragraphs = extract_paragraphs_for_chapter(
|
||
all_paragraphs, content_start, content_end
|
||
)
|
||
|
||
# Get title from CHAPTER_TITLES mapping, or try to extract from p223
|
||
title = CHAPTER_TITLES.get(ch_num, f"Chapter {ch_num}")
|
||
|
||
# Try to extract title from the first p223 in the range
|
||
for i in range(content_start, min(content_start + 10, content_end)):
|
||
p = all_paragraphs[i]
|
||
cls = get_css_class(p)
|
||
if cls == CHAPTER_TITLE_CLASS:
|
||
extracted = extract_chapter_title(p)
|
||
if extracted:
|
||
_, extracted_title = extracted
|
||
if extracted_title:
|
||
title = extracted_title
|
||
break
|
||
|
||
ch = {
|
||
"book": "gg",
|
||
"chapter": ch_num,
|
||
"title": title,
|
||
"type": "chapter",
|
||
"paragraphs": paragraphs,
|
||
}
|
||
chapters.append(ch)
|
||
|
||
# Step 6: Build intro segments list
|
||
intro_segments = []
|
||
if introduction:
|
||
intro_segments.append(introduction)
|
||
|
||
# Step 7: Build chapter listing for intro.json
|
||
chapter_listing = [
|
||
{"number": ch["chapter"], "title": ch["title"]} for ch in chapters
|
||
]
|
||
|
||
return {
|
||
"intro_segments": intro_segments,
|
||
"chapters": chapters,
|
||
"chapter_listing": chapter_listing,
|
||
}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Output
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def write_json(data, filepath):
|
||
"""Write data as formatted JSON to filepath, creating directories as needed."""
|
||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||
with open(filepath, "w", encoding="utf-8") as f:
|
||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||
|
||
|
||
def main():
|
||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
html_path = os.path.join(base_dir, "public", "html", "other.html")
|
||
output_base = os.path.join(base_dir, "lib", "jsons", "books", "gg")
|
||
|
||
if not os.path.exists(html_path):
|
||
print(f"ERROR: {html_path} not found!")
|
||
sys.exit(1)
|
||
|
||
print(f"Reading {html_path}...")
|
||
with open(html_path, "r", encoding="utf-8") as f:
|
||
soup = BeautifulSoup(f, "html.parser")
|
||
|
||
print("Extracting Gītār Gān...\n")
|
||
result = extract_gg(soup)
|
||
|
||
intro_segments = result["intro_segments"]
|
||
chapters = result["chapters"]
|
||
chapter_listing = result["chapter_listing"]
|
||
|
||
# --- Write intro.json ---
|
||
intro_meta = {
|
||
"book": "gg",
|
||
"title": "Gītār Gān",
|
||
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
||
"year": 1995,
|
||
"description": "A Bengali poetic rendering of Bhagavad-gītā As It Is by Śrīla Prabhupāda.",
|
||
"totalChapters": 18,
|
||
"chapters": chapter_listing,
|
||
}
|
||
intro_path = os.path.join(output_base, "intro.json")
|
||
write_json(intro_meta, intro_path)
|
||
print(f"\n Written: {intro_path}")
|
||
|
||
# --- Write intro segments ---
|
||
print(f"\n--- Intro Segments ---")
|
||
for seg in intro_segments:
|
||
filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
|
||
write_json(seg, filepath)
|
||
print(
|
||
f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}"
|
||
)
|
||
|
||
# --- Write chapters ---
|
||
print(f"\n--- Chapters ---")
|
||
for ch in chapters:
|
||
ch_num_str = str(ch["chapter"]).zfill(2)
|
||
filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
|
||
write_json(ch, filepath)
|
||
print(
|
||
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
|
||
f"{ch['title']}"
|
||
)
|
||
|
||
# --- Summary ---
|
||
total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments)
|
||
total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
|
||
total_paras = total_intro_paras + total_chapter_paras
|
||
|
||
print(f"\n{'=' * 60}")
|
||
print(f" EXTRACTION SUMMARY")
|
||
print(f"{'=' * 60}")
|
||
print(f" Intro segments: {len(intro_segments)}")
|
||
for seg in intro_segments:
|
||
print(
|
||
f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs"
|
||
)
|
||
print(f" Chapters: {len(chapters)}")
|
||
print(f" Total paragraphs: {total_paras}")
|
||
print(f" Intro: {total_intro_paras}")
|
||
print(f" Chapters: {total_chapter_paras}")
|
||
if chapters:
|
||
print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
|
||
print(f" Output directory: {output_base}/")
|
||
|
||
# --- Samples ---
|
||
print(f"\n--- Sample: Chapter 1 ---")
|
||
if chapters:
|
||
ch1 = chapters[0]
|
||
for i, p in enumerate(ch1["paragraphs"][:5]):
|
||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
||
print(f" [{i}] {preview}...")
|
||
|
||
if len(chapters) >= 9:
|
||
print(f"\n--- Sample: Chapter 9 ---")
|
||
ch9 = [c for c in chapters if c["chapter"] == 9]
|
||
if ch9:
|
||
for i, p in enumerate(ch9[0]["paragraphs"][:3]):
|
||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
||
print(f" [{i}] {preview}...")
|
||
|
||
if len(chapters) >= 18:
|
||
print(f"\n--- Sample: Chapter 18 (last) ---")
|
||
last = chapters[-1]
|
||
for i, p in enumerate(last["paragraphs"][:3]):
|
||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
||
print(f" [{i}] {preview}...")
|
||
|
||
# --- Quality Checks ---
|
||
print(f"\n--- Quality Checks ---")
|
||
|
||
# Check all 18 chapters present
|
||
expected_chapters = set(range(1, 19))
|
||
actual_chapters = set(ch["chapter"] for ch in chapters)
|
||
missing = expected_chapters - actual_chapters
|
||
extra = actual_chapters - expected_chapters
|
||
if missing:
|
||
print(f" ⚠ Missing chapters: {sorted(missing)}")
|
||
if extra:
|
||
print(f" ⚠ Extra chapters: {sorted(extra)}")
|
||
if not missing and not extra:
|
||
print(f" ✅ All 18 chapters present")
|
||
|
||
# Check intro segments
|
||
expected_intros = {"introduction"}
|
||
actual_intros = {s["slug"] for s in intro_segments}
|
||
missing_intros = expected_intros - actual_intros
|
||
if missing_intros:
|
||
print(f" ⚠ Missing intro segments: {missing_intros}")
|
||
else:
|
||
print(f" ✅ Introduction segment present")
|
||
|
||
# Check empty chapters
|
||
empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
|
||
if empty_chapters:
|
||
print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
|
||
else:
|
||
print(f" ✅ No empty chapters")
|
||
|
||
# Check ScaGoudy decoding — look for common encoded characters
|
||
encoded_chars = set("äåöëòçüéàùè")
|
||
suspect_count = 0
|
||
all_sections = chapters + intro_segments
|
||
for section in all_sections:
|
||
for p in section["paragraphs"]:
|
||
plain = re.sub(r"<[^>]+>", "", p)
|
||
for c in encoded_chars:
|
||
if c in plain:
|
||
suspect_count += 1
|
||
break
|
||
if suspect_count == 0:
|
||
print(f" ✅ No suspected undecoded ScaGoudy characters found")
|
||
else:
|
||
print(
|
||
f" ℹ {suspect_count} paragraphs contain characters that may be undecoded ScaGoudy"
|
||
)
|
||
|
||
# Check verse blocks
|
||
verse_block_count = 0
|
||
for ch in chapters:
|
||
for p in ch["paragraphs"]:
|
||
if '<div style="text-align:center"><i>' in p:
|
||
verse_block_count += 1
|
||
print(f" ℹ {verse_block_count} verse blocks in chapters")
|
||
|
||
# Check text headings
|
||
text_heading_count = 0
|
||
for ch in chapters:
|
||
for p in ch["paragraphs"]:
|
||
if re.match(r"<b>Texts?\s+\d+", p):
|
||
text_heading_count += 1
|
||
print(f" ℹ {text_heading_count} text number headings in chapters")
|
||
|
||
# ScaGoudy decoding spot check
|
||
print(f"\n--- ScaGoudy Decoding Spot Check ---")
|
||
if chapters:
|
||
ch1_text = " ".join(chapters[0]["paragraphs"][:15])
|
||
if "Kṛṣṇa" in ch1_text:
|
||
print(f" ✅ 'Kṛṣṇa' correctly decoded in Chapter 1")
|
||
elif "Kåñëa" in ch1_text:
|
||
print(f" ⚠ 'Kåñëa' still encoded — ScaGoudy decoding may have failed!")
|
||
else:
|
||
print(
|
||
f" ℹ 'Kṛṣṇa' not found in Chapter 1 first 15 paragraphs (may be normal)"
|
||
)
|
||
|
||
if "Kurukṣetra" in ch1_text:
|
||
print(f" ✅ 'Kurukṣetra' correctly decoded in Chapter 1")
|
||
elif "Kurukñetra" in ch1_text:
|
||
print(f" ⚠ 'Kurukñetra' still encoded")
|
||
else:
|
||
print(f" ℹ 'Kurukṣetra' not found in Chapter 1 first 15 paragraphs")
|
||
|
||
# Sample verse blocks
|
||
print(f"\n--- Sample Verse Blocks ---")
|
||
shown = 0
|
||
for ch in chapters:
|
||
for p in ch["paragraphs"]:
|
||
if '<div style="text-align:center"><i>' in p:
|
||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
||
print(f" Ch {ch['chapter']}: {preview}...")
|
||
shown += 1
|
||
if shown >= 3:
|
||
break
|
||
if shown >= 3:
|
||
break
|
||
|
||
# Check chapter headings
|
||
print(f"\n--- Chapter Headings ---")
|
||
heading_count = 0
|
||
for ch in chapters:
|
||
for p in ch["paragraphs"]:
|
||
if p.startswith("<b>Chapter"):
|
||
heading_count += 1
|
||
preview = re.sub(r"<[^>]+>", "", p)[:80]
|
||
if heading_count <= 5:
|
||
print(f" Ch {ch['chapter']}: {preview}")
|
||
break
|
||
print(f" Total chapter headings: {heading_count}")
|
||
|
||
# Check closing statements
|
||
print(f"\n--- Closing Statements ---")
|
||
closing_count = 0
|
||
for ch in chapters:
|
||
for p in ch["paragraphs"]:
|
||
plain = re.sub(r"<[^>]+>", "", p)
|
||
if "Bhaktivedānta" in plain and "sings" in plain.lower():
|
||
closing_count += 1
|
||
break
|
||
print(f" Chapters with closing statement: {closing_count} / {len(chapters)}")
|
||
|
||
print(f"\nDone! ✨")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|