751 lines
24 KiB
Python
751 lines
24 KiB
Python
#!/usr/bin/env python3
|
|
"""Extract Easy Journey to Other Planets (1970 Edition) from other.html.
|
|
|
|
The book has 2 intro segments (front-pages, preface) and 2 chapters.
|
|
|
|
Output:
|
|
lib/jsons/books/ej/intro.json — Book metadata
|
|
lib/jsons/books/ej/intro/*.json — Intro segments
|
|
lib/jsons/books/ej/chapters/*.json — Chapters
|
|
|
|
Usage:
|
|
python3 scripts/ej_extract.py
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
from bs4 import BeautifulSoup, Tag
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ScaGoudy → IAST Unicode mapping
|
|
# ---------------------------------------------------------------------------
|
|
SCAGOUDY_TO_IAST = {
|
|
"ä": "ā",
|
|
"Ä": "Ā",
|
|
"å": "ṛ",
|
|
"Å": "Ṛ",
|
|
"ñ": "ṣ",
|
|
"Ñ": "Ṣ",
|
|
"ö": "ṭ",
|
|
"Ö": "Ṭ",
|
|
"ë": "ṇ",
|
|
"Ë": "Ṇ",
|
|
"ò": "ḍ",
|
|
"Ò": "Ḍ",
|
|
"ç": "ś",
|
|
"Ç": "Ś",
|
|
"ì": "ṅ",
|
|
"ï": "ñ",
|
|
"ü": "ū",
|
|
"Ü": "Ū",
|
|
"é": "ī",
|
|
"É": "Ī",
|
|
"à": "ṁ",
|
|
"ù": "ḥ",
|
|
"è": "ṝ",
|
|
"û": "ṁl",
|
|
"ÿ": "ḷ",
|
|
}
|
|
|
|
|
|
def decode_scagoudy(text: str) -> str:
|
|
"""Replace ScaGoudy-encoded characters with proper IAST Unicode."""
|
|
for src, dst in SCAGOUDY_TO_IAST.items():
|
|
text = text.replace(src, dst)
|
|
return text
|
|
|
|
|
|
def decode_html_content(tag: Tag) -> str:
|
|
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
|
|
inner = "".join(str(child) for child in tag.children)
|
|
inner = decode_scagoudy(inner)
|
|
# Remove span tags with CSS classes (RTF conversion artifacts)
|
|
inner = re.sub(
|
|
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
|
|
)
|
|
# Clean up Apple-tab-span leftovers (tab characters)
|
|
inner = inner.replace("\t", " ")
|
|
# Normalize whitespace (but preserve <br> tags)
|
|
inner = re.sub(r"\s+", " ", inner).strip()
|
|
return inner
|
|
|
|
|
|
def decode_text(tag: Tag) -> str:
|
|
"""Get plain text from a tag with ScaGoudy decoding."""
|
|
return decode_scagoudy(tag.get_text().strip())
|
|
|
|
|
|
def get_css_class(tag: Tag) -> str:
|
|
"""Get the first CSS class of a tag, or empty string."""
|
|
classes = tag.get("class", [])
|
|
return classes[0] if classes else ""
|
|
|
|
|
|
def is_empty(tag: Tag) -> bool:
|
|
"""Check if a tag is empty or whitespace-only."""
|
|
text = tag.get_text().strip()
|
|
return not text or text == "\xa0"
|
|
|
|
|
|
def is_dash_line(text: str) -> bool:
|
|
"""Return True if text is only dashes, spaces, and/or whitespace."""
|
|
cleaned = text.strip()
|
|
if not cleaned:
|
|
return False
|
|
return all(c in "-—–\u2014\u2013 \t" for c in cleaned)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# CSS class categorization for EJ'70
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Book title header class
|
|
BOOK_HEADER_CLASS = "p12"
|
|
|
|
# Section bookmark class (Preface, Glossary)
|
|
SECTION_BOOKMARK_CLASS = "p64"
|
|
|
|
# Chapter bookmark class
|
|
CHAPTER_BOOKMARK_CLASS = "p49"
|
|
|
|
# Body paragraph classes (primary content)
|
|
BODY_CLASSES = frozenset(["p45", "p63", "p72", "p71"])
|
|
|
|
# Sanskrit verse pāda classes (italic, centered)
|
|
VERSE_PADA_CLASSES = frozenset(["p10", "p73"])
|
|
|
|
# Section heading (bold) — "Preface", "1 Anti-material Worlds", etc.
|
|
SECTION_HEADING_CLASS = "p31"
|
|
|
|
# Front-pages specific classes
|
|
BOOK_LIST_CLASS = "p68"
|
|
TITLE_PAGE_CLASS = "p3"
|
|
BOOK_TITLE_CLASS = "p69"
|
|
|
|
# Price text — skip
|
|
PRICE_CLASS = "p42"
|
|
|
|
# Glossary header — end boundary marker
|
|
GLOSSARY_HEADER_CLASS = "p19"
|
|
|
|
# Classes to skip entirely (empty/decorative)
|
|
SKIP_CLASSES = frozenset(["p9", "p70"])
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Intro segment definitions
|
|
# ---------------------------------------------------------------------------
|
|
|
|
INTRO_SEGMENTS = [
|
|
{
|
|
"slug": "front-pages",
|
|
"title": "Front Pages",
|
|
"order": 1,
|
|
"boundary_type": "start", # from book start to first section bookmark
|
|
},
|
|
{
|
|
"slug": "preface",
|
|
"title": "Preface",
|
|
"order": 2,
|
|
"bookmark_text": "EJ'70:Preface",
|
|
},
|
|
]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Extraction helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def find_ej_boundaries(all_paragraphs: list) -> tuple:
|
|
"""Find the start and end paragraph indices of the EJ'70 content.
|
|
|
|
Start: <p class="p12"> containing "Easy Journey" and "1970"
|
|
End: next <p class="p12"> after start, or the EJ'70:Glossary bookmark
|
|
"""
|
|
start_idx = None
|
|
end_idx = len(all_paragraphs)
|
|
|
|
for i, p in enumerate(all_paragraphs):
|
|
cls = get_css_class(p)
|
|
if cls != BOOK_HEADER_CLASS:
|
|
continue
|
|
text = decode_text(p)
|
|
if "Easy Journey" in text and "1970" in text:
|
|
start_idx = i
|
|
elif start_idx is not None:
|
|
# Next p12 after EJ start = end of EJ
|
|
end_idx = i
|
|
break
|
|
|
|
if start_idx is None:
|
|
print(
|
|
"ERROR: Could not find 'Easy Journey to other planets (1970)' header in other.html!"
|
|
)
|
|
sys.exit(1)
|
|
|
|
# Also check for the EJ'70:Glossary bookmark as an end boundary
|
|
for i in range(start_idx + 1, end_idx):
|
|
p = all_paragraphs[i]
|
|
cls = get_css_class(p)
|
|
if cls == SECTION_BOOKMARK_CLASS:
|
|
text = decode_text(p)
|
|
if "EJ'70:Glossary" in text:
|
|
end_idx = i
|
|
break
|
|
|
|
return start_idx, end_idx
|
|
|
|
|
|
def find_all_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
|
|
"""Find all bookmark tags (p64 and p49) in the EJ range.
|
|
|
|
Returns list of (paragraph_index, bookmark_text, bookmark_type).
|
|
bookmark_type is 'section' for p64, 'chapter' for p49.
|
|
"""
|
|
bookmarks = []
|
|
for i in range(start_idx, end_idx):
|
|
p = all_paragraphs[i]
|
|
cls = get_css_class(p)
|
|
if cls == SECTION_BOOKMARK_CLASS:
|
|
text = decode_text(p)
|
|
if "EJ'70:" in text:
|
|
bookmarks.append((i, text, "section"))
|
|
elif cls == CHAPTER_BOOKMARK_CLASS:
|
|
text = decode_text(p)
|
|
if "EJ'70:" in text:
|
|
bookmarks.append((i, text, "chapter"))
|
|
return bookmarks
|
|
|
|
|
|
def parse_chapter_bookmark(text: str) -> tuple:
|
|
"""Parse a chapter bookmark like "EJ'70:1 Anti-material Worlds".
|
|
|
|
Returns (chapter_number, title) or None if not a chapter bookmark.
|
|
"""
|
|
m = re.match(r"EJ'70:(\d+)\s+(.+)", text.strip())
|
|
if m:
|
|
return int(m.group(1)), m.group(2).strip()
|
|
return None
|
|
|
|
|
|
def parse_section_bookmark(text: str) -> str:
|
|
"""Parse a section bookmark like "EJ'70:Preface" or "EJ'70:Glossary".
|
|
|
|
Returns the section name or None.
|
|
"""
|
|
m = re.match(r"EJ'70:(\w+)", text.strip())
|
|
if m:
|
|
return m.group(1).strip()
|
|
return None
|
|
|
|
|
|
def extract_paragraphs(
|
|
all_paragraphs: list,
|
|
start_idx: int,
|
|
end_idx: int,
|
|
is_front_pages: bool = False,
|
|
) -> list:
|
|
"""Extract paragraphs for a section (chapter or intro segment).
|
|
|
|
Handles all CSS classes within the EJ content.
|
|
Groups consecutive p10/p73 verse pādas into centered italic divs.
|
|
"""
|
|
result = []
|
|
verse_accumulator = []
|
|
|
|
def flush_verse():
|
|
if verse_accumulator:
|
|
joined = "<br>".join(verse_accumulator)
|
|
result.append(f'<div style="text-align:center"><i>{joined}</i></div>')
|
|
verse_accumulator.clear()
|
|
|
|
for i in range(start_idx, end_idx):
|
|
p = all_paragraphs[i]
|
|
cls = get_css_class(p)
|
|
|
|
# Skip empty paragraphs
|
|
if is_empty(p):
|
|
continue
|
|
|
|
# Skip noise classes
|
|
if cls in SKIP_CLASSES:
|
|
continue
|
|
|
|
# Skip bookmark tags (both p64 and p49)
|
|
if cls in (SECTION_BOOKMARK_CLASS, CHAPTER_BOOKMARK_CLASS):
|
|
continue
|
|
|
|
# Skip book title header (p12)
|
|
if cls == BOOK_HEADER_CLASS:
|
|
continue
|
|
|
|
# Skip price text (p42)
|
|
if cls == PRICE_CLASS:
|
|
continue
|
|
|
|
# Skip glossary header (p19)
|
|
if cls == GLOSSARY_HEADER_CLASS:
|
|
continue
|
|
|
|
# Check for dash lines — skip them
|
|
text = decode_text(p)
|
|
if is_dash_line(text):
|
|
continue
|
|
|
|
# Sanskrit verse pādas (p10, p73) — accumulate into centered block
|
|
if cls in VERSE_PADA_CLASSES:
|
|
html = decode_html_content(p)
|
|
if html:
|
|
# Remove inner <i> tags before re-wrapping
|
|
stripped = re.sub(r"</?i>", "", html)
|
|
verse_accumulator.append(stripped)
|
|
continue
|
|
|
|
# Non-verse paragraph — flush any accumulated verse first
|
|
flush_verse()
|
|
|
|
# Section heading (p31) — include as paragraph
|
|
if cls == SECTION_HEADING_CLASS:
|
|
html = decode_html_content(p)
|
|
if html:
|
|
result.append(html)
|
|
continue
|
|
|
|
# Book title bold (p69) — include in front-pages
|
|
if cls == BOOK_TITLE_CLASS:
|
|
html = decode_html_content(p)
|
|
if html:
|
|
result.append(html)
|
|
continue
|
|
|
|
# Title page text (p3) — include in front-pages
|
|
if cls == TITLE_PAGE_CLASS:
|
|
html = decode_html_content(p)
|
|
if html:
|
|
result.append(html)
|
|
continue
|
|
|
|
# Book list items (p68) — include in front-pages
|
|
if cls == BOOK_LIST_CLASS:
|
|
html = decode_html_content(p)
|
|
if html:
|
|
result.append(html)
|
|
continue
|
|
|
|
# Body paragraphs (p45, p63, p72, p71)
|
|
if cls in BODY_CLASSES:
|
|
html = decode_html_content(p)
|
|
if html:
|
|
result.append(html)
|
|
continue
|
|
|
|
# Any other class — try to extract content (be permissive)
|
|
html = decode_html_content(p)
|
|
if html and not is_dash_line(text):
|
|
result.append(html)
|
|
|
|
# Flush any trailing verse
|
|
flush_verse()
|
|
|
|
return result
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main extraction logic
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def extract_ej(soup: BeautifulSoup) -> dict:
|
|
"""Extract the entire Easy Journey to Other Planets (1970) from the parsed HTML.
|
|
|
|
Returns a dict with:
|
|
"intro_segments": list of intro segment dicts
|
|
"chapters": list of chapter dicts
|
|
"chapter_listing": list of {number, title} for intro.json
|
|
"""
|
|
all_paragraphs = list(soup.find_all("p"))
|
|
|
|
# Step 1: Find EJ boundaries
|
|
ej_start, ej_end = find_ej_boundaries(all_paragraphs)
|
|
print(f" EJ'70 content: paragraphs {ej_start} to {ej_end}")
|
|
print(f" Total paragraphs in range: {ej_end - ej_start}")
|
|
|
|
# Step 2: Find all bookmarks
|
|
bookmarks = find_all_bookmarks(all_paragraphs, ej_start, ej_end)
|
|
print(f" Found {len(bookmarks)} EJ'70 bookmarks")
|
|
for pos, text, btype in bookmarks:
|
|
print(f" [{btype}] paragraph {pos}: {text}")
|
|
|
|
# Step 3: Categorize bookmarks into section vs chapter
|
|
section_bookmarks = [] # (para_idx, bookmark_text)
|
|
chapter_bookmarks = [] # (para_idx, chapter_num, title)
|
|
|
|
for pos, text, btype in bookmarks:
|
|
ch = parse_chapter_bookmark(text)
|
|
if ch:
|
|
chapter_bookmarks.append((pos, ch[0], ch[1]))
|
|
else:
|
|
section_name = parse_section_bookmark(text)
|
|
if section_name and section_name != "Glossary":
|
|
section_bookmarks.append((pos, text))
|
|
|
|
print(f" Section bookmarks: {len(section_bookmarks)}")
|
|
print(f" Chapter bookmarks: {len(chapter_bookmarks)}")
|
|
|
|
# Step 4: Collect all boundary positions for range detection
|
|
# These are all the positions where a new section/chapter starts
|
|
all_boundary_positions = sorted(
|
|
[pos for pos, _ in section_bookmarks] + [pos for pos, _, _ in chapter_bookmarks]
|
|
)
|
|
|
|
# Step 5: Extract intro segments
|
|
intro_segments = []
|
|
|
|
# --- Front Pages ---
|
|
# From book start (ej_start) to first bookmark (Preface)
|
|
front_pages_end = all_boundary_positions[0] if all_boundary_positions else ej_end
|
|
front_pages_paragraphs = extract_paragraphs(
|
|
all_paragraphs, ej_start, front_pages_end, is_front_pages=True
|
|
)
|
|
intro_segments.append(
|
|
{
|
|
"slug": "front-pages",
|
|
"title": "Front Pages",
|
|
"type": "intro-segment",
|
|
"book": "ej",
|
|
"order": 1,
|
|
"paragraphs": front_pages_paragraphs,
|
|
}
|
|
)
|
|
|
|
# --- Preface ---
|
|
# From EJ'70:Preface bookmark to first chapter bookmark
|
|
preface_pos = None
|
|
for pos, text in section_bookmarks:
|
|
if "Preface" in text:
|
|
preface_pos = pos
|
|
break
|
|
|
|
if preface_pos is not None:
|
|
# Content starts after the bookmark
|
|
preface_start = preface_pos + 1
|
|
|
|
# Content ends at the first chapter bookmark
|
|
preface_end = ej_end
|
|
for s in all_boundary_positions:
|
|
if s > preface_pos:
|
|
preface_end = s
|
|
break
|
|
|
|
preface_paragraphs = extract_paragraphs(
|
|
all_paragraphs, preface_start, preface_end
|
|
)
|
|
intro_segments.append(
|
|
{
|
|
"slug": "preface",
|
|
"title": "Preface",
|
|
"type": "intro-segment",
|
|
"book": "ej",
|
|
"order": 2,
|
|
"paragraphs": preface_paragraphs,
|
|
}
|
|
)
|
|
else:
|
|
print(" WARNING: Could not find Preface bookmark!")
|
|
|
|
# Step 6: Extract chapters
|
|
chapters = []
|
|
for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_bookmarks):
|
|
content_start = ch_pos + 1
|
|
|
|
# Content ends at next chapter bookmark or ej_end
|
|
if idx + 1 < len(chapter_bookmarks):
|
|
content_end = chapter_bookmarks[idx + 1][0]
|
|
else:
|
|
content_end = ej_end
|
|
|
|
paragraphs = extract_paragraphs(all_paragraphs, content_start, content_end)
|
|
|
|
ch = {
|
|
"book": "ej",
|
|
"chapter": ch_num,
|
|
"title": ch_title,
|
|
"type": "chapter",
|
|
"paragraphs": paragraphs,
|
|
}
|
|
chapters.append(ch)
|
|
|
|
# Step 7: Build chapter listing for intro.json
|
|
chapter_listing = [
|
|
{"number": ch["chapter"], "title": ch["title"]} for ch in chapters
|
|
]
|
|
|
|
return {
|
|
"intro_segments": intro_segments,
|
|
"chapters": chapters,
|
|
"chapter_listing": chapter_listing,
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Output
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def write_json(data, filepath):
|
|
"""Write data as formatted JSON to filepath, creating directories as needed."""
|
|
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
|
with open(filepath, "w", encoding="utf-8") as f:
|
|
json.dump(data, f, ensure_ascii=False, indent=2)
|
|
|
|
|
|
def main():
|
|
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
html_path = os.path.join(base_dir, "public", "html", "other.html")
|
|
output_base = os.path.join(base_dir, "lib", "jsons", "books", "ej")
|
|
|
|
if not os.path.exists(html_path):
|
|
print(f"ERROR: {html_path} not found!")
|
|
sys.exit(1)
|
|
|
|
print(f"Reading {html_path}...")
|
|
with open(html_path, "r", encoding="utf-8") as f:
|
|
soup = BeautifulSoup(f, "html.parser")
|
|
|
|
print("Extracting Easy Journey to Other Planets (1970 Edition)...\n")
|
|
result = extract_ej(soup)
|
|
|
|
intro_segments = result["intro_segments"]
|
|
chapters = result["chapters"]
|
|
chapter_listing = result["chapter_listing"]
|
|
|
|
# --- Write intro.json ---
|
|
intro_meta = {
|
|
"book": "ej",
|
|
"title": "Easy Journey to Other Planets",
|
|
"subtitle": "By Practice of Supreme Yoga",
|
|
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
|
"year": 1970,
|
|
"description": "How one can transfer oneself to other planets by the process of bhakti-yoga.",
|
|
"totalChapters": 2,
|
|
"chapters": chapter_listing,
|
|
}
|
|
intro_path = os.path.join(output_base, "intro.json")
|
|
write_json(intro_meta, intro_path)
|
|
print(f"\n Written: {intro_path}")
|
|
|
|
# --- Write intro segments ---
|
|
print(f"\n--- Intro Segments ---")
|
|
for seg in intro_segments:
|
|
filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
|
|
write_json(seg, filepath)
|
|
print(
|
|
f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}"
|
|
)
|
|
|
|
# --- Write chapters ---
|
|
print(f"\n--- Chapters ---")
|
|
for ch in chapters:
|
|
ch_num_str = str(ch["chapter"]).zfill(2)
|
|
filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
|
|
write_json(ch, filepath)
|
|
print(
|
|
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
|
|
f"{ch['title']}"
|
|
)
|
|
|
|
# --- Summary ---
|
|
total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments)
|
|
total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
|
|
total_paras = total_intro_paras + total_chapter_paras
|
|
|
|
print(f"\n{'=' * 60}")
|
|
print(f" EXTRACTION SUMMARY")
|
|
print(f"{'=' * 60}")
|
|
print(f" Intro segments: {len(intro_segments)}")
|
|
print(f" Chapters: {len(chapters)}")
|
|
print(f" Total paragraphs: {total_paras}")
|
|
print(f" Intro: {total_intro_paras}")
|
|
print(f" Chapters: {total_chapter_paras}")
|
|
if chapters:
|
|
print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
|
|
print(f" Output directory: {output_base}/")
|
|
|
|
# --- Samples ---
|
|
print(f"\n--- Sample: Front Pages ---")
|
|
if intro_segments:
|
|
front = intro_segments[0]
|
|
for i, p in enumerate(front["paragraphs"][:5]):
|
|
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
|
print(f" [{i}] {preview}...")
|
|
|
|
if len(intro_segments) > 1:
|
|
print(f"\n--- Sample: Preface ---")
|
|
preface = intro_segments[1]
|
|
for i, p in enumerate(preface["paragraphs"][:3]):
|
|
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
|
print(f" [{i}] {preview}...")
|
|
|
|
if chapters:
|
|
print(f"\n--- Sample: Chapter 1 ({chapters[0]['title']}) ---")
|
|
ch1 = chapters[0]
|
|
for i, p in enumerate(ch1["paragraphs"][:5]):
|
|
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
|
print(f" [{i}] {preview}...")
|
|
|
|
if chapters and len(chapters) > 1:
|
|
print(
|
|
f"\n--- Sample: Chapter {chapters[-1]['chapter']} ({chapters[-1]['title']}) ---"
|
|
)
|
|
last = chapters[-1]
|
|
for i, p in enumerate(last["paragraphs"][:5]):
|
|
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
|
print(f" [{i}] {preview}...")
|
|
|
|
# --- Quality Checks ---
|
|
print(f"\n--- Quality Checks ---")
|
|
|
|
# Check all 2 chapters present
|
|
expected_chapters = set(range(1, 3))
|
|
actual_chapters = set(ch["chapter"] for ch in chapters)
|
|
missing = expected_chapters - actual_chapters
|
|
extra = actual_chapters - expected_chapters
|
|
if missing:
|
|
print(f" ⚠ Missing chapters: {sorted(missing)}")
|
|
if extra:
|
|
print(f" ⚠ Extra chapters: {sorted(extra)}")
|
|
if not missing and not extra:
|
|
print(f" ✅ All 2 chapters present")
|
|
|
|
# Check all 2 intro segments present
|
|
expected_intros = {"front-pages", "preface"}
|
|
actual_intros = {s["slug"] for s in intro_segments}
|
|
missing_intros = expected_intros - actual_intros
|
|
extra_intros = actual_intros - expected_intros
|
|
if missing_intros:
|
|
print(f" ⚠ Missing intro segments: {missing_intros}")
|
|
if extra_intros:
|
|
print(f" ⚠ Extra intro segments: {extra_intros}")
|
|
if not missing_intros and not extra_intros:
|
|
print(f" ✅ All 2 intro segments present")
|
|
|
|
# Check empty chapters
|
|
empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
|
|
if empty_chapters:
|
|
print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
|
|
else:
|
|
print(f" ✅ No empty chapters")
|
|
|
|
# Check empty intro segments
|
|
empty_intros = [s["slug"] for s in intro_segments if len(s["paragraphs"]) == 0]
|
|
if empty_intros:
|
|
print(f" ⚠ Empty intro segments: {empty_intros}")
|
|
else:
|
|
print(f" ✅ No empty intro segments")
|
|
|
|
# Check ScaGoudy decoding
|
|
print(f"\n--- ScaGoudy Decoding Check ---")
|
|
all_text = ""
|
|
for ch in chapters:
|
|
all_text += " ".join(ch["paragraphs"][:20])
|
|
for seg in intro_segments:
|
|
all_text += " ".join(seg["paragraphs"][:10])
|
|
|
|
# Check for properly decoded words
|
|
if "Kṛṣṇa" in all_text:
|
|
print(f" ✅ 'Kṛṣṇa' found (ScaGoudy properly decoded)")
|
|
else:
|
|
print(f" ⚠ 'Kṛṣṇa' not found — ScaGoudy decoding may have failed")
|
|
|
|
if "Bhagavad-gītā" in all_text:
|
|
print(f" ✅ 'Bhagavad-gītā' found (ScaGoudy properly decoded)")
|
|
elif "Gītā" in all_text or "gītā" in all_text:
|
|
print(f" ✅ 'gītā' found (ScaGoudy properly decoded)")
|
|
else:
|
|
print(f" ⚠ 'Gītā' not found — ScaGoudy decoding may have failed")
|
|
|
|
if "Prabhupāda" in all_text:
|
|
print(f" ✅ 'Prabhupāda' found (ScaGoudy properly decoded)")
|
|
|
|
# Check for un-decoded ScaGoudy remnants (common encoded chars)
|
|
scagoudy_remnants = 0
|
|
# Look for patterns that are very likely ScaGoudy artifacts
|
|
# e.g., "Kåñëa" instead of "Kṛṣṇa"
|
|
for ch in chapters:
|
|
for p in ch["paragraphs"]:
|
|
if "Kåñëa" in p or "Prabhupäda" in p:
|
|
scagoudy_remnants += 1
|
|
for seg in intro_segments:
|
|
for p in seg["paragraphs"]:
|
|
if "Kåñëa" in p or "Prabhupäda" in p:
|
|
scagoudy_remnants += 1
|
|
if scagoudy_remnants == 0:
|
|
print(f" ✅ No ScaGoudy remnants detected")
|
|
else:
|
|
print(f" ⚠ {scagoudy_remnants} paragraphs with ScaGoudy remnants")
|
|
|
|
# Check for centered verse blocks
|
|
verse_block_count = 0
|
|
for ch in chapters:
|
|
for p in ch["paragraphs"]:
|
|
if '<div style="text-align:center"><i>' in p:
|
|
verse_block_count += 1
|
|
if verse_block_count > 0:
|
|
print(f" ✅ {verse_block_count} centered Sanskrit verse blocks found")
|
|
else:
|
|
print(f" ⚠ No centered verse blocks found — verse grouping may have failed")
|
|
|
|
# Check for residual span tags
|
|
span_count = 0
|
|
for ch in chapters:
|
|
for p in ch["paragraphs"]:
|
|
if 'class="' in p and "<span" in p:
|
|
span_count += 1
|
|
for seg in intro_segments:
|
|
for p in seg["paragraphs"]:
|
|
if 'class="' in p and "<span" in p:
|
|
span_count += 1
|
|
if span_count == 0:
|
|
print(f" ✅ No residual span class artifacts found")
|
|
else:
|
|
print(f" ⚠ {span_count} paragraphs still contain span class artifacts")
|
|
|
|
# Check for dash-line paragraphs that shouldn't be there
|
|
dash_count = 0
|
|
for ch in chapters:
|
|
for p in ch["paragraphs"]:
|
|
plain = re.sub(r"<[^>]+>", "", p)
|
|
if is_dash_line(plain):
|
|
dash_count += 1
|
|
for seg in intro_segments:
|
|
for p in seg["paragraphs"]:
|
|
plain = re.sub(r"<[^>]+>", "", p)
|
|
if is_dash_line(plain):
|
|
dash_count += 1
|
|
if dash_count == 0:
|
|
print(f" ✅ No dashed-line paragraphs in output")
|
|
else:
|
|
print(f" ⚠ {dash_count} dashed-line paragraphs found in output")
|
|
|
|
# Show a verse block sample
|
|
print(f"\n--- Sample: Verse Block ---")
|
|
for ch in chapters:
|
|
for p in ch["paragraphs"]:
|
|
if '<div style="text-align:center"><i>' in p:
|
|
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
|
print(f" {preview}...")
|
|
break
|
|
else:
|
|
continue
|
|
break
|
|
|
|
print(f"\nDone! ✨")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|