1072 lines
37 KiB
Python
1072 lines
37 KiB
Python
#!/usr/bin/env python3
|
||
"""Extract The Science of Self-Realization (1977) from other.html.
|
||
|
||
Hierarchical sub-chapter extraction: each sub-chapter (e.g., SSR 1a, SSR 1b)
|
||
becomes its OWN chapter JSON file with a parent-child relationship.
|
||
|
||
Parent chapters (SSR 1–8) are containers only — no content of their own.
|
||
|
||
Numbering: SSR 1a → slug "1-1", SSR 1b → slug "1-2", etc.
|
||
|
||
Sub-chapter bookmarks use class p49 or p64.
|
||
Parent chapter bookmarks use class p13 (chapters 1-7) or p151 (chapter 8).
|
||
p27 = Roman numeral parent chapter heading (skip).
|
||
p36 = Sub-chapter title duplicate immediately after bookmark (skip first one).
|
||
|
||
Output:
|
||
lib/jsons/books/ssr/intro.json — Book metadata with hierarchical chapters
|
||
lib/jsons/books/ssr/intro/*.json — Intro segments
|
||
lib/jsons/books/ssr/chapters/*.json — Sub-chapter files (1-1.json, 1-2.json, ...)
|
||
|
||
Usage:
|
||
python3 scripts/ssr_extract.py
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import re
|
||
import sys
|
||
|
||
from bs4 import BeautifulSoup, Tag
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# ScaGoudy → IAST Unicode mapping
|
||
# ---------------------------------------------------------------------------
|
||
SCAGOUDY_TO_IAST = {
|
||
"ä": "ā",
|
||
"Ä": "Ā",
|
||
"å": "ṛ",
|
||
"Å": "Ṛ",
|
||
"ñ": "ṣ",
|
||
"Ñ": "Ṣ",
|
||
"ö": "ṭ",
|
||
"Ö": "Ṭ",
|
||
"ë": "ṇ",
|
||
"Ë": "Ṇ",
|
||
"ò": "ḍ",
|
||
"Ò": "Ḍ",
|
||
"ç": "ś",
|
||
"Ç": "Ś",
|
||
"ì": "ṅ",
|
||
"ï": "ñ",
|
||
"ü": "ū",
|
||
"Ü": "Ū",
|
||
"é": "ī",
|
||
"É": "Ī",
|
||
"à": "ṁ",
|
||
"ù": "ḥ",
|
||
"è": "ṝ",
|
||
"û": "ṁl",
|
||
"ÿ": "ḷ",
|
||
}
|
||
|
||
|
||
def decode_scagoudy(text: str) -> str:
|
||
"""Replace ScaGoudy-encoded characters with proper IAST Unicode."""
|
||
for src, dst in SCAGOUDY_TO_IAST.items():
|
||
text = text.replace(src, dst)
|
||
return text
|
||
|
||
|
||
def decode_html_content(tag: Tag) -> str:
|
||
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
|
||
inner = "".join(str(child) for child in tag.children)
|
||
inner = decode_scagoudy(inner)
|
||
# Remove span tags with CSS classes (RTF conversion artifacts)
|
||
inner = re.sub(
|
||
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
|
||
)
|
||
# Clean up Apple-tab-span leftovers (tab characters)
|
||
inner = inner.replace("\t", " ")
|
||
# Normalize whitespace (but preserve <br> tags)
|
||
inner = re.sub(r"\s+", " ", inner).strip()
|
||
return inner
|
||
|
||
|
||
def decode_text(tag: Tag) -> str:
|
||
"""Get plain text from a tag with ScaGoudy decoding."""
|
||
return decode_scagoudy(tag.get_text().strip())
|
||
|
||
|
||
def get_css_class(tag: Tag) -> str:
|
||
"""Get the first CSS class of a tag, or empty string."""
|
||
classes = tag.get("class", [])
|
||
return classes[0] if classes else ""
|
||
|
||
|
||
def is_empty(tag: Tag) -> bool:
|
||
"""Check if a tag is empty or whitespace-only."""
|
||
text = tag.get_text().strip()
|
||
return not text or text == "\xa0"
|
||
|
||
|
||
def is_dash_line(text: str) -> bool:
|
||
"""Return True if text is only dashes, spaces, and/or whitespace."""
|
||
cleaned = text.strip()
|
||
if not cleaned:
|
||
return False
|
||
return all(c in "-—–\u2014\u2013 \t*" for c in cleaned)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# CSS class constants
|
||
# ---------------------------------------------------------------------------
|
||
|
||
BOOK_HEADER_CLASS = "p12"
|
||
CHAPTER_BOOKMARK_CLASSES = frozenset(["p13", "p151"])
|
||
SUBSECTION_BOOKMARK_CLASSES = frozenset(["p49", "p64"])
|
||
VERSE_PADA_CLASSES = frozenset(["p53"])
|
||
SKIP_CLASSES = frozenset(["p15", "p133"])
|
||
|
||
# Parent chapter heading class (Roman numeral, e.g. "I. Learning the Science of the Soul")
|
||
PARENT_HEADING_CLASS = "p27"
|
||
|
||
# Sub-chapter title class (duplicate of bookmark title)
|
||
SUBCHAPTER_TITLE_CLASS = "p36"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Clean chapter titles (pre-decoded IAST)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
CLEAN_PARENT_TITLES = {
|
||
1: "Learning the Science of the Soul",
|
||
2: "Choosing a Spiritual Master",
|
||
3: "Looking at the Cultural Background",
|
||
4: "Understanding Kṛṣṇa and Christ",
|
||
5: "Practicing Yoga in the Age of Quarrel",
|
||
6: "Finding the Cure for Today's Social Ills",
|
||
7: "Returning to the Eternal Religion",
|
||
8: "Reaching Life's Perfection",
|
||
}
|
||
|
||
# Intro segment definitions
|
||
INTRO_SEGMENT_DEFS = [
|
||
{
|
||
"bookmark_text": "SSR Dedication",
|
||
"slug": "dedication",
|
||
"title": "Dedication",
|
||
"order": 1,
|
||
},
|
||
{
|
||
"bookmark_text": "SSR Foreword",
|
||
"slug": "foreword",
|
||
"title": "Foreword",
|
||
"order": 2,
|
||
},
|
||
{
|
||
"bookmark_text": "SSR Introduction",
|
||
"slug": "introduction",
|
||
"title": "Introduction",
|
||
"order": 3,
|
||
},
|
||
{
|
||
"bookmark_text_pattern": r"SSR Appreciation",
|
||
"slug": "appreciations",
|
||
"title": "Appreciations of Śrīla Prabhupāda's Work",
|
||
"order": 4,
|
||
},
|
||
]
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Boundary detection
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def find_ssr_boundaries(all_paragraphs: list) -> tuple:
|
||
"""Find the start and end paragraph indices of the SSR content.
|
||
|
||
Start: <p class="p12"> containing "Science of Self-Realization" and "1977"
|
||
End: next <p class="p12"> after SSR content.
|
||
"""
|
||
start_idx = None
|
||
end_idx = len(all_paragraphs)
|
||
|
||
for i, p in enumerate(all_paragraphs):
|
||
cls = get_css_class(p)
|
||
text = p.get_text().strip()
|
||
|
||
if (
|
||
cls == BOOK_HEADER_CLASS
|
||
and "Science of Self-Realization" in text
|
||
and "1977" in text
|
||
):
|
||
start_idx = i
|
||
|
||
if start_idx is not None and i > start_idx and cls == BOOK_HEADER_CLASS:
|
||
end_idx = i
|
||
break
|
||
|
||
if start_idx is None:
|
||
print("ERROR: Could not find 'The Science of Self-Realization (1977)' header!")
|
||
sys.exit(1)
|
||
|
||
return start_idx, end_idx
|
||
|
||
|
||
def find_main_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
|
||
"""Find all SSR main bookmark tags (p13 and p151) in range.
|
||
|
||
Returns list of (paragraph_index, bookmark_text, css_class).
|
||
"""
|
||
bookmarks = []
|
||
for i in range(start_idx, end_idx):
|
||
p = all_paragraphs[i]
|
||
cls = get_css_class(p)
|
||
if cls not in CHAPTER_BOOKMARK_CLASSES:
|
||
continue
|
||
text = decode_text(p)
|
||
if text.startswith("SSR"):
|
||
bookmarks.append((i, text, cls))
|
||
return bookmarks
|
||
|
||
|
||
def find_sub_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
|
||
"""Find all SSR sub-chapter bookmark tags (p49 and p64) matching SSR Nx: pattern.
|
||
|
||
Returns list of (paragraph_index, parent_chapter_num, letter_code, title, raw_text, css_class).
|
||
"""
|
||
bookmarks = []
|
||
for i in range(start_idx, end_idx):
|
||
p = all_paragraphs[i]
|
||
cls = get_css_class(p)
|
||
if cls not in SUBSECTION_BOOKMARK_CLASSES:
|
||
continue
|
||
text = decode_text(p)
|
||
collapsed = re.sub(r"\s+", " ", text.strip())
|
||
m = re.match(r"SSR\s+(\d+)([a-z]+):\s*(.*)", collapsed)
|
||
if m:
|
||
parent_num = int(m.group(1))
|
||
letter = m.group(2)
|
||
title = m.group(3).strip()
|
||
title = re.sub(r"</?i>", "", title).strip()
|
||
bookmarks.append((i, parent_num, letter, title, collapsed, cls))
|
||
return bookmarks
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Paragraph extraction
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def extract_paragraphs(
|
||
all_paragraphs: list,
|
||
start_idx: int,
|
||
end_idx: int,
|
||
skip_bookmark_classes: bool = True,
|
||
skip_p27: bool = False,
|
||
skip_first_p36_matching: str = None,
|
||
) -> list:
|
||
"""Extract paragraphs for a section.
|
||
|
||
Handles all CSS classes within SSR content.
|
||
Groups consecutive p53 verse pādas into centered italic divs.
|
||
|
||
Args:
|
||
all_paragraphs: full list of <p> tags
|
||
start_idx: first paragraph index (inclusive)
|
||
end_idx: last paragraph index (exclusive)
|
||
skip_bookmark_classes: if True, skip p13/p151/p49/p64 bookmark tags
|
||
skip_p27: if True, skip the p27 parent chapter heading
|
||
skip_first_p36_matching: if set, skip the FIRST p36 whose text matches
|
||
this string (case-insensitive, normalized). Used to remove duplicate
|
||
sub-chapter title headings.
|
||
"""
|
||
result = []
|
||
verse_accumulator = []
|
||
first_p36_skipped = False
|
||
|
||
def flush_verse():
|
||
if verse_accumulator:
|
||
joined = "<br>".join(verse_accumulator)
|
||
result.append(f'<div style="text-align:center"><i>{joined}</i></div>')
|
||
verse_accumulator.clear()
|
||
|
||
for i in range(start_idx, end_idx):
|
||
p = all_paragraphs[i]
|
||
cls = get_css_class(p)
|
||
|
||
# Skip empty paragraphs
|
||
if is_empty(p):
|
||
continue
|
||
|
||
# Skip noise classes
|
||
if cls in SKIP_CLASSES:
|
||
continue
|
||
|
||
# Skip book title header (p12)
|
||
if cls == BOOK_HEADER_CLASS:
|
||
continue
|
||
|
||
# Skip bookmark classes (p13, p151, p49, p64) — already captured in metadata
|
||
if skip_bookmark_classes and cls in (
|
||
CHAPTER_BOOKMARK_CLASSES | SUBSECTION_BOOKMARK_CLASSES
|
||
):
|
||
continue
|
||
|
||
# Skip p27 parent chapter heading if requested
|
||
if skip_p27 and cls == PARENT_HEADING_CLASS:
|
||
continue
|
||
|
||
# Skip the first p36 that matches the sub-chapter title (duplicate heading)
|
||
if (
|
||
not first_p36_skipped
|
||
and skip_first_p36_matching is not None
|
||
and cls == SUBCHAPTER_TITLE_CLASS
|
||
):
|
||
p36_text = _normalize_for_match(decode_text(p))
|
||
target = _normalize_for_match(skip_first_p36_matching)
|
||
if p36_text == target:
|
||
first_p36_skipped = True
|
||
continue
|
||
|
||
# Check for dash lines — skip
|
||
text = decode_text(p)
|
||
if is_dash_line(text):
|
||
continue
|
||
|
||
# Sanskrit verse pādas (p53) — accumulate into centered block
|
||
if cls in VERSE_PADA_CLASSES:
|
||
html = decode_html_content(p)
|
||
if html:
|
||
stripped = re.sub(r"</?i>", "", html)
|
||
verse_accumulator.append(stripped)
|
||
continue
|
||
|
||
# Non-verse paragraph — flush any accumulated verse first
|
||
flush_verse()
|
||
|
||
# Extract HTML content
|
||
html = decode_html_content(p)
|
||
if html and not is_dash_line(text):
|
||
# Detect Sanskrit verses disguised as body paragraphs:
|
||
# they start with <i> and contain <br> (multi-line italic = verse)
|
||
stripped_html = html.strip()
|
||
if stripped_html.startswith("<i>") and "<br" in stripped_html:
|
||
inner = re.sub(r"</?i>", "", stripped_html)
|
||
result.append(f'<div style="text-align:center"><i>{inner}</i></div>')
|
||
else:
|
||
result.append(html)
|
||
|
||
# Flush any trailing verse
|
||
flush_verse()
|
||
|
||
return result
|
||
|
||
|
||
def _normalize_for_match(text: str) -> str:
|
||
"""Normalize text for fuzzy title matching: lowercase, collapse whitespace, strip quotes."""
|
||
text = re.sub(r"\s+", " ", text.strip().lower())
|
||
text = text.strip('""\u201c\u201d\'')
|
||
return text
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Main extraction
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def extract_ssr(soup: BeautifulSoup) -> dict:
|
||
"""Extract the entire Science of Self-Realization from parsed HTML.
|
||
|
||
Returns dict with:
|
||
"intro_segments": list of intro segment dicts
|
||
"sub_chapters": list of sub-chapter dicts
|
||
"parent_chapters": list of parent chapter info for intro.json
|
||
"""
|
||
all_paragraphs = list(soup.find_all("p"))
|
||
|
||
# --- Step 1: Find SSR boundaries ---
|
||
ssr_start, ssr_end = find_ssr_boundaries(all_paragraphs)
|
||
print(f" SSR content: paragraphs {ssr_start} to {ssr_end}")
|
||
print(f" Total paragraphs in range: {ssr_end - ssr_start}")
|
||
|
||
# --- Step 2: Find main bookmarks (p13/p151) ---
|
||
main_bookmarks = find_main_bookmarks(all_paragraphs, ssr_start, ssr_end)
|
||
print(f" Found {len(main_bookmarks)} main bookmarks (p13 + p151)")
|
||
for pos, bm_text, bm_cls in main_bookmarks:
|
||
print(f" [{bm_cls}] paragraph {pos}: {bm_text}")
|
||
|
||
# --- Step 3: Find all sub-chapter bookmarks (p49/p64) ---
|
||
sub_bookmarks = find_sub_bookmarks(all_paragraphs, ssr_start, ssr_end)
|
||
print(f" Found {len(sub_bookmarks)} sub-chapter bookmarks (p49 + p64)")
|
||
|
||
# --- Step 4: Classify main bookmarks ---
|
||
intro_bookmarks = [] # (idx, text, slug, title, order)
|
||
chapter_bookmarks = [] # (idx, text, ch_num, ch_title)
|
||
appendix_idx = ssr_end
|
||
|
||
for pos, bm_text, bm_cls in main_bookmarks:
|
||
collapsed = re.sub(r"\s+", " ", bm_text.strip())
|
||
m = re.match(r"SSR\s+(\d+):\s+(.+)", collapsed)
|
||
if m:
|
||
ch_num = int(m.group(1))
|
||
ch_title = m.group(2).strip()
|
||
chapter_bookmarks.append((pos, bm_text, ch_num, ch_title))
|
||
continue
|
||
|
||
if "Appendix" in bm_text:
|
||
appendix_idx = pos
|
||
continue
|
||
|
||
matched_intro = False
|
||
for seg_def in INTRO_SEGMENT_DEFS:
|
||
if (
|
||
"bookmark_text" in seg_def
|
||
and bm_text.strip() == seg_def["bookmark_text"]
|
||
):
|
||
intro_bookmarks.append(
|
||
(pos, bm_text, seg_def["slug"], seg_def["title"], seg_def["order"])
|
||
)
|
||
matched_intro = True
|
||
break
|
||
elif "bookmark_text_pattern" in seg_def and re.search(
|
||
seg_def["bookmark_text_pattern"], bm_text
|
||
):
|
||
intro_bookmarks.append(
|
||
(pos, bm_text, seg_def["slug"], seg_def["title"], seg_def["order"])
|
||
)
|
||
matched_intro = True
|
||
break
|
||
|
||
if not matched_intro:
|
||
print(f" ⚠ Unclassified main bookmark: {bm_text}")
|
||
|
||
print(f"\n Intro bookmarks: {len(intro_bookmarks)}")
|
||
for pos, bm_text, slug, title, order in intro_bookmarks:
|
||
print(f" [{order}] {slug}: {title} (paragraph {pos})")
|
||
|
||
print(f" Chapter bookmarks: {len(chapter_bookmarks)}")
|
||
for pos, bm_text, ch_num, ch_title in chapter_bookmarks:
|
||
print(f" Ch {ch_num}: {ch_title} (paragraph {pos})")
|
||
|
||
print(f" Appendix boundary: paragraph {appendix_idx}")
|
||
|
||
# --- Step 5: Build sorted boundary list for intro/appendix delimiting ---
|
||
boundaries = []
|
||
|
||
for pos, bm_text, slug, title, order in intro_bookmarks:
|
||
boundaries.append(
|
||
(pos, "intro", {"slug": slug, "title": title, "order": order})
|
||
)
|
||
|
||
for pos, bm_text, ch_num, ch_title in chapter_bookmarks:
|
||
boundaries.append((pos, "chapter", {"ch_num": ch_num, "ch_title": ch_title}))
|
||
|
||
boundaries.append((appendix_idx, "appendix", {}))
|
||
boundaries.sort(key=lambda x: x[0])
|
||
|
||
# --- Step 6: Extract front pages ---
|
||
intro_segments = []
|
||
|
||
first_boundary_idx = boundaries[0][0] if boundaries else ssr_end
|
||
|
||
front_pages_paragraphs = extract_paragraphs(
|
||
all_paragraphs, ssr_start, first_boundary_idx
|
||
)
|
||
intro_segments.append(
|
||
{
|
||
"slug": "front-pages",
|
||
"title": "Front Pages",
|
||
"type": "intro-segment",
|
||
"book": "ssr",
|
||
"order": 0,
|
||
"paragraphs": front_pages_paragraphs,
|
||
}
|
||
)
|
||
|
||
# --- Step 7: Extract intro segments (Dedication, Foreword, Introduction, Appreciations) ---
|
||
for bi in range(len(boundaries)):
|
||
b_pos, b_type, b_data = boundaries[bi]
|
||
|
||
content_start = b_pos + 1 # skip the bookmark tag itself
|
||
if bi + 1 < len(boundaries):
|
||
content_end = boundaries[bi + 1][0]
|
||
else:
|
||
content_end = ssr_end
|
||
|
||
if b_type == "intro":
|
||
paragraphs = extract_paragraphs(all_paragraphs, content_start, content_end)
|
||
intro_segments.append(
|
||
{
|
||
"slug": b_data["slug"],
|
||
"title": b_data["title"],
|
||
"type": "intro-segment",
|
||
"book": "ssr",
|
||
"order": b_data["order"],
|
||
"paragraphs": paragraphs,
|
||
}
|
||
)
|
||
|
||
elif b_type == "appendix":
|
||
# About the Author segment — from appendix to end of SSR
|
||
paragraphs = extract_paragraphs(all_paragraphs, content_start, ssr_end)
|
||
intro_segments.append(
|
||
{
|
||
"slug": "about-the-author",
|
||
"title": "About the Author",
|
||
"type": "intro-segment",
|
||
"book": "ssr",
|
||
"order": 99,
|
||
"paragraphs": paragraphs,
|
||
}
|
||
)
|
||
|
||
# --- Step 8: Build parent chapter → sub-chapter structure ---
|
||
|
||
# Group sub-bookmarks by parent chapter
|
||
from collections import OrderedDict, defaultdict
|
||
|
||
subs_by_parent = defaultdict(list)
|
||
for pos, parent_num, letter, title, raw, cls in sub_bookmarks:
|
||
subs_by_parent[parent_num].append(
|
||
{
|
||
"pos": pos,
|
||
"parent_num": parent_num,
|
||
"letter": letter,
|
||
"title": title,
|
||
"raw": raw,
|
||
"cls": cls,
|
||
}
|
||
)
|
||
|
||
# Build chapter_bookmark lookup: ch_num → (pos, title)
|
||
ch_bm_lookup = {}
|
||
for pos, bm_text, ch_num, ch_title in chapter_bookmarks:
|
||
ch_bm_lookup[ch_num] = (pos, ch_title)
|
||
|
||
# Determine end boundary for each parent chapter
|
||
ch_nums_sorted = sorted(ch_bm_lookup.keys())
|
||
ch_end_lookup = {}
|
||
for idx, ch_num in enumerate(ch_nums_sorted):
|
||
if idx + 1 < len(ch_nums_sorted):
|
||
next_ch = ch_nums_sorted[idx + 1]
|
||
ch_end_lookup[ch_num] = ch_bm_lookup[next_ch][0]
|
||
else:
|
||
ch_end_lookup[ch_num] = appendix_idx
|
||
|
||
# --- Step 9: Build sub-chapter list with sequential numbering ---
|
||
sub_chapters = []
|
||
parent_chapters = []
|
||
|
||
for ch_num in ch_nums_sorted:
|
||
parent_pos, parent_raw_title = ch_bm_lookup[ch_num]
|
||
parent_title = CLEAN_PARENT_TITLES.get(
|
||
ch_num, decode_scagoudy(parent_raw_title)
|
||
)
|
||
parent_end = ch_end_lookup[ch_num]
|
||
|
||
subs = subs_by_parent.get(ch_num, [])
|
||
# Sort sub-bookmarks by position
|
||
subs.sort(key=lambda s: s["pos"])
|
||
|
||
sub_entries = [] # for intro.json
|
||
sub_counter = 0
|
||
|
||
# Check if there's content between parent bookmark and first sub-bookmark
|
||
# that needs to be treated as the first sub-chapter (no "a" letter)
|
||
first_sub_pos = subs[0]["pos"] if subs else parent_end
|
||
# Content starts after parent bookmark + p27 heading
|
||
parent_content_start = parent_pos + 1
|
||
|
||
# Determine if there's a gap with real content between parent and first sub
|
||
has_implicit_first = False
|
||
if subs:
|
||
first_letter = subs[0]["letter"]
|
||
# If first sub-chapter letter is NOT "a", there's content before it
|
||
if first_letter != "a":
|
||
has_implicit_first = True
|
||
else:
|
||
# Even if first is "a", check whether parent bookmark and sub "a"
|
||
# have substantial content between them (beyond p27 heading)
|
||
# In practice: parent at N, p27 at N+1, p49/p64 at N+2 = no gap
|
||
gap = subs[0]["pos"] - parent_pos
|
||
# gap of 2 means just p13 + p27, no content
|
||
has_implicit_first = False
|
||
|
||
if has_implicit_first:
|
||
# Create an implicit first sub-chapter from parent content
|
||
sub_counter += 1
|
||
slug = f"{ch_num}-{sub_counter}"
|
||
|
||
# Title: derive from the first meaningful heading in the gap.
|
||
# Look for a p64 bookmark (which is the "a" bookmark on a different class)
|
||
# or a p36 heading, or fall back to parent title.
|
||
implicit_title = None
|
||
|
||
# Check for a p64 sub-bookmark that's actually the "a" entry
|
||
# (already captured in subs_by_parent if it matched the SSR Nx: pattern)
|
||
# Actually, p64 entries ARE in our sub_bookmarks list already.
|
||
# But the "has_implicit_first" case means first_letter != "a" —
|
||
# i.e., no "a" entry at all. So look for content headings.
|
||
|
||
# Scan for the first p36 or meaningful text in the gap
|
||
for gi in range(parent_content_start, first_sub_pos):
|
||
gp = all_paragraphs[gi]
|
||
gcls = get_css_class(gp)
|
||
if gcls == PARENT_HEADING_CLASS:
|
||
continue # skip Roman numeral heading
|
||
if gcls == SUBCHAPTER_TITLE_CLASS:
|
||
implicit_title = decode_scagoudy(
|
||
re.sub(r"\s+", " ", gp.get_text().strip())
|
||
)
|
||
break
|
||
if gcls in CHAPTER_BOOKMARK_CLASSES | SUBSECTION_BOOKMARK_CLASSES:
|
||
continue
|
||
if not is_empty(gp):
|
||
# Use the parent title as fallback
|
||
break
|
||
|
||
if not implicit_title:
|
||
implicit_title = parent_title
|
||
|
||
paragraphs = extract_paragraphs(
|
||
all_paragraphs,
|
||
parent_content_start,
|
||
first_sub_pos,
|
||
skip_p27=True,
|
||
skip_first_p36_matching=implicit_title,
|
||
)
|
||
|
||
sub_chapters.append(
|
||
{
|
||
"book": "ssr",
|
||
"chapter": slug,
|
||
"parentChapter": ch_num,
|
||
"subChapter": sub_counter,
|
||
"title": implicit_title,
|
||
"parentTitle": parent_title,
|
||
"type": "chapter",
|
||
"paragraphs": paragraphs,
|
||
}
|
||
)
|
||
sub_entries.append({"slug": slug, "title": implicit_title})
|
||
|
||
# Now process each explicit sub-bookmark
|
||
for si, sub in enumerate(subs):
|
||
sub_counter += 1
|
||
slug = f"{ch_num}-{sub_counter}"
|
||
|
||
sub_title = decode_scagoudy(sub["title"])
|
||
|
||
# Content range: from bookmark pos to next sub-bookmark (or parent end)
|
||
content_start = sub[
|
||
"pos"
|
||
] # we skip the bookmark tag itself inside extract_paragraphs
|
||
if si + 1 < len(subs):
|
||
content_end = subs[si + 1]["pos"]
|
||
else:
|
||
content_end = parent_end
|
||
|
||
# For the first explicit sub in a chapter that starts at "a",
|
||
# content_start is the bookmark, but we also need to skip the p27
|
||
# heading that precedes it if this is the very first sub.
|
||
is_first_explicit = si == 0 and not has_implicit_first
|
||
skip_p27_flag = is_first_explicit
|
||
|
||
paragraphs = extract_paragraphs(
|
||
all_paragraphs,
|
||
content_start,
|
||
content_end,
|
||
skip_p27=skip_p27_flag,
|
||
skip_first_p36_matching=sub_title,
|
||
)
|
||
|
||
sub_chapters.append(
|
||
{
|
||
"book": "ssr",
|
||
"chapter": slug,
|
||
"parentChapter": ch_num,
|
||
"subChapter": sub_counter,
|
||
"title": sub_title,
|
||
"parentTitle": parent_title,
|
||
"type": "chapter",
|
||
"paragraphs": paragraphs,
|
||
}
|
||
)
|
||
sub_entries.append({"slug": slug, "title": sub_title})
|
||
|
||
parent_chapters.append(
|
||
{
|
||
"number": ch_num,
|
||
"title": parent_title,
|
||
"subChapters": sub_entries,
|
||
}
|
||
)
|
||
|
||
print(
|
||
f"\n Parent Ch {ch_num}: {parent_title} → {len(sub_entries)} sub-chapters"
|
||
)
|
||
for entry in sub_entries:
|
||
# Find matching sub_chapter for paragraph count
|
||
matching = [sc for sc in sub_chapters if sc["chapter"] == entry["slug"]]
|
||
para_count = len(matching[0]["paragraphs"]) if matching else 0
|
||
print(
|
||
f" {entry['slug']:>5}: {para_count:>3} paragraphs — {entry['title']}"
|
||
)
|
||
|
||
return {
|
||
"intro_segments": intro_segments,
|
||
"sub_chapters": sub_chapters,
|
||
"parent_chapters": parent_chapters,
|
||
}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Output helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def write_json(data, filepath):
|
||
"""Write data as formatted JSON to filepath, creating directories as needed."""
|
||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||
with open(filepath, "w", encoding="utf-8") as f:
|
||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Main
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def main():
|
||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
html_path = os.path.join(base_dir, "public", "html", "other.html")
|
||
output_base = os.path.join(base_dir, "lib", "jsons", "books", "ssr")
|
||
|
||
if not os.path.exists(html_path):
|
||
print(f"ERROR: {html_path} not found!")
|
||
sys.exit(1)
|
||
|
||
print(f"Reading {html_path}...")
|
||
with open(html_path, "r", encoding="utf-8") as f:
|
||
soup = BeautifulSoup(f, "html.parser")
|
||
|
||
print("Extracting The Science of Self-Realization (1977)...\n")
|
||
result = extract_ssr(soup)
|
||
|
||
intro_segments = result["intro_segments"]
|
||
sub_chapters = result["sub_chapters"]
|
||
parent_chapters = result["parent_chapters"]
|
||
|
||
# --- Write intro.json ---
|
||
intro_meta = {
|
||
"book": "ssr",
|
||
"title": "The Science of Self-Realization",
|
||
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
||
"year": 1977,
|
||
"description": "Articles and conversations on the science of self-realization, collected from magazines, lectures, and interviews.",
|
||
"totalChapters": 8,
|
||
"chapters": parent_chapters,
|
||
}
|
||
intro_path = os.path.join(output_base, "intro.json")
|
||
write_json(intro_meta, intro_path)
|
||
print(f"\n Written: {intro_path}")
|
||
|
||
# --- Write intro segments ---
|
||
print(f"\n--- Intro Segments ---")
|
||
for seg in intro_segments:
|
||
filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
|
||
write_json(seg, filepath)
|
||
print(
|
||
f" {seg['order']:>2}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}"
|
||
)
|
||
|
||
# --- Write sub-chapter files ---
|
||
print(f"\n--- Sub-Chapters ---")
|
||
for sc in sub_chapters:
|
||
slug = sc["chapter"]
|
||
filepath = os.path.join(output_base, "chapters", f"{slug}.json")
|
||
write_json(sc, filepath)
|
||
print(
|
||
f" {slug:>5}: {len(sc['paragraphs']):>3} paragraphs — "
|
||
f"[Ch {sc['parentChapter']}] {sc['title']}"
|
||
)
|
||
|
||
# --- Summary ---
|
||
total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments)
|
||
total_chapter_paras = sum(len(sc["paragraphs"]) for sc in sub_chapters)
|
||
total_paras = total_intro_paras + total_chapter_paras
|
||
|
||
print(f"\n{'=' * 60}")
|
||
print(f" EXTRACTION SUMMARY")
|
||
print(f"{'=' * 60}")
|
||
print(f" Intro segments: {len(intro_segments)}")
|
||
print(f" Parent chapters: {len(parent_chapters)}")
|
||
print(f" Sub-chapters: {len(sub_chapters)}")
|
||
print(f" Total paragraphs: {total_paras}")
|
||
print(f" Intro: {total_intro_paras}")
|
||
print(f" Chapters: {total_chapter_paras}")
|
||
if sub_chapters:
|
||
print(f" Avg paras/sub-ch: {total_chapter_paras / len(sub_chapters):.1f}")
|
||
print(f" Output directory: {output_base}/")
|
||
|
||
# --- Per-parent breakdown ---
|
||
print(f"\n--- Parent Chapter Breakdown ---")
|
||
for pc in parent_chapters:
|
||
ch_num = pc["number"]
|
||
sub_count = len(pc["subChapters"])
|
||
ch_paras = sum(
|
||
len(sc["paragraphs"])
|
||
for sc in sub_chapters
|
||
if sc["parentChapter"] == ch_num
|
||
)
|
||
print(
|
||
f" Ch {ch_num}: {sub_count:>2} sub-chapters, {ch_paras:>4} paragraphs — {pc['title']}"
|
||
)
|
||
|
||
# --- Samples ---
|
||
print(f"\n--- Sample: Front Pages ---")
|
||
if intro_segments:
|
||
front = intro_segments[0]
|
||
for i, p in enumerate(front["paragraphs"][:5]):
|
||
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
||
print(f" [{i}] {preview}...")
|
||
|
||
if len(intro_segments) > 1:
|
||
print(f"\n--- Sample: Dedication ---")
|
||
ded = intro_segments[1]
|
||
for i, p in enumerate(ded["paragraphs"][:5]):
|
||
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
||
print(f" [{i}] {preview}...")
|
||
|
||
if sub_chapters:
|
||
first_sc = sub_chapters[0]
|
||
print(f"\n--- Sample: {first_sc['chapter']} ({first_sc['title']}) ---")
|
||
for i, p in enumerate(first_sc["paragraphs"][:5]):
|
||
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
||
print(f" [{i}] {preview}...")
|
||
|
||
if len(sub_chapters) > 1:
|
||
last_sc = sub_chapters[-1]
|
||
print(f"\n--- Sample: {last_sc['chapter']} ({last_sc['title']}) ---")
|
||
for i, p in enumerate(last_sc["paragraphs"][:5]):
|
||
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
||
print(f" [{i}] {preview}...")
|
||
|
||
# --- Quality Checks ---
|
||
print(f"\n--- Quality Checks ---")
|
||
|
||
# Check all 8 parent chapters present
|
||
expected_parents = set(range(1, 9))
|
||
actual_parents = {pc["number"] for pc in parent_chapters}
|
||
missing = expected_parents - actual_parents
|
||
extra = actual_parents - expected_parents
|
||
if missing:
|
||
print(f" ⚠ Missing parent chapters: {sorted(missing)}")
|
||
if extra:
|
||
print(f" ⚠ Extra parent chapters: {sorted(extra)}")
|
||
if not missing and not extra:
|
||
print(f" ✅ All 8 parent chapters present")
|
||
|
||
# Check all 6 intro segments present
|
||
expected_intros = {
|
||
"front-pages",
|
||
"dedication",
|
||
"foreword",
|
||
"introduction",
|
||
"appreciations",
|
||
"about-the-author",
|
||
}
|
||
actual_intros = {s["slug"] for s in intro_segments}
|
||
missing_intros = expected_intros - actual_intros
|
||
extra_intros = actual_intros - expected_intros
|
||
if missing_intros:
|
||
print(f" ⚠ Missing intro segments: {missing_intros}")
|
||
if extra_intros:
|
||
print(f" ⚠ Extra intro segments: {extra_intros}")
|
||
if not missing_intros and not extra_intros:
|
||
print(f" ✅ All 6 intro segments present")
|
||
|
||
# Check empty sub-chapters
|
||
empty_scs = [sc["chapter"] for sc in sub_chapters if len(sc["paragraphs"]) == 0]
|
||
if empty_scs:
|
||
print(f" ⚠ Empty sub-chapters (0 paragraphs): {empty_scs}")
|
||
else:
|
||
print(f" ✅ No empty sub-chapters")
|
||
|
||
# Check empty intro segments
|
||
empty_intros = [s["slug"] for s in intro_segments if len(s["paragraphs"]) == 0]
|
||
if empty_intros:
|
||
print(f" ⚠ Empty intro segments: {empty_intros}")
|
||
else:
|
||
print(f" ✅ No empty intro segments")
|
||
|
||
# Check for duplicate headings in first 3 paragraphs of each sub-chapter
|
||
print(f"\n--- Duplicate Heading Check ---")
|
||
dup_count = 0
|
||
for sc in sub_chapters:
|
||
title_norm = _normalize_for_match(sc["title"])
|
||
for p in sc["paragraphs"][:3]:
|
||
plain = re.sub(r"<[^>]+>", "", p).strip()
|
||
plain_norm = _normalize_for_match(plain)
|
||
if plain_norm == title_norm and len(plain) < 150:
|
||
print(f" ⚠ Duplicate heading in {sc['chapter']}: '{plain[:60]}'")
|
||
dup_count += 1
|
||
break
|
||
if dup_count == 0:
|
||
print(f" ✅ No duplicate headings in first 3 paragraphs")
|
||
else:
|
||
print(f" ⚠ {dup_count} sub-chapters have duplicate headings")
|
||
|
||
# Check for Roman numeral parent headings leaking into content
|
||
print(f"\n--- Parent Heading Leak Check ---")
|
||
roman_pattern = re.compile(r"^(I{1,3}|IV|V|VI{0,3}|VIII?)\.\s", re.IGNORECASE)
|
||
leak_count = 0
|
||
for sc in sub_chapters:
|
||
for p in sc["paragraphs"][:2]:
|
||
plain = re.sub(r"<[^>]+>", "", p).strip()
|
||
if roman_pattern.match(plain):
|
||
print(f" ⚠ Roman numeral heading in {sc['chapter']}: '{plain[:60]}'")
|
||
leak_count += 1
|
||
break
|
||
if leak_count == 0:
|
||
print(f" ✅ No Roman numeral parent headings in sub-chapter content")
|
||
|
||
# Check ScaGoudy decoding
|
||
print(f"\n--- ScaGoudy Decoding Check ---")
|
||
all_text = ""
|
||
for sc in sub_chapters:
|
||
all_text += " ".join(sc["paragraphs"][:20])
|
||
for seg in intro_segments:
|
||
all_text += " ".join(seg["paragraphs"][:10])
|
||
|
||
if "Kṛṣṇa" in all_text:
|
||
print(f" ✅ 'Kṛṣṇa' found (ScaGoudy properly decoded)")
|
||
else:
|
||
print(f" ⚠ 'Kṛṣṇa' not found — ScaGoudy decoding may have failed")
|
||
|
||
if "Bhagavad-gītā" in all_text or "gītā" in all_text or "Gītā" in all_text:
|
||
print(f" ✅ 'gītā' found (ScaGoudy properly decoded)")
|
||
else:
|
||
print(f" ⚠ 'Gītā' not found — ScaGoudy decoding may have failed")
|
||
|
||
if "Prabhupāda" in all_text:
|
||
print(f" ✅ 'Prabhupāda' found (ScaGoudy properly decoded)")
|
||
|
||
# Check for un-decoded ScaGoudy remnants
|
||
scagoudy_remnants = 0
|
||
for sc in sub_chapters:
|
||
for p in sc["paragraphs"]:
|
||
if "Kåñëa" in p or "Prabhupäda" in p:
|
||
scagoudy_remnants += 1
|
||
for seg in intro_segments:
|
||
for p in seg["paragraphs"]:
|
||
if "Kåñëa" in p or "Prabhupäda" in p:
|
||
scagoudy_remnants += 1
|
||
if scagoudy_remnants == 0:
|
||
print(f" ✅ No ScaGoudy remnants detected")
|
||
else:
|
||
print(f" ⚠ {scagoudy_remnants} paragraphs with ScaGoudy remnants")
|
||
|
||
# Check for centered verse blocks
|
||
verse_block_count = 0
|
||
for sc in sub_chapters:
|
||
for p in sc["paragraphs"]:
|
||
if '<div style="text-align:center"><i>' in p:
|
||
verse_block_count += 1
|
||
if verse_block_count > 0:
|
||
print(f" ✅ {verse_block_count} centered Sanskrit verse blocks found")
|
||
else:
|
||
print(f" ⚠ No centered verse blocks found")
|
||
|
||
# Check for residual span tags
|
||
span_count = 0
|
||
for sc in sub_chapters:
|
||
for p in sc["paragraphs"]:
|
||
if 'class="' in p and "<span" in p:
|
||
span_count += 1
|
||
for seg in intro_segments:
|
||
for p in seg["paragraphs"]:
|
||
if 'class="' in p and "<span" in p:
|
||
span_count += 1
|
||
if span_count == 0:
|
||
print(f" ✅ No residual span class artifacts found")
|
||
else:
|
||
print(f" ⚠ {span_count} paragraphs still contain span class artifacts")
|
||
|
||
# Check for dash-line paragraphs
|
||
dash_count = 0
|
||
for sc in sub_chapters:
|
||
for p in sc["paragraphs"]:
|
||
plain = re.sub(r"<[^>]+>", "", p)
|
||
if is_dash_line(plain):
|
||
dash_count += 1
|
||
for seg in intro_segments:
|
||
for p in seg["paragraphs"]:
|
||
plain = re.sub(r"<[^>]+>", "", p)
|
||
if is_dash_line(plain):
|
||
dash_count += 1
|
||
if dash_count == 0:
|
||
print(f" ✅ No dashed-line paragraphs in output")
|
||
else:
|
||
print(f" ⚠ {dash_count} dashed-line paragraphs found in output")
|
||
|
||
# Show a verse block sample
|
||
print(f"\n--- Sample: Verse Block ---")
|
||
for sc in sub_chapters:
|
||
for p in sc["paragraphs"]:
|
||
if '<div style="text-align:center"><i>' in p:
|
||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
||
print(f" {preview}...")
|
||
break
|
||
else:
|
||
continue
|
||
break
|
||
|
||
# Sub-chapter size check
|
||
print(f"\n--- Sub-Chapter Size Check ---")
|
||
very_small = []
|
||
very_large = []
|
||
for sc in sub_chapters:
|
||
count = len(sc["paragraphs"])
|
||
if count < 2:
|
||
very_small.append((sc["chapter"], count, sc["title"]))
|
||
elif count > 200:
|
||
very_large.append((sc["chapter"], count, sc["title"]))
|
||
if very_small:
|
||
print(f" ⚠ Very small sub-chapters (< 2 paragraphs):")
|
||
for slug, count, title in very_small:
|
||
print(f" {slug}: {count} paragraphs — {title}")
|
||
else:
|
||
print(f" ✅ All sub-chapters have ≥ 2 paragraphs")
|
||
if very_large:
|
||
print(f" ⚠ Very large sub-chapters (> 200 paragraphs):")
|
||
for slug, count, title in very_large:
|
||
print(f" {slug}: {count} paragraphs — {title}")
|
||
else:
|
||
print(f" ✅ All sub-chapters have ≤ 200 paragraphs")
|
||
|
||
# Sub-chapter title decoding check
|
||
print(f"\n--- Sub-Chapter Title Decoding Check ---")
|
||
bad_titles = 0
|
||
for sc in sub_chapters:
|
||
title = sc["title"]
|
||
if "å" in title or "ë" in title or "ñ" in title or "ä" in title:
|
||
print(f" ⚠ {sc['chapter']} title has un-decoded ScaGoudy: {title}")
|
||
bad_titles += 1
|
||
if bad_titles == 0:
|
||
print(f" ✅ All {len(sub_chapters)} sub-chapter titles properly decoded")
|
||
|
||
# Verify slug numbering continuity within each parent
|
||
print(f"\n--- Slug Numbering Check ---")
|
||
for pc in parent_chapters:
|
||
ch_num = pc["number"]
|
||
slugs = [s["slug"] for s in pc["subChapters"]]
|
||
expected_slugs = [f"{ch_num}-{i + 1}" for i in range(len(slugs))]
|
||
if slugs == expected_slugs:
|
||
print(
|
||
f" ✅ Ch {ch_num}: {len(slugs)} sub-chapters, slugs {slugs[0]}..{slugs[-1]}"
|
||
)
|
||
else:
|
||
print(
|
||
f" ⚠ Ch {ch_num}: slug mismatch! expected {expected_slugs}, got {slugs}"
|
||
)
|
||
|
||
print(f"\nDone! ✨")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|