feat: sva extracted and seeded
This commit is contained in:
gosangam
@@ -0,0 +1,904 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract Life Comes from Life (1979) from other.html.
|
||||
|
||||
The book has 3 intro segments (foreword, introduction, about-the-author),
|
||||
and 16 chapters ("The First Morning Walk" through "The Sixteenth Morning Walk").
|
||||
|
||||
Output:
|
||||
lib/jsons/books/lcl/intro.json — Book metadata
|
||||
lib/jsons/books/lcl/intro/*.json — Intro segments
|
||||
lib/jsons/books/lcl/chapters/*.json — Chapters (01–16)
|
||||
|
||||
Usage:
|
||||
python3 scripts/lcl_extract.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ScaGoudy → IAST Unicode mapping
|
||||
# ---------------------------------------------------------------------------
|
||||
SCAGOUDY_TO_IAST = {
|
||||
"ä": "ā",
|
||||
"Ä": "Ā",
|
||||
"å": "ṛ",
|
||||
"Å": "Ṛ",
|
||||
"ñ": "ṣ",
|
||||
"Ñ": "Ṣ",
|
||||
"ö": "ṭ",
|
||||
"Ö": "Ṭ",
|
||||
"ë": "ṇ",
|
||||
"Ë": "Ṇ",
|
||||
"ò": "ḍ",
|
||||
"Ò": "Ḍ",
|
||||
"ç": "ś",
|
||||
"Ç": "Ś",
|
||||
"ì": "ṅ",
|
||||
"ï": "ñ",
|
||||
"ü": "ū",
|
||||
"Ü": "Ū",
|
||||
"é": "ī",
|
||||
"É": "Ī",
|
||||
"à": "ṁ",
|
||||
"ù": "ḥ",
|
||||
"è": "ṝ",
|
||||
"û": "ṁl",
|
||||
"ÿ": "ḷ",
|
||||
}
|
||||
|
||||
|
||||
def decode_scagoudy(text: str) -> str:
|
||||
"""Replace ScaGoudy-encoded characters with proper IAST Unicode."""
|
||||
for src, dst in SCAGOUDY_TO_IAST.items():
|
||||
text = text.replace(src, dst)
|
||||
return text
|
||||
|
||||
|
||||
def decode_html_content(tag: Tag) -> str:
|
||||
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
|
||||
inner = "".join(str(child) for child in tag.children)
|
||||
inner = decode_scagoudy(inner)
|
||||
# Remove span tags with CSS classes (RTF conversion artifacts)
|
||||
inner = re.sub(
|
||||
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
|
||||
)
|
||||
# Clean up Apple-tab-span leftovers (tab characters)
|
||||
inner = inner.replace("\t", " ")
|
||||
# Normalize whitespace
|
||||
inner = re.sub(r"\s+", " ", inner).strip()
|
||||
return inner
|
||||
|
||||
|
||||
def decode_text(tag: Tag) -> str:
|
||||
"""Get plain text from a tag with ScaGoudy decoding."""
|
||||
return decode_scagoudy(tag.get_text().strip())
|
||||
|
||||
|
||||
def get_css_class(tag: Tag) -> str:
|
||||
"""Get the first CSS class of a tag, or empty string."""
|
||||
classes = tag.get("class", [])
|
||||
return classes[0] if classes else ""
|
||||
|
||||
|
||||
def is_empty(tag: Tag) -> bool:
|
||||
"""Check if a tag is empty or whitespace-only."""
|
||||
text = tag.get_text().strip()
|
||||
return not text or text == "\xa0"
|
||||
|
||||
|
||||
def is_dash_line(text: str) -> bool:
|
||||
"""Return True if text is only dashes, spaces, and/or whitespace."""
|
||||
cleaned = text.strip()
|
||||
if not cleaned:
|
||||
return False
|
||||
return all(c in "-—–\u2014\u2013 \t" for c in cleaned)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CSS class categorization for LCL
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Book title header class (bold, e.g. "LIFE COMES FROM LIFE (1979)")
|
||||
BOOK_HEADER_CLASS = "p166"
|
||||
|
||||
# Subtitle class
|
||||
SUBTITLE_CLASS = "p167"
|
||||
|
||||
# Bookmark classes — used for section boundary detection
|
||||
BOOKMARK_CLASS = "p13"
|
||||
BOOKMARK_CLASS_168 = "p168" # used for Foreword / Introduction bookmarks
|
||||
|
||||
# Body paragraph classes (primary content)
|
||||
BODY_CLASSES = frozenset(["p170", "p171", "p6", "p68", "p182"])
|
||||
|
||||
# Dialogue line classes — the main content classes (speakers in bold)
|
||||
DIALOGUE_CLASSES = frozenset(["p114", "p118", "p178", "p181"])
|
||||
|
||||
# Sanskrit verse pāda classes (italic, centered)
|
||||
VERSE_PADA_CLASSES = frozenset(["p5", "p180"])
|
||||
|
||||
# Verse translation classes
|
||||
TRANSLATION_CLASSES = frozenset(["p29", "p94", "p184", "p186"])
|
||||
|
||||
# Chapter heading (bold)
|
||||
CHAPTER_HEADING_CLASS = "p27"
|
||||
|
||||
# Section subheading classes (topic markers within chapters)
|
||||
SECTION_HEADING_CLASSES = frozenset(["p36", "p64"])
|
||||
|
||||
# Section subheading bookmark (contains topic title text)
|
||||
SECTION_BOOKMARK_CLASS = "p49"
|
||||
|
||||
# Foreword heading (bold)
|
||||
FOREWORD_HEADING_CLASS = "p169"
|
||||
|
||||
# Title page / founder-ācārya line
|
||||
TITLE_PAGE_CLASS = "p3"
|
||||
|
||||
# Companion description (italic, who accompanies Prabhupāda)
|
||||
COMPANION_CLASS = "p177"
|
||||
|
||||
# Block quote classes
|
||||
BLOCKQUOTE_CLASSES = frozenset(["p173", "p175"])
|
||||
|
||||
# Attribution classes
|
||||
ATTRIBUTION_CLASSES = frozenset(["p174", "p176", "p42"])
|
||||
|
||||
# Reference / citation line
|
||||
REFERENCE_CLASS = "p187"
|
||||
|
||||
# Classes to skip entirely (empty/decorative/break)
|
||||
SKIP_CLASSES = frozenset(["p15", "p172", "p179", "p183", "p135"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Ordinal mapping for chapter names
|
||||
# ---------------------------------------------------------------------------
|
||||
ORDINAL_MAP = {
|
||||
"First": 1,
|
||||
"Second": 2,
|
||||
"Third": 3,
|
||||
"Fourth": 4,
|
||||
"Fifth": 5,
|
||||
"Sixth": 6,
|
||||
"Seventh": 7,
|
||||
"Eighth": 8,
|
||||
"Ninth": 9,
|
||||
"Tenth": 10,
|
||||
"Eleventh": 11,
|
||||
"Twelfth": 12,
|
||||
"Thirteenth": 13,
|
||||
"Fourteenth": 14,
|
||||
"Fifteenth": 15,
|
||||
"Sixteenth": 16,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Extraction helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def find_lcl_boundaries(all_paragraphs: list) -> tuple:
|
||||
"""Find the start and end paragraph indices of the LCL content.
|
||||
|
||||
Start: <p class="p166"> containing "LIFE COMES FROM LIFE"
|
||||
End: <p class="p13"> containing "LCL: The Challenge"
|
||||
"""
|
||||
start_idx = None
|
||||
end_idx = len(all_paragraphs)
|
||||
|
||||
for i, p in enumerate(all_paragraphs):
|
||||
cls = get_css_class(p)
|
||||
text = decode_text(p)
|
||||
|
||||
# Find start
|
||||
if start_idx is None:
|
||||
if cls == BOOK_HEADER_CLASS and "LIFE COMES FROM LIFE" in text.upper():
|
||||
start_idx = i
|
||||
continue
|
||||
|
||||
# Find end: the "LCL: The Challenge" bookmark
|
||||
if cls == BOOKMARK_CLASS:
|
||||
if "LCL:" in text and "The Challenge" in text:
|
||||
end_idx = i
|
||||
break
|
||||
|
||||
if start_idx is None:
|
||||
print("ERROR: Could not find 'LIFE COMES FROM LIFE' header in other.html!")
|
||||
sys.exit(1)
|
||||
|
||||
return start_idx, end_idx
|
||||
|
||||
|
||||
def find_all_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
|
||||
"""Find all LCL bookmark tags in the LCL range.
|
||||
|
||||
Bookmarks can be either p13 or p168 class with text containing "LCL:".
|
||||
Returns list of (paragraph_index, bookmark_text).
|
||||
"""
|
||||
bookmarks = []
|
||||
for i in range(start_idx, end_idx):
|
||||
p = all_paragraphs[i]
|
||||
cls = get_css_class(p)
|
||||
if cls not in (BOOKMARK_CLASS, BOOKMARK_CLASS_168):
|
||||
continue
|
||||
# Get text, stripping span noise
|
||||
text = decode_text(p).strip()
|
||||
if text.startswith("LCL:"):
|
||||
bookmarks.append((i, text))
|
||||
return bookmarks
|
||||
|
||||
|
||||
def parse_chapter_bookmark(text: str):
|
||||
"""Parse a chapter bookmark like 'LCL: The First Morning Walk'.
|
||||
|
||||
Returns (chapter_number, title) or None if not a chapter bookmark.
|
||||
"""
|
||||
m = re.match(r"LCL:\s+The\s+(\w+)\s+Morning\s+Walk", text.strip())
|
||||
if m:
|
||||
ordinal = m.group(1)
|
||||
num = ORDINAL_MAP.get(ordinal)
|
||||
if num is not None:
|
||||
# Reconstruct the title from the matched text
|
||||
title = f"The {ordinal} Morning Walk"
|
||||
return num, title
|
||||
return None
|
||||
|
||||
|
||||
def parse_intro_bookmark(text: str):
|
||||
"""Check if this is a Foreword or Introduction bookmark.
|
||||
|
||||
Returns the slug ("foreword" or "introduction") or None.
|
||||
"""
|
||||
stripped = text.strip()
|
||||
if stripped == "LCL: Foreword":
|
||||
return "foreword"
|
||||
if stripped == "LCL: Introduction":
|
||||
return "introduction"
|
||||
return None
|
||||
|
||||
|
||||
def parse_about_author_bookmark(text: str) -> bool:
|
||||
"""Check if this is the About the Author bookmark."""
|
||||
return "His Divine Grace" in text and text.strip().startswith("LCL:")
|
||||
|
||||
|
||||
def extract_paragraphs(
|
||||
all_paragraphs: list,
|
||||
start_idx: int,
|
||||
end_idx: int,
|
||||
) -> list:
|
||||
"""Extract paragraphs for a section (chapter, intro segment).
|
||||
|
||||
Handles all CSS classes within the LCL content.
|
||||
Groups consecutive p5/p180 verse pādas into centered italic divs.
|
||||
Detects verse patterns in dialogue/body (starts with <i>, has <br>).
|
||||
Deduplicates p49 + p36 pairs (only keeps the p36 bold version).
|
||||
"""
|
||||
result = []
|
||||
verse_accumulator = []
|
||||
|
||||
# Track the last p49 topic text so we can skip it if p36 duplicates it
|
||||
last_p49_text = None
|
||||
|
||||
def flush_verse():
|
||||
if verse_accumulator:
|
||||
joined = "<br>".join(verse_accumulator)
|
||||
result.append(f'<div style="text-align:center"><i>{joined}</i></div>')
|
||||
verse_accumulator.clear()
|
||||
|
||||
for i in range(start_idx, end_idx):
|
||||
p = all_paragraphs[i]
|
||||
cls = get_css_class(p)
|
||||
|
||||
# Skip empty paragraphs
|
||||
if is_empty(p):
|
||||
continue
|
||||
|
||||
# Skip noise classes
|
||||
if cls in SKIP_CLASSES:
|
||||
continue
|
||||
|
||||
# Skip bookmark tags (p13, p168)
|
||||
if cls in (BOOKMARK_CLASS, BOOKMARK_CLASS_168):
|
||||
continue
|
||||
|
||||
# Skip book title header (p166)
|
||||
if cls == BOOK_HEADER_CLASS:
|
||||
continue
|
||||
|
||||
# Skip subtitle (p167)
|
||||
if cls == SUBTITLE_CLASS:
|
||||
continue
|
||||
|
||||
# Check for dash lines — skip them
|
||||
text = decode_text(p)
|
||||
if is_dash_line(text):
|
||||
continue
|
||||
|
||||
# Sanskrit verse pādas (p5, p180) — accumulate into centered block
|
||||
if cls in VERSE_PADA_CLASSES:
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
# Remove inner <i> tags before re-wrapping
|
||||
stripped = re.sub(r"</?i>", "", html)
|
||||
verse_accumulator.append(stripped)
|
||||
continue
|
||||
|
||||
# Non-verse paragraph — flush any accumulated verse first
|
||||
flush_verse()
|
||||
|
||||
# Section subheading bookmark (p49) — topic title
|
||||
# Only include if NOT immediately followed by a p36 with the same text
|
||||
if cls == SECTION_BOOKMARK_CLASS:
|
||||
p49_text = text.strip()
|
||||
# Look ahead to see if the next non-empty paragraph is p36 with same content
|
||||
has_p36_duplicate = False
|
||||
for j in range(i + 1, min(i + 4, end_idx)):
|
||||
next_p = all_paragraphs[j]
|
||||
next_cls = get_css_class(next_p)
|
||||
if is_empty(next_p) or next_cls in SKIP_CLASSES:
|
||||
continue
|
||||
if next_cls == "p36":
|
||||
next_text = decode_text(next_p).strip()
|
||||
if next_text == p49_text:
|
||||
has_p36_duplicate = True
|
||||
break
|
||||
if has_p36_duplicate:
|
||||
# Skip p49; the p36 version will be included
|
||||
last_p49_text = p49_text
|
||||
continue
|
||||
else:
|
||||
# No p36 duplicate — include p49 as bold subheading
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
# Wrap in bold if not already
|
||||
if not html.strip().startswith("<b>"):
|
||||
html = f"<b>{html}</b>"
|
||||
result.append(html)
|
||||
last_p49_text = None
|
||||
continue
|
||||
|
||||
# Section subheading (p36) — bold topic title
|
||||
if cls == "p36":
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
# Ensure it's bold
|
||||
if not html.strip().startswith("<b>"):
|
||||
html = f"<b>{html}</b>"
|
||||
result.append(html)
|
||||
last_p49_text = None
|
||||
continue
|
||||
|
||||
# Section heading variant (p64) — topic title (not bold in HTML)
|
||||
if cls == "p64":
|
||||
# p64 is typically followed by p36 with the same text;
|
||||
# if so, skip p64 and let p36 handle it
|
||||
p64_text = text.strip()
|
||||
has_p36_duplicate = False
|
||||
for j in range(i + 1, min(i + 4, end_idx)):
|
||||
next_p = all_paragraphs[j]
|
||||
next_cls = get_css_class(next_p)
|
||||
if is_empty(next_p) or next_cls in SKIP_CLASSES:
|
||||
continue
|
||||
if next_cls == "p36":
|
||||
next_text = decode_text(next_p).strip()
|
||||
if next_text == p64_text:
|
||||
has_p36_duplicate = True
|
||||
break
|
||||
if has_p36_duplicate:
|
||||
# Skip p64; the p36 version will be included
|
||||
continue
|
||||
else:
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
if not html.strip().startswith("<b>"):
|
||||
html = f"<b>{html}</b>"
|
||||
result.append(html)
|
||||
continue
|
||||
|
||||
# Dialogue lines (p114, p118, p178, p181) — the main content classes
|
||||
if cls in DIALOGUE_CLASSES:
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
# Detect verses disguised as dialogue: starts with <i> and has <br>
|
||||
stripped_html = html.strip()
|
||||
if stripped_html.startswith("<i>") and "<br" in stripped_html:
|
||||
inner = re.sub(r"</?i>", "", stripped_html)
|
||||
result.append(
|
||||
f'<div style="text-align:center"><i>{inner}</i></div>'
|
||||
)
|
||||
else:
|
||||
result.append(html)
|
||||
continue
|
||||
|
||||
# Chapter heading (p27) — include as paragraph
|
||||
if cls == CHAPTER_HEADING_CLASS:
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
result.append(html)
|
||||
continue
|
||||
|
||||
# Foreword heading (p169) — include
|
||||
if cls == FOREWORD_HEADING_CLASS:
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
result.append(html)
|
||||
continue
|
||||
|
||||
# Title page / founder-ācārya line (p3) — include
|
||||
if cls == TITLE_PAGE_CLASS:
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
result.append(html)
|
||||
continue
|
||||
|
||||
# Companion description (p177) — italic description
|
||||
if cls == COMPANION_CLASS:
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
result.append(html)
|
||||
continue
|
||||
|
||||
# Block quotes (p173, p175)
|
||||
if cls in BLOCKQUOTE_CLASSES:
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
result.append(html)
|
||||
continue
|
||||
|
||||
# Attribution text (p174, p176, p42)
|
||||
if cls in ATTRIBUTION_CLASSES:
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
result.append(html)
|
||||
continue
|
||||
|
||||
# Verse translations (p29, p94, p184, p186)
|
||||
if cls in TRANSLATION_CLASSES:
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
result.append(html)
|
||||
continue
|
||||
|
||||
# Reference / citation line (p187)
|
||||
if cls == REFERENCE_CLASS:
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
result.append(html)
|
||||
continue
|
||||
|
||||
# Body paragraphs (p170, p171, p6, p68, p182)
|
||||
if cls in BODY_CLASSES:
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
# Detect Sanskrit verses disguised as body paragraphs:
|
||||
# they start with <i> and contain <br> (multi-line italic = verse)
|
||||
stripped_html = html.strip()
|
||||
if stripped_html.startswith("<i>") and "<br" in stripped_html:
|
||||
inner = re.sub(r"</?i>", "", stripped_html)
|
||||
result.append(
|
||||
f'<div style="text-align:center"><i>{inner}</i></div>'
|
||||
)
|
||||
else:
|
||||
result.append(html)
|
||||
continue
|
||||
|
||||
# Any other class — try to extract content (be permissive)
|
||||
html = decode_html_content(p)
|
||||
if html and not is_dash_line(text):
|
||||
result.append(html)
|
||||
|
||||
# Flush any trailing verse
|
||||
flush_verse()
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main extraction logic
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def extract_lcl(soup: BeautifulSoup) -> dict:
|
||||
"""Extract the entire Life Comes from Life from the parsed HTML.
|
||||
|
||||
Returns a dict with:
|
||||
"intro_segments": list of intro segment dicts
|
||||
"chapters": list of chapter dicts
|
||||
"chapter_listing": list of {number, title} for intro.json
|
||||
"""
|
||||
all_paragraphs = list(soup.find_all("p"))
|
||||
|
||||
# Step 1: Find LCL boundaries
|
||||
lcl_start, lcl_end = find_lcl_boundaries(all_paragraphs)
|
||||
print(f" LCL content: paragraphs {lcl_start} to {lcl_end}")
|
||||
print(f" Total paragraphs in range: {lcl_end - lcl_start}")
|
||||
|
||||
# Step 2: Find all bookmarks
|
||||
bookmarks = find_all_bookmarks(all_paragraphs, lcl_start, lcl_end)
|
||||
print(f" Found {len(bookmarks)} LCL bookmarks")
|
||||
for pos, text in bookmarks:
|
||||
print(f" Para ~{pos}: {text}")
|
||||
|
||||
# Step 3: Categorize bookmarks
|
||||
foreword_bookmark = None # paragraph index
|
||||
introduction_bookmark = None
|
||||
about_author_bookmark = None
|
||||
chapter_bookmarks = [] # (para_idx, chapter_num, title)
|
||||
|
||||
for pos, text in bookmarks:
|
||||
ch = parse_chapter_bookmark(text)
|
||||
if ch:
|
||||
chapter_bookmarks.append((pos, ch[0], ch[1]))
|
||||
continue
|
||||
intro_slug = parse_intro_bookmark(text)
|
||||
if intro_slug == "foreword":
|
||||
foreword_bookmark = pos
|
||||
elif intro_slug == "introduction":
|
||||
introduction_bookmark = pos
|
||||
elif parse_about_author_bookmark(text):
|
||||
about_author_bookmark = pos
|
||||
|
||||
print(
|
||||
f" Foreword bookmark: {'found' if foreword_bookmark is not None else 'NOT FOUND'}"
|
||||
)
|
||||
print(
|
||||
f" Introduction bookmark: {'found' if introduction_bookmark is not None else 'NOT FOUND'}"
|
||||
)
|
||||
print(f" Chapter bookmarks: {len(chapter_bookmarks)}")
|
||||
print(
|
||||
f" About the Author bookmark: {'found' if about_author_bookmark is not None else 'NOT FOUND'}"
|
||||
)
|
||||
|
||||
# Build an ordered list of all section start positions for boundary computation
|
||||
all_section_starts = sorted(
|
||||
([foreword_bookmark] if foreword_bookmark is not None else [])
|
||||
+ ([introduction_bookmark] if introduction_bookmark is not None else [])
|
||||
+ [pos for pos, _, _ in chapter_bookmarks]
|
||||
+ ([about_author_bookmark] if about_author_bookmark is not None else [])
|
||||
)
|
||||
|
||||
def next_section_after(pos):
|
||||
"""Return the next section start after `pos`, or lcl_end."""
|
||||
for s in all_section_starts:
|
||||
if s > pos:
|
||||
return s
|
||||
return lcl_end
|
||||
|
||||
# Step 4: Extract Foreword
|
||||
foreword = None
|
||||
if foreword_bookmark is not None:
|
||||
fw_content_start = foreword_bookmark + 1
|
||||
fw_content_end = next_section_after(foreword_bookmark)
|
||||
fw_paras = extract_paragraphs(all_paragraphs, fw_content_start, fw_content_end)
|
||||
foreword = {
|
||||
"slug": "foreword",
|
||||
"title": "Foreword",
|
||||
"type": "intro-segment",
|
||||
"book": "lcl",
|
||||
"order": 1,
|
||||
"paragraphs": fw_paras,
|
||||
}
|
||||
|
||||
# Step 5: Extract Introduction
|
||||
introduction = None
|
||||
if introduction_bookmark is not None:
|
||||
intro_content_start = introduction_bookmark + 1
|
||||
intro_content_end = next_section_after(introduction_bookmark)
|
||||
intro_paras = extract_paragraphs(
|
||||
all_paragraphs, intro_content_start, intro_content_end
|
||||
)
|
||||
introduction = {
|
||||
"slug": "introduction",
|
||||
"title": "Introduction",
|
||||
"type": "intro-segment",
|
||||
"book": "lcl",
|
||||
"order": 2,
|
||||
"paragraphs": intro_paras,
|
||||
}
|
||||
|
||||
# Step 6: Extract chapters 1–16
|
||||
chapters = []
|
||||
for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_bookmarks):
|
||||
content_start = ch_pos + 1
|
||||
content_end = next_section_after(ch_pos)
|
||||
|
||||
paragraphs = extract_paragraphs(all_paragraphs, content_start, content_end)
|
||||
|
||||
ch = {
|
||||
"book": "lcl",
|
||||
"chapter": ch_num,
|
||||
"title": ch_title,
|
||||
"type": "chapter",
|
||||
"paragraphs": paragraphs,
|
||||
}
|
||||
chapters.append(ch)
|
||||
|
||||
# Step 7: Extract About the Author
|
||||
about_the_author = None
|
||||
if about_author_bookmark is not None:
|
||||
ata_content_start = about_author_bookmark + 1
|
||||
ata_content_end = lcl_end
|
||||
ata_paras = extract_paragraphs(
|
||||
all_paragraphs, ata_content_start, ata_content_end
|
||||
)
|
||||
about_the_author = {
|
||||
"slug": "about-the-author",
|
||||
"title": "About the Author",
|
||||
"type": "intro-segment",
|
||||
"book": "lcl",
|
||||
"order": 99,
|
||||
"paragraphs": ata_paras,
|
||||
}
|
||||
|
||||
# Step 8: Build intro segments list
|
||||
intro_segments = []
|
||||
if foreword:
|
||||
intro_segments.append(foreword)
|
||||
if introduction:
|
||||
intro_segments.append(introduction)
|
||||
if about_the_author:
|
||||
intro_segments.append(about_the_author)
|
||||
|
||||
# Step 9: Build chapter listing for intro.json
|
||||
chapter_listing = [
|
||||
{"number": ch["chapter"], "title": ch["title"]} for ch in chapters
|
||||
]
|
||||
|
||||
return {
|
||||
"intro_segments": intro_segments,
|
||||
"chapters": chapters,
|
||||
"chapter_listing": chapter_listing,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Output
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def write_json(data, filepath):
|
||||
"""Write data as formatted JSON to filepath, creating directories as needed."""
|
||||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||||
with open(filepath, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "other.html")
|
||||
output_base = os.path.join(base_dir, "lib", "jsons", "books", "lcl")
|
||||
|
||||
if not os.path.exists(html_path):
|
||||
print(f"ERROR: {html_path} not found!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Reading {html_path}...")
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
soup = BeautifulSoup(f, "html.parser")
|
||||
|
||||
print("Extracting Life Comes from Life (1979)...\n")
|
||||
result = extract_lcl(soup)
|
||||
|
||||
intro_segments = result["intro_segments"]
|
||||
chapters = result["chapters"]
|
||||
chapter_listing = result["chapter_listing"]
|
||||
|
||||
# --- Write intro.json ---
|
||||
intro_meta = {
|
||||
"book": "lcl",
|
||||
"title": "Life Comes from Life",
|
||||
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
||||
"year": 1979,
|
||||
"description": "Morning walks with Śrīla Prabhupāda — a critique of modern science's theories on the origin of life.",
|
||||
"totalChapters": 16,
|
||||
"chapters": chapter_listing,
|
||||
}
|
||||
intro_path = os.path.join(output_base, "intro.json")
|
||||
write_json(intro_meta, intro_path)
|
||||
print(f"\n Written: {intro_path}")
|
||||
|
||||
# --- Write intro segments ---
|
||||
print(f"\n--- Intro Segments ---")
|
||||
for seg in intro_segments:
|
||||
filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
|
||||
write_json(seg, filepath)
|
||||
print(
|
||||
f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}"
|
||||
)
|
||||
|
||||
# --- Write chapters ---
|
||||
print(f"\n--- Chapters ---")
|
||||
for ch in chapters:
|
||||
ch_num_str = str(ch["chapter"]).zfill(2)
|
||||
filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
|
||||
write_json(ch, filepath)
|
||||
print(
|
||||
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
|
||||
f"{ch['title']}"
|
||||
)
|
||||
|
||||
# --- Summary ---
|
||||
total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments)
|
||||
total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
|
||||
total_paras = total_intro_paras + total_chapter_paras
|
||||
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f" EXTRACTION SUMMARY")
|
||||
print(f"{'=' * 60}")
|
||||
print(f" Intro segments: {len(intro_segments)}")
|
||||
for seg in intro_segments:
|
||||
print(
|
||||
f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs"
|
||||
)
|
||||
print(f" Chapters: {len(chapters)}")
|
||||
print(f" Total paragraphs: {total_paras}")
|
||||
print(f" Intro: {total_intro_paras}")
|
||||
print(f" Chapters: {total_chapter_paras}")
|
||||
if chapters:
|
||||
print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
|
||||
print(f" Output directory: {output_base}/")
|
||||
|
||||
# --- Samples ---
|
||||
print(f"\n--- Sample: Chapter 1 ---")
|
||||
if chapters:
|
||||
ch1 = chapters[0]
|
||||
for i, p in enumerate(ch1["paragraphs"][:5]):
|
||||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
||||
print(f" [{i}] {preview}...")
|
||||
|
||||
if len(chapters) >= 8:
|
||||
print(f"\n--- Sample: Chapter 8 ---")
|
||||
ch8 = [c for c in chapters if c["chapter"] == 8]
|
||||
if ch8:
|
||||
for i, p in enumerate(ch8[0]["paragraphs"][:3]):
|
||||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
||||
print(f" [{i}] {preview}...")
|
||||
|
||||
if len(chapters) >= 16:
|
||||
print(f"\n--- Sample: Chapter 16 (last) ---")
|
||||
last = chapters[-1]
|
||||
for i, p in enumerate(last["paragraphs"][:3]):
|
||||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
||||
print(f" [{i}] {preview}...")
|
||||
|
||||
# --- Quality Checks ---
|
||||
print(f"\n--- Quality Checks ---")
|
||||
|
||||
# Check all 16 chapters present
|
||||
expected_chapters = set(range(1, 17))
|
||||
actual_chapters = set(ch["chapter"] for ch in chapters)
|
||||
missing = expected_chapters - actual_chapters
|
||||
extra = actual_chapters - expected_chapters
|
||||
if missing:
|
||||
print(f" ⚠ Missing chapters: {sorted(missing)}")
|
||||
if extra:
|
||||
print(f" ⚠ Extra chapters: {sorted(extra)}")
|
||||
if not missing and not extra:
|
||||
print(f" ✅ All 16 chapters present")
|
||||
|
||||
# Check intro segments
|
||||
expected_intros = {"foreword", "introduction", "about-the-author"}
|
||||
actual_intros = {s["slug"] for s in intro_segments}
|
||||
missing_intros = expected_intros - actual_intros
|
||||
if missing_intros:
|
||||
print(f" ⚠ Missing intro segments: {missing_intros}")
|
||||
else:
|
||||
print(f" ✅ All 3 intro segments present")
|
||||
|
||||
# Check empty chapters
|
||||
empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
|
||||
if empty_chapters:
|
||||
print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
|
||||
else:
|
||||
print(f" ✅ No empty chapters")
|
||||
|
||||
# Check ScaGoudy decoding — look for common encoded characters
|
||||
encoded_chars = set("äåöëòçüéàùè")
|
||||
suspect_count = 0
|
||||
all_sections = chapters + intro_segments
|
||||
for section in all_sections:
|
||||
for p in section["paragraphs"]:
|
||||
plain = re.sub(r"<[^>]+>", "", p)
|
||||
for c in encoded_chars:
|
||||
if c in plain:
|
||||
suspect_count += 1
|
||||
break
|
||||
if suspect_count == 0:
|
||||
print(f" ✅ No suspected undecoded ScaGoudy characters found")
|
||||
else:
|
||||
print(
|
||||
f" ℹ {suspect_count} paragraphs contain characters that may be undecoded ScaGoudy"
|
||||
)
|
||||
|
||||
# Check verse blocks
|
||||
verse_block_count = 0
|
||||
for ch in chapters:
|
||||
for p in ch["paragraphs"]:
|
||||
if '<div style="text-align:center"><i>' in p:
|
||||
verse_block_count += 1
|
||||
print(f" ℹ {verse_block_count} verse blocks in chapters")
|
||||
|
||||
# ScaGoudy decoding spot check
|
||||
print(f"\n--- ScaGoudy Decoding Spot Check ---")
|
||||
if chapters:
|
||||
ch1_text = " ".join(chapters[0]["paragraphs"][:15])
|
||||
if "Prabhupāda" in ch1_text:
|
||||
print(f" ✅ 'Prabhupāda' correctly decoded")
|
||||
elif "Prabhupäda" in ch1_text:
|
||||
print(f" ⚠ ScaGoudy decoding incomplete for 'Prabhupäda'")
|
||||
else:
|
||||
print(f" ℹ 'Prabhupāda' not found in Chapter 1 first 15 paragraphs")
|
||||
|
||||
if "Kṛṣṇa" in ch1_text:
|
||||
print(f" ✅ 'Kṛṣṇa' correctly decoded in Chapter 1")
|
||||
elif "Kåñëa" in ch1_text:
|
||||
print(f" ⚠ 'Kåñëa' still encoded — ScaGoudy decoding may have failed!")
|
||||
else:
|
||||
print(
|
||||
f" ℹ 'Kṛṣṇa' not found in Chapter 1 first 15 paragraphs (may be normal)"
|
||||
)
|
||||
|
||||
# Check for Vedas
|
||||
if "Vedas" in ch1_text or "Vedic" in ch1_text:
|
||||
print(f" ✅ 'Vedas'/'Vedic' found in Chapter 1")
|
||||
|
||||
# Dialogue speaker check
|
||||
print(f"\n--- Dialogue Speaker Check ---")
|
||||
speaker_counts = {}
|
||||
for ch in chapters:
|
||||
for p in ch["paragraphs"]:
|
||||
# Look for bold speaker patterns: <b>SPEAKER.</b> or <b>SPEAKER:</b>
|
||||
m = re.match(r"<b>([^<]+?)[.:]</b>", p)
|
||||
if m:
|
||||
speaker = m.group(1).strip()
|
||||
speaker_counts[speaker] = speaker_counts.get(speaker, 0) + 1
|
||||
if speaker_counts:
|
||||
print(f" Speakers found:")
|
||||
for speaker, count in sorted(speaker_counts.items(), key=lambda x: -x[1]):
|
||||
print(f" {speaker}: {count} lines")
|
||||
else:
|
||||
print(f" ⚠ No dialogue speakers detected")
|
||||
|
||||
# Sample verse blocks
|
||||
print(f"\n--- Sample Verse Blocks ---")
|
||||
shown = 0
|
||||
for ch in chapters:
|
||||
for p in ch["paragraphs"]:
|
||||
if '<div style="text-align:center"><i>' in p:
|
||||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
||||
print(f" Ch {ch['chapter']}: {preview}...")
|
||||
shown += 1
|
||||
if shown >= 3:
|
||||
break
|
||||
if shown >= 3:
|
||||
break
|
||||
|
||||
# Check section subheadings (topic titles)
|
||||
print(f"\n--- Section Subheadings in Chapters ---")
|
||||
subheading_count = 0
|
||||
sample_subheadings = []
|
||||
for ch in chapters:
|
||||
for p in ch["paragraphs"]:
|
||||
plain = re.sub(r"<[^>]+>", "", p).strip()
|
||||
# A bold-only paragraph that's short is likely a subheading
|
||||
if p.startswith("<b>") and p.endswith("</b>") and len(plain) < 80:
|
||||
subheading_count += 1
|
||||
if len(sample_subheadings) < 5:
|
||||
sample_subheadings.append(f" Ch {ch['chapter']}: {plain}")
|
||||
print(f" Total subheadings: {subheading_count}")
|
||||
for s in sample_subheadings:
|
||||
print(s)
|
||||
|
||||
print(f"\nDone! ✨")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user