1083 lines
36 KiB
Python
1083 lines
36 KiB
Python
#!/usr/bin/env python3
|
|
"""Extract Dialectic Spiritualism (1985) from other.html.
|
|
|
|
Hierarchical sub-chapter extraction: each philosopher sub-chapter
|
|
becomes its OWN chapter JSON file with a parent-child relationship.
|
|
|
|
Parent chapters (13 total with Roman numeral prefixes) are containers only.
|
|
|
|
Sub-chapter bookmarks use class p49 with "DS:" prefix.
|
|
Parent chapter bookmarks use class p13 with concatenated Roman numerals.
|
|
|
|
Output:
|
|
lib/jsons/books/ds/intro.json — Book metadata with hierarchical chapters
|
|
lib/jsons/books/ds/intro/*.json — Intro segments (front-pages, foreword, preface)
|
|
lib/jsons/books/ds/chapters/*.json — Sub-chapter files (1-1.json, 1-2.json, ...)
|
|
|
|
Usage:
|
|
python3 scripts/ds_extract.py
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
from collections import defaultdict
|
|
|
|
from bs4 import BeautifulSoup, Tag
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ScaGoudy → IAST Unicode mapping
|
|
# ---------------------------------------------------------------------------
|
|
SCAGOUDY_TO_IAST = {
|
|
"ä": "ā",
|
|
"Ä": "Ā",
|
|
"å": "ṛ",
|
|
"Å": "Ṛ",
|
|
"ñ": "ṣ",
|
|
"Ñ": "Ṣ",
|
|
"ö": "ṭ",
|
|
"Ö": "Ṭ",
|
|
"ë": "ṇ",
|
|
"Ë": "Ṇ",
|
|
"ò": "ḍ",
|
|
"Ò": "Ḍ",
|
|
"ç": "ś",
|
|
"Ç": "Ś",
|
|
"ì": "ṅ",
|
|
"ï": "ñ",
|
|
"ü": "ū",
|
|
"Ü": "Ū",
|
|
"é": "ī",
|
|
"É": "Ī",
|
|
"à": "ṁ",
|
|
"ù": "ḥ",
|
|
"è": "ṝ",
|
|
"û": "ṁl",
|
|
"ÿ": "ḷ",
|
|
}
|
|
|
|
|
|
def decode_scagoudy(text: str) -> str:
|
|
"""Replace ScaGoudy-encoded characters with proper IAST Unicode."""
|
|
for src, dst in SCAGOUDY_TO_IAST.items():
|
|
text = text.replace(src, dst)
|
|
return text
|
|
|
|
|
|
def decode_html_content(tag: Tag) -> str:
|
|
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
|
|
inner = "".join(str(child) for child in tag.children)
|
|
inner = decode_scagoudy(inner)
|
|
# Remove span tags with CSS classes (RTF conversion artifacts)
|
|
inner = re.sub(
|
|
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
|
|
)
|
|
# Clean up Apple-tab-span leftovers (tab characters)
|
|
inner = inner.replace("\t", " ")
|
|
# Normalize whitespace (but preserve <br> tags)
|
|
inner = re.sub(r"\s+", " ", inner).strip()
|
|
return inner
|
|
|
|
|
|
def decode_text(tag: Tag) -> str:
|
|
"""Get plain text from a tag with ScaGoudy decoding."""
|
|
return decode_scagoudy(tag.get_text().strip())
|
|
|
|
|
|
def get_css_class(tag: Tag) -> str:
|
|
"""Get the first CSS class of a tag, or empty string."""
|
|
classes = tag.get("class", [])
|
|
return classes[0] if classes else ""
|
|
|
|
|
|
def is_empty(tag: Tag) -> bool:
|
|
"""Check if a tag is empty or whitespace-only."""
|
|
text = tag.get_text().strip()
|
|
return not text or text == "\xa0"
|
|
|
|
|
|
def is_dash_line(text: str) -> bool:
|
|
"""Return True if text is only dashes, spaces, and/or whitespace."""
|
|
cleaned = text.strip()
|
|
if not cleaned:
|
|
return False
|
|
return all(c in "-—–\u2014\u2013 \t*" for c in cleaned)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# CSS class constants
|
|
# ---------------------------------------------------------------------------
|
|
|
|
BOOK_HEADER_CLASS = "p12"
|
|
|
|
# Parent chapter bookmark class (p13) — Roman numeral concatenated with title
|
|
CHAPTER_BOOKMARK_CLASS = "p13"
|
|
|
|
# Sub-chapter bookmark class (p49) — "DS:Philosopher Name (dates)"
|
|
SUBCHAPTER_BOOKMARK_CLASS = "p49"
|
|
|
|
# Verse/quote classes — group consecutive into centered italic divs
|
|
VERSE_QUOTE_CLASSES = frozenset(["p24", "p66", "p121", "p134", "p157", "p163"])
|
|
|
|
# Body/dialogue classes — include as content
|
|
BODY_CLASSES = frozenset(
|
|
[
|
|
"p118", # Main dialogue lines (bold speaker + text)
|
|
"p156", # Body/dialogue
|
|
"p6", # Body
|
|
"p159", # Body
|
|
"p160", # Body
|
|
"p140", # Body
|
|
"p44", # Verse translations
|
|
"p23", # Misc body
|
|
"p162", # Dialogue (bold formatting variant)
|
|
"p161", # Quote/verse translations
|
|
"p112", # Dialogue variant
|
|
"p158", # Quote
|
|
"p56", # Quote/translation
|
|
"p55", # TRANSLATION/PURPORT labels
|
|
]
|
|
)
|
|
|
|
# Front matter classes
|
|
FRONT_MATTER_CLASSES = frozenset(
|
|
[
|
|
"p5", # Title/front pages
|
|
"p25", # Contact/front
|
|
"p18", # Front pages
|
|
"p154", # Back cover label
|
|
"p155", # Foreword heading
|
|
"p26", # Misc front
|
|
]
|
|
)
|
|
|
|
# Skip classes — empty or noise
|
|
SKIP_CLASSES = frozenset(["p15", "p153"])
|
|
|
|
# Parent chapter heading class (p27) — Roman numeral headings, philosopher headings
|
|
PARENT_HEADING_CLASS = "p27"
|
|
|
|
# Sub-chapter title class (p31) — duplicate of bookmark title
|
|
SUBCHAPTER_TITLE_CLASS = "p31"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Roman numeral parsing
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Ordered longest-first to prevent partial matches
|
|
ROMAN_NUMERALS_ORDERED = [
|
|
"XIII",
|
|
"XII",
|
|
"XI",
|
|
"X",
|
|
"IX",
|
|
"VIII",
|
|
"VII",
|
|
"VI",
|
|
"V",
|
|
"IV",
|
|
"III",
|
|
"II",
|
|
"I",
|
|
]
|
|
|
|
ROMAN_TO_ARABIC = {
|
|
"I": 1,
|
|
"II": 2,
|
|
"III": 3,
|
|
"IV": 4,
|
|
"V": 5,
|
|
"VI": 6,
|
|
"VII": 7,
|
|
"VIII": 8,
|
|
"IX": 9,
|
|
"X": 10,
|
|
"XI": 11,
|
|
"XII": 12,
|
|
"XIII": 13,
|
|
}
|
|
|
|
# Clean chapter titles (pre-decoded IAST)
|
|
CLEAN_PARENT_TITLES = {
|
|
1: "The Greek Foundation",
|
|
2: "The Early Theists",
|
|
3: "Scholasticism",
|
|
4: "Renaissance Thought",
|
|
5: "Rationalism",
|
|
6: "British Empiricism",
|
|
7: "German Idealism",
|
|
8: "Evolutionary Naturalism",
|
|
9: "Utilitarianism and Positivism",
|
|
10: "Communism",
|
|
11: "American Pragmatism",
|
|
12: "Existentialism",
|
|
13: "Psychoanalysis",
|
|
}
|
|
|
|
|
|
def parse_roman_chapter(text: str):
|
|
"""Parse a p13 bookmark text like 'ITHE GREEK FOUNDATION'.
|
|
|
|
Returns (chapter_number, raw_title) or None if not a valid chapter heading.
|
|
"""
|
|
text = text.strip()
|
|
for roman in ROMAN_NUMERALS_ORDERED:
|
|
if text.startswith(roman):
|
|
remainder = text[len(roman) :]
|
|
if remainder:
|
|
return ROMAN_TO_ARABIC[roman], remainder.strip()
|
|
return None
|
|
|
|
|
|
def parse_ds_sub_bookmark(text: str):
|
|
"""Parse a p49 bookmark text like 'DS:Socrates (470?-399 B.C.)'.
|
|
|
|
Returns the title after 'DS:' or None if not a DS bookmark.
|
|
"""
|
|
m = re.match(r"DS:(.+)", text.strip())
|
|
if m:
|
|
return m.group(1).strip()
|
|
return None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Boundary detection
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def find_ds_boundaries(all_paragraphs: list) -> tuple:
|
|
"""Find the start and end paragraph indices of the DS content.
|
|
|
|
Start: <p class="p12"> containing "Dialectic Spiritualism" and "1985"
|
|
End: next <p class="p12"> after DS content.
|
|
"""
|
|
start_idx = None
|
|
end_idx = len(all_paragraphs)
|
|
|
|
for i, p in enumerate(all_paragraphs):
|
|
cls = get_css_class(p)
|
|
text = p.get_text().strip()
|
|
|
|
if (
|
|
cls == BOOK_HEADER_CLASS
|
|
and "Dialectic Spiritualism" in text
|
|
and "1985" in text
|
|
):
|
|
start_idx = i
|
|
|
|
if start_idx is not None and i > start_idx and cls == BOOK_HEADER_CLASS:
|
|
end_idx = i
|
|
break
|
|
|
|
if start_idx is None:
|
|
print("ERROR: Could not find 'Dialectic Spiritualism (1985)' header!")
|
|
sys.exit(1)
|
|
|
|
return start_idx, end_idx
|
|
|
|
|
|
def find_chapter_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
|
|
"""Find all DS main chapter bookmark tags (p13) in range.
|
|
|
|
Returns list of (paragraph_index, chapter_number, raw_title).
|
|
"""
|
|
bookmarks = []
|
|
for i in range(start_idx, end_idx):
|
|
p = all_paragraphs[i]
|
|
cls = get_css_class(p)
|
|
if cls != CHAPTER_BOOKMARK_CLASS:
|
|
continue
|
|
text = decode_text(p)
|
|
result = parse_roman_chapter(text)
|
|
if result:
|
|
ch_num, raw_title = result
|
|
bookmarks.append((i, ch_num, raw_title))
|
|
return bookmarks
|
|
|
|
|
|
def find_sub_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
|
|
"""Find all DS sub-chapter bookmark tags (p49 with 'DS:' prefix) in range.
|
|
|
|
Returns list of (paragraph_index, title).
|
|
"""
|
|
bookmarks = []
|
|
for i in range(start_idx, end_idx):
|
|
p = all_paragraphs[i]
|
|
cls = get_css_class(p)
|
|
if cls != SUBCHAPTER_BOOKMARK_CLASS:
|
|
continue
|
|
text = decode_text(p)
|
|
title = parse_ds_sub_bookmark(text)
|
|
if title is not None:
|
|
bookmarks.append((i, title))
|
|
return bookmarks
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Paragraph extraction
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _normalize_for_match(text: str) -> str:
|
|
"""Normalize text for fuzzy title matching: lowercase, collapse whitespace, strip quotes."""
|
|
text = re.sub(r"\s+", " ", text.strip().lower())
|
|
text = text.strip('""\u201c\u201d\'')
|
|
return text
|
|
|
|
|
|
def extract_paragraphs(
|
|
all_paragraphs: list,
|
|
start_idx: int,
|
|
end_idx: int,
|
|
skip_bookmarks: bool = True,
|
|
skip_p27: bool = True,
|
|
skip_first_p31_matching: str = None,
|
|
is_front_matter: bool = False,
|
|
) -> list:
|
|
"""Extract paragraphs for a section.
|
|
|
|
Handles all CSS classes within DS content.
|
|
Groups consecutive verse quote pādas (p24, p66, etc.) into centered italic divs.
|
|
|
|
Args:
|
|
all_paragraphs: full list of <p> tags
|
|
start_idx: first paragraph index (inclusive)
|
|
end_idx: last paragraph index (exclusive)
|
|
skip_bookmarks: if True, skip p13 and p49 bookmark tags
|
|
skip_p27: if True, skip the p27 parent chapter heading
|
|
skip_first_p31_matching: if set, skip the FIRST p31 whose text matches
|
|
this string (case-insensitive, normalized). Used to remove duplicate
|
|
sub-chapter title headings.
|
|
is_front_matter: if True, include front matter classes
|
|
"""
|
|
result = []
|
|
verse_accumulator = []
|
|
first_p31_skipped = False
|
|
|
|
def flush_verse():
|
|
if verse_accumulator:
|
|
joined = "<br>".join(verse_accumulator)
|
|
result.append(f'<div style="text-align:center"><i>{joined}</i></div>')
|
|
verse_accumulator.clear()
|
|
|
|
for i in range(start_idx, end_idx):
|
|
p = all_paragraphs[i]
|
|
cls = get_css_class(p)
|
|
|
|
# Skip empty paragraphs
|
|
if is_empty(p):
|
|
continue
|
|
|
|
# Skip noise classes
|
|
if cls in SKIP_CLASSES:
|
|
continue
|
|
|
|
# Skip book title header (p12)
|
|
if cls == BOOK_HEADER_CLASS:
|
|
continue
|
|
|
|
# Skip bookmark classes (p13, p49) — already captured in metadata
|
|
if skip_bookmarks:
|
|
if cls == CHAPTER_BOOKMARK_CLASS:
|
|
continue
|
|
if cls == SUBCHAPTER_BOOKMARK_CLASS:
|
|
continue
|
|
|
|
# Skip p27 parent chapter heading if requested
|
|
if skip_p27 and cls == PARENT_HEADING_CLASS:
|
|
continue
|
|
|
|
# Skip the first p31 that matches the sub-chapter title (duplicate heading)
|
|
if (
|
|
not first_p31_skipped
|
|
and skip_first_p31_matching is not None
|
|
and cls == SUBCHAPTER_TITLE_CLASS
|
|
):
|
|
p31_text = _normalize_for_match(decode_text(p))
|
|
target = _normalize_for_match(skip_first_p31_matching)
|
|
if p31_text == target:
|
|
first_p31_skipped = True
|
|
continue
|
|
|
|
# Check for dash lines — skip
|
|
text = decode_text(p)
|
|
if is_dash_line(text):
|
|
continue
|
|
|
|
# Sanskrit/Bengali verse quotes (p24, p66, p121, p134, p157, p163)
|
|
# — accumulate into centered block
|
|
if cls in VERSE_QUOTE_CLASSES:
|
|
html = decode_html_content(p)
|
|
if html:
|
|
# Strip <i> tags for uniform formatting (we'll wrap in <i> at flush)
|
|
stripped = re.sub(r"</?i>", "", html)
|
|
verse_accumulator.append(stripped)
|
|
continue
|
|
|
|
# Non-verse paragraph — flush any accumulated verse first
|
|
flush_verse()
|
|
|
|
# Front matter classes
|
|
if is_front_matter and cls in FRONT_MATTER_CLASSES:
|
|
html = decode_html_content(p)
|
|
if html:
|
|
result.append(html)
|
|
continue
|
|
|
|
# Body/dialogue classes
|
|
if cls in BODY_CLASSES:
|
|
html = decode_html_content(p)
|
|
if html and not is_dash_line(text):
|
|
# Detect Sanskrit verses disguised as body paragraphs:
|
|
# they start with <i> and contain <br> (multi-line italic = verse)
|
|
stripped_html = html.strip()
|
|
if stripped_html.startswith("<i>") and "<br" in stripped_html:
|
|
inner = re.sub(r"</?i>", "", stripped_html)
|
|
result.append(
|
|
f'<div style="text-align:center"><i>{inner}</i></div>'
|
|
)
|
|
else:
|
|
result.append(html)
|
|
continue
|
|
|
|
# Sub-chapter title heading (p31) — include if not the duplicate
|
|
if cls == SUBCHAPTER_TITLE_CLASS:
|
|
html = decode_html_content(p)
|
|
if html:
|
|
result.append(html)
|
|
continue
|
|
|
|
# Front matter classes (if not in front matter mode, still include p155, etc.)
|
|
if cls in FRONT_MATTER_CLASSES:
|
|
html = decode_html_content(p)
|
|
if html:
|
|
stripped_html = html.strip()
|
|
if stripped_html.startswith("<i>") and "<br" in stripped_html:
|
|
inner = re.sub(r"</?i>", "", stripped_html)
|
|
result.append(
|
|
f'<div style="text-align:center"><i>{inner}</i></div>'
|
|
)
|
|
else:
|
|
result.append(html)
|
|
continue
|
|
|
|
# Any other class — try to include if it has content
|
|
html = decode_html_content(p)
|
|
if html and not is_dash_line(text):
|
|
# Detect multi-line italic = verse
|
|
stripped_html = html.strip()
|
|
if stripped_html.startswith("<i>") and "<br" in stripped_html:
|
|
inner = re.sub(r"</?i>", "", stripped_html)
|
|
result.append(f'<div style="text-align:center"><i>{inner}</i></div>')
|
|
else:
|
|
result.append(html)
|
|
|
|
# Flush any trailing verse
|
|
flush_verse()
|
|
|
|
return result
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main extraction
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def extract_ds(soup: BeautifulSoup) -> dict:
|
|
"""Extract the entire Dialectic Spiritualism from parsed HTML.
|
|
|
|
Returns dict with:
|
|
"intro_segments": list of intro segment dicts
|
|
"sub_chapters": list of sub-chapter dicts
|
|
"parent_chapters": list of parent chapter info for intro.json
|
|
"""
|
|
all_paragraphs = list(soup.find_all("p"))
|
|
|
|
# --- Step 1: Find DS boundaries ---
|
|
ds_start, ds_end = find_ds_boundaries(all_paragraphs)
|
|
print(f" DS content: paragraphs {ds_start} to {ds_end}")
|
|
print(f" Total paragraphs in range: {ds_end - ds_start}")
|
|
|
|
# --- Step 2: Find main chapter bookmarks (p13) ---
|
|
chapter_bookmarks = find_chapter_bookmarks(all_paragraphs, ds_start, ds_end)
|
|
print(f" Found {len(chapter_bookmarks)} chapter bookmarks (p13)")
|
|
for pos, ch_num, raw_title in chapter_bookmarks:
|
|
clean_title = CLEAN_PARENT_TITLES.get(ch_num, raw_title)
|
|
print(f" Ch {ch_num}: {clean_title} (paragraph {pos})")
|
|
|
|
# --- Step 3: Find all sub-chapter bookmarks (p49 with DS: prefix) ---
|
|
all_sub_bookmarks = find_sub_bookmarks(all_paragraphs, ds_start, ds_end)
|
|
print(f" Found {len(all_sub_bookmarks)} sub-chapter bookmarks (p49 DS:)")
|
|
for pos, title in all_sub_bookmarks:
|
|
print(f" paragraph {pos}: {title}")
|
|
|
|
# --- Step 4: Classify sub-bookmarks as intro segments or philosopher sub-chapters ---
|
|
# Known intro segments
|
|
INTRO_SEGMENTS_MAP = {
|
|
"Forward": {"slug": "foreword", "title": "Foreword", "order": 1},
|
|
"Preface": {"slug": "preface", "title": "Preface", "order": 2},
|
|
}
|
|
|
|
intro_bookmark_list = [] # (pos, slug, title, order)
|
|
philosopher_bookmarks = [] # (pos, title)
|
|
|
|
for pos, title in all_sub_bookmarks:
|
|
if title in INTRO_SEGMENTS_MAP:
|
|
seg = INTRO_SEGMENTS_MAP[title]
|
|
intro_bookmark_list.append((pos, seg["slug"], seg["title"], seg["order"]))
|
|
else:
|
|
philosopher_bookmarks.append((pos, title))
|
|
|
|
print(f"\n Intro bookmarks: {len(intro_bookmark_list)}")
|
|
for pos, slug, title, order in intro_bookmark_list:
|
|
print(f" [{order}] {slug}: {title} (paragraph {pos})")
|
|
|
|
print(f" Philosopher bookmarks: {len(philosopher_bookmarks)}")
|
|
|
|
# --- Step 5: Determine parent chapter for each philosopher ---
|
|
# Build chapter boundary lookup: ch_num → (start_pos, end_pos)
|
|
ch_bm_lookup = {}
|
|
ch_nums_sorted = sorted(set(ch_num for _, ch_num, _ in chapter_bookmarks))
|
|
|
|
for i, (pos, ch_num, raw_title) in enumerate(chapter_bookmarks):
|
|
ch_bm_lookup[ch_num] = pos
|
|
|
|
ch_end_lookup = {}
|
|
for idx, ch_num in enumerate(ch_nums_sorted):
|
|
if idx + 1 < len(ch_nums_sorted):
|
|
next_ch = ch_nums_sorted[idx + 1]
|
|
ch_end_lookup[ch_num] = ch_bm_lookup[next_ch]
|
|
else:
|
|
ch_end_lookup[ch_num] = ds_end
|
|
|
|
# Assign each philosopher to its parent chapter based on position
|
|
def find_parent_chapter(pos: int) -> int:
|
|
"""Find which parent chapter a position falls under."""
|
|
result_ch = None
|
|
for ch_num in ch_nums_sorted:
|
|
ch_start = ch_bm_lookup[ch_num]
|
|
if pos >= ch_start:
|
|
result_ch = ch_num
|
|
else:
|
|
break
|
|
return result_ch
|
|
|
|
philosophers_by_parent = defaultdict(list)
|
|
for pos, title in philosopher_bookmarks:
|
|
parent = find_parent_chapter(pos)
|
|
if parent is not None:
|
|
philosophers_by_parent[parent].append((pos, title))
|
|
else:
|
|
print(f" ⚠ Could not find parent chapter for: {title} at paragraph {pos}")
|
|
|
|
# --- Step 6: Extract front pages ---
|
|
intro_segments = []
|
|
|
|
# Front pages: from DS header to first intro bookmark (DS:Forward)
|
|
first_intro_pos = (
|
|
intro_bookmark_list[0][0]
|
|
if intro_bookmark_list
|
|
else (chapter_bookmarks[0][0] if chapter_bookmarks else ds_end)
|
|
)
|
|
|
|
front_pages_paragraphs = extract_paragraphs(
|
|
all_paragraphs,
|
|
ds_start,
|
|
first_intro_pos,
|
|
is_front_matter=True,
|
|
)
|
|
intro_segments.append(
|
|
{
|
|
"slug": "front-pages",
|
|
"title": "Front Pages",
|
|
"type": "intro-segment",
|
|
"book": "ds",
|
|
"order": 0,
|
|
"paragraphs": front_pages_paragraphs,
|
|
}
|
|
)
|
|
|
|
# --- Step 7: Extract intro segments (Foreword, Preface) ---
|
|
# Build boundary list for intro segments
|
|
intro_boundaries = []
|
|
for pos, slug, title, order in intro_bookmark_list:
|
|
intro_boundaries.append((pos, slug, title, order))
|
|
|
|
# Sort by position
|
|
intro_boundaries.sort(key=lambda x: x[0])
|
|
|
|
# First chapter boundary
|
|
first_chapter_pos = chapter_bookmarks[0][0] if chapter_bookmarks else ds_end
|
|
|
|
for bi in range(len(intro_boundaries)):
|
|
b_pos, b_slug, b_title, b_order = intro_boundaries[bi]
|
|
|
|
content_start = b_pos + 1 # skip the bookmark tag itself
|
|
if bi + 1 < len(intro_boundaries):
|
|
content_end = intro_boundaries[bi + 1][0]
|
|
else:
|
|
content_end = first_chapter_pos
|
|
|
|
paragraphs = extract_paragraphs(
|
|
all_paragraphs,
|
|
content_start,
|
|
content_end,
|
|
skip_p27=(b_slug == "preface"), # skip "Preface" p27 heading
|
|
)
|
|
|
|
intro_segments.append(
|
|
{
|
|
"slug": b_slug,
|
|
"title": b_title,
|
|
"type": "intro-segment",
|
|
"book": "ds",
|
|
"order": b_order,
|
|
"paragraphs": paragraphs,
|
|
}
|
|
)
|
|
|
|
# --- Step 8: Extract sub-chapters (philosophers) ---
|
|
sub_chapters = []
|
|
parent_chapters = []
|
|
|
|
for ch_num in ch_nums_sorted:
|
|
parent_title = CLEAN_PARENT_TITLES.get(ch_num, f"Chapter {ch_num}")
|
|
parent_end = ch_end_lookup[ch_num]
|
|
|
|
philosophers = philosophers_by_parent.get(ch_num, [])
|
|
# Sort by position
|
|
philosophers.sort(key=lambda x: x[0])
|
|
|
|
sub_entries = [] # for intro.json
|
|
sub_counter = 0
|
|
|
|
for pi, (phil_pos, phil_title) in enumerate(philosophers):
|
|
sub_counter += 1
|
|
slug = f"{ch_num}-{sub_counter}"
|
|
|
|
# Decode the title with ScaGoudy
|
|
decoded_title = decode_scagoudy(phil_title)
|
|
|
|
# Content range: from bookmark to next philosopher (or parent chapter end)
|
|
content_start = phil_pos # extract_paragraphs will skip bookmark internally
|
|
if pi + 1 < len(philosophers):
|
|
content_end = philosophers[pi + 1][0]
|
|
else:
|
|
content_end = parent_end
|
|
|
|
# For the first philosopher in a chapter, also skip p27 chapter heading
|
|
skip_p27_flag = True # Always skip p27 headings
|
|
|
|
paragraphs = extract_paragraphs(
|
|
all_paragraphs,
|
|
content_start,
|
|
content_end,
|
|
skip_p27=skip_p27_flag,
|
|
skip_first_p31_matching=decoded_title,
|
|
)
|
|
|
|
sub_chapters.append(
|
|
{
|
|
"book": "ds",
|
|
"chapter": slug,
|
|
"parentChapter": ch_num,
|
|
"subChapter": sub_counter,
|
|
"title": decoded_title,
|
|
"parentTitle": parent_title,
|
|
"type": "chapter",
|
|
"paragraphs": paragraphs,
|
|
}
|
|
)
|
|
sub_entries.append({"slug": slug, "title": decoded_title})
|
|
|
|
parent_chapters.append(
|
|
{
|
|
"number": ch_num,
|
|
"title": parent_title,
|
|
"subChapters": sub_entries,
|
|
}
|
|
)
|
|
|
|
print(
|
|
f"\n Parent Ch {ch_num}: {parent_title} → {len(sub_entries)} sub-chapters"
|
|
)
|
|
for entry in sub_entries:
|
|
matching = [sc for sc in sub_chapters if sc["chapter"] == entry["slug"]]
|
|
para_count = len(matching[0]["paragraphs"]) if matching else 0
|
|
print(
|
|
f" {entry['slug']:>5}: {para_count:>3} paragraphs — {entry['title']}"
|
|
)
|
|
|
|
return {
|
|
"intro_segments": intro_segments,
|
|
"sub_chapters": sub_chapters,
|
|
"parent_chapters": parent_chapters,
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Output helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def write_json(data, filepath):
|
|
"""Write data as formatted JSON to filepath, creating directories as needed."""
|
|
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
|
with open(filepath, "w", encoding="utf-8") as f:
|
|
json.dump(data, f, ensure_ascii=False, indent=2)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def main():
|
|
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
html_path = os.path.join(base_dir, "public", "html", "other.html")
|
|
output_base = os.path.join(base_dir, "lib", "jsons", "books", "ds")
|
|
|
|
if not os.path.exists(html_path):
|
|
print(f"ERROR: {html_path} not found!")
|
|
sys.exit(1)
|
|
|
|
print(f"Reading {html_path}...")
|
|
with open(html_path, "r", encoding="utf-8") as f:
|
|
soup = BeautifulSoup(f, "html.parser")
|
|
|
|
print("Extracting Dialectic Spiritualism (1985)...\n")
|
|
result = extract_ds(soup)
|
|
|
|
intro_segments = result["intro_segments"]
|
|
sub_chapters = result["sub_chapters"]
|
|
parent_chapters = result["parent_chapters"]
|
|
|
|
# --- Write intro.json ---
|
|
intro_meta = {
|
|
"book": "ds",
|
|
"title": "Dialectic Spiritualism",
|
|
"subtitle": "A Vedic View of Western Philosophy",
|
|
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
|
"year": 1985,
|
|
"description": "Śrīla Prabhupāda's analysis of Western philosophers from Socrates to Sartre, presenting the Vedic perspective on Western philosophy.",
|
|
"totalChapters": 13,
|
|
"chapters": parent_chapters,
|
|
}
|
|
intro_path = os.path.join(output_base, "intro.json")
|
|
write_json(intro_meta, intro_path)
|
|
print(f"\n Written: {intro_path}")
|
|
|
|
# --- Write intro segments ---
|
|
print(f"\n--- Intro Segments ---")
|
|
for seg in intro_segments:
|
|
filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
|
|
write_json(seg, filepath)
|
|
print(
|
|
f" {seg['order']:>2}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}"
|
|
)
|
|
|
|
# --- Write sub-chapter files ---
|
|
print(f"\n--- Sub-Chapters ---")
|
|
for sc in sub_chapters:
|
|
slug = sc["chapter"]
|
|
filepath = os.path.join(output_base, "chapters", f"{slug}.json")
|
|
write_json(sc, filepath)
|
|
print(
|
|
f" {slug:>5}: {len(sc['paragraphs']):>3} paragraphs — "
|
|
f"[Ch {sc['parentChapter']}] {sc['title']}"
|
|
)
|
|
|
|
# --- Summary ---
|
|
total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments)
|
|
total_chapter_paras = sum(len(sc["paragraphs"]) for sc in sub_chapters)
|
|
total_paras = total_intro_paras + total_chapter_paras
|
|
|
|
print(f"\n{'=' * 60}")
|
|
print(f" EXTRACTION SUMMARY")
|
|
print(f"{'=' * 60}")
|
|
print(f" Intro segments: {len(intro_segments)}")
|
|
print(f" Parent chapters: {len(parent_chapters)}")
|
|
print(f" Sub-chapters: {len(sub_chapters)}")
|
|
print(f" Total paragraphs: {total_paras}")
|
|
print(f" Intro: {total_intro_paras}")
|
|
print(f" Chapters: {total_chapter_paras}")
|
|
if sub_chapters:
|
|
print(f" Avg paras/sub-ch: {total_chapter_paras / len(sub_chapters):.1f}")
|
|
print(f" Output directory: {output_base}/")
|
|
|
|
# --- Per-parent breakdown ---
|
|
print(f"\n--- Parent Chapter Breakdown ---")
|
|
for pc in parent_chapters:
|
|
ch_num = pc["number"]
|
|
sub_count = len(pc["subChapters"])
|
|
ch_paras = sum(
|
|
len(sc["paragraphs"])
|
|
for sc in sub_chapters
|
|
if sc["parentChapter"] == ch_num
|
|
)
|
|
print(
|
|
f" Ch {ch_num:>2}: {sub_count:>2} sub-chapters, {ch_paras:>4} paragraphs — {pc['title']}"
|
|
)
|
|
|
|
# --- Samples ---
|
|
print(f"\n--- Sample: Front Pages ---")
|
|
if intro_segments:
|
|
front = intro_segments[0]
|
|
for i, p in enumerate(front["paragraphs"][:5]):
|
|
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
|
print(f" [{i}] {preview}...")
|
|
|
|
if len(intro_segments) > 1:
|
|
print(f"\n--- Sample: {intro_segments[1]['title']} ---")
|
|
seg = intro_segments[1]
|
|
for i, p in enumerate(seg["paragraphs"][:5]):
|
|
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
|
print(f" [{i}] {preview}...")
|
|
|
|
if sub_chapters:
|
|
first_sc = sub_chapters[0]
|
|
print(f"\n--- Sample: {first_sc['chapter']} ({first_sc['title']}) ---")
|
|
for i, p in enumerate(first_sc["paragraphs"][:5]):
|
|
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
|
print(f" [{i}] {preview}...")
|
|
|
|
if len(sub_chapters) > 1:
|
|
last_sc = sub_chapters[-1]
|
|
print(f"\n--- Sample: {last_sc['chapter']} ({last_sc['title']}) ---")
|
|
for i, p in enumerate(last_sc["paragraphs"][:5]):
|
|
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
|
print(f" [{i}] {preview}...")
|
|
|
|
# --- Quality Checks ---
|
|
print(f"\n--- Quality Checks ---")
|
|
|
|
# Check all 13 parent chapters present
|
|
expected_parents = set(range(1, 14))
|
|
actual_parents = {pc["number"] for pc in parent_chapters}
|
|
missing = expected_parents - actual_parents
|
|
extra = actual_parents - expected_parents
|
|
if missing:
|
|
print(f" ⚠ Missing parent chapters: {sorted(missing)}")
|
|
if extra:
|
|
print(f" ⚠ Extra parent chapters: {sorted(extra)}")
|
|
if not missing and not extra:
|
|
print(f" ✅ All 13 parent chapters present")
|
|
|
|
# Check all 3 intro segments present
|
|
expected_intros = {"front-pages", "foreword", "preface"}
|
|
actual_intros = {s["slug"] for s in intro_segments}
|
|
missing_intros = expected_intros - actual_intros
|
|
extra_intros = actual_intros - expected_intros
|
|
if missing_intros:
|
|
print(f" ⚠ Missing intro segments: {missing_intros}")
|
|
if extra_intros:
|
|
print(f" ⚠ Extra intro segments: {extra_intros}")
|
|
if not missing_intros and not extra_intros:
|
|
print(f" ✅ All 3 intro segments present")
|
|
|
|
# Check 38 philosopher sub-chapters
|
|
if len(sub_chapters) == 38:
|
|
print(f" ✅ All 38 philosopher sub-chapters extracted")
|
|
else:
|
|
print(f" ⚠ Expected 38 sub-chapters, got {len(sub_chapters)}")
|
|
|
|
# Check empty sub-chapters
|
|
empty_scs = [sc["chapter"] for sc in sub_chapters if len(sc["paragraphs"]) == 0]
|
|
if empty_scs:
|
|
print(f" ⚠ Empty sub-chapters (0 paragraphs): {empty_scs}")
|
|
else:
|
|
print(f" ✅ No empty sub-chapters")
|
|
|
|
# Check empty intro segments
|
|
empty_intros = [s["slug"] for s in intro_segments if len(s["paragraphs"]) == 0]
|
|
if empty_intros:
|
|
print(f" ⚠ Empty intro segments: {empty_intros}")
|
|
else:
|
|
print(f" ✅ No empty intro segments")
|
|
|
|
# Check for duplicate headings in first 3 paragraphs of each sub-chapter
|
|
print(f"\n--- Duplicate Heading Check ---")
|
|
dup_count = 0
|
|
for sc in sub_chapters:
|
|
title_norm = _normalize_for_match(sc["title"])
|
|
for p in sc["paragraphs"][:3]:
|
|
plain = re.sub(r"<[^>]+>", "", p).strip()
|
|
plain_norm = _normalize_for_match(plain)
|
|
if plain_norm == title_norm and len(plain) < 150:
|
|
print(f" ⚠ Duplicate heading in {sc['chapter']}: '{plain[:60]}'")
|
|
dup_count += 1
|
|
break
|
|
if dup_count == 0:
|
|
print(f" ✅ No duplicate headings in first 3 paragraphs")
|
|
else:
|
|
print(f" ⚠ {dup_count} sub-chapters have duplicate headings")
|
|
|
|
# Check for Roman numeral parent headings leaking into content
|
|
print(f"\n--- Parent Heading Leak Check ---")
|
|
roman_pattern = re.compile(
|
|
r"^(I{1,3}|IV|V|VI{0,3}|VII|VIII|IX|X|XI{0,3}|XII|XIII)\s*[A-Z]"
|
|
)
|
|
leak_count = 0
|
|
for sc in sub_chapters:
|
|
for p in sc["paragraphs"][:2]:
|
|
plain = re.sub(r"<[^>]+>", "", p).strip()
|
|
if roman_pattern.match(plain):
|
|
print(f" ⚠ Roman numeral heading in {sc['chapter']}: '{plain[:60]}'")
|
|
leak_count += 1
|
|
break
|
|
if leak_count == 0:
|
|
print(f" ✅ No Roman numeral parent headings in sub-chapter content")
|
|
|
|
# Check ScaGoudy decoding
|
|
print(f"\n--- ScaGoudy Decoding Check ---")
|
|
all_text = ""
|
|
for sc in sub_chapters:
|
|
all_text += " ".join(sc["paragraphs"][:20])
|
|
for seg in intro_segments:
|
|
all_text += " ".join(seg["paragraphs"][:10])
|
|
|
|
if "Kṛṣṇa" in all_text:
|
|
print(f" ✅ 'Kṛṣṇa' found (ScaGoudy properly decoded)")
|
|
else:
|
|
print(f" ⚠ 'Kṛṣṇa' not found — ScaGoudy decoding may have failed")
|
|
|
|
if "Bhagavad-gītā" in all_text or "gītā" in all_text or "Gītā" in all_text:
|
|
print(f" ✅ 'gītā' found (ScaGoudy properly decoded)")
|
|
else:
|
|
print(f" ⚠ 'Gītā' not found — ScaGoudy decoding may have failed")
|
|
|
|
if "Prabhupāda" in all_text:
|
|
print(f" ✅ 'Prabhupāda' found (ScaGoudy properly decoded)")
|
|
|
|
# Check for un-decoded ScaGoudy remnants
|
|
scagoudy_remnants = 0
|
|
for sc in sub_chapters:
|
|
for p in sc["paragraphs"]:
|
|
if "Kåñëa" in p or "Prabhupäda" in p:
|
|
scagoudy_remnants += 1
|
|
for seg in intro_segments:
|
|
for p in seg["paragraphs"]:
|
|
if "Kåñëa" in p or "Prabhupäda" in p:
|
|
scagoudy_remnants += 1
|
|
if scagoudy_remnants == 0:
|
|
print(f" ✅ No ScaGoudy remnants detected")
|
|
else:
|
|
print(f" ⚠ {scagoudy_remnants} paragraphs with ScaGoudy remnants")
|
|
|
|
# Check for centered verse blocks
|
|
verse_block_count = 0
|
|
for sc in sub_chapters:
|
|
for p in sc["paragraphs"]:
|
|
if '<div style="text-align:center"><i>' in p:
|
|
verse_block_count += 1
|
|
if verse_block_count > 0:
|
|
print(f" ✅ {verse_block_count} centered Sanskrit verse blocks found")
|
|
else:
|
|
print(f" ⚠ No centered verse blocks found")
|
|
|
|
# Check for residual span tags
|
|
span_count = 0
|
|
for sc in sub_chapters:
|
|
for p in sc["paragraphs"]:
|
|
if 'class="' in p and "<span" in p:
|
|
span_count += 1
|
|
for seg in intro_segments:
|
|
for p in seg["paragraphs"]:
|
|
if 'class="' in p and "<span" in p:
|
|
span_count += 1
|
|
if span_count == 0:
|
|
print(f" ✅ No residual span class artifacts found")
|
|
else:
|
|
print(f" ⚠ {span_count} paragraphs still contain span class artifacts")
|
|
|
|
# Check for dash-line paragraphs
|
|
dash_count = 0
|
|
for sc in sub_chapters:
|
|
for p in sc["paragraphs"]:
|
|
plain = re.sub(r"<[^>]+>", "", p)
|
|
if is_dash_line(plain):
|
|
dash_count += 1
|
|
for seg in intro_segments:
|
|
for p in seg["paragraphs"]:
|
|
plain = re.sub(r"<[^>]+>", "", p)
|
|
if is_dash_line(plain):
|
|
dash_count += 1
|
|
if dash_count == 0:
|
|
print(f" ✅ No dashed-line paragraphs in output")
|
|
else:
|
|
print(f" ⚠ {dash_count} dashed-line paragraphs found in output")
|
|
|
|
# Show a verse block sample
|
|
print(f"\n--- Sample: Verse Block ---")
|
|
for sc in sub_chapters:
|
|
for p in sc["paragraphs"]:
|
|
if '<div style="text-align:center"><i>' in p:
|
|
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
|
print(f" {preview}...")
|
|
break
|
|
else:
|
|
continue
|
|
break
|
|
|
|
# Sub-chapter size check
|
|
print(f"\n--- Sub-Chapter Size Check ---")
|
|
very_small = []
|
|
very_large = []
|
|
for sc in sub_chapters:
|
|
count = len(sc["paragraphs"])
|
|
if count < 2:
|
|
very_small.append((sc["chapter"], count, sc["title"]))
|
|
elif count > 500:
|
|
very_large.append((sc["chapter"], count, sc["title"]))
|
|
if very_small:
|
|
print(f" ⚠ Very small sub-chapters (< 2 paragraphs):")
|
|
for slug, count, title in very_small:
|
|
print(f" {slug}: {count} paragraphs — {title}")
|
|
else:
|
|
print(f" ✅ All sub-chapters have ≥ 2 paragraphs")
|
|
if very_large:
|
|
print(f" ⚠ Very large sub-chapters (> 500 paragraphs):")
|
|
for slug, count, title in very_large:
|
|
print(f" {slug}: {count} paragraphs — {title}")
|
|
else:
|
|
print(f" ✅ All sub-chapters have ≤ 500 paragraphs")
|
|
|
|
# Sub-chapter title decoding check
|
|
print(f"\n--- Sub-Chapter Title Decoding Check ---")
|
|
bad_titles = 0
|
|
for sc in sub_chapters:
|
|
title = sc["title"]
|
|
if "å" in title or "ë" in title or "ñ" in title or "ä" in title:
|
|
print(f" ⚠ {sc['chapter']} title has un-decoded ScaGoudy: {title}")
|
|
bad_titles += 1
|
|
if bad_titles == 0:
|
|
print(f" ✅ All {len(sub_chapters)} sub-chapter titles properly decoded")
|
|
|
|
# Verify slug numbering continuity within each parent
|
|
print(f"\n--- Slug Numbering Check ---")
|
|
for pc in parent_chapters:
|
|
ch_num = pc["number"]
|
|
slugs = [s["slug"] for s in pc["subChapters"]]
|
|
expected_slugs = [f"{ch_num}-{i + 1}" for i in range(len(slugs))]
|
|
if slugs == expected_slugs:
|
|
print(
|
|
f" ✅ Ch {ch_num:>2}: {len(slugs)} sub-chapters, slugs {slugs[0]}..{slugs[-1]}"
|
|
)
|
|
else:
|
|
print(
|
|
f" ⚠ Ch {ch_num:>2}: slug mismatch! expected {expected_slugs}, got {slugs}"
|
|
)
|
|
|
|
# List all philosopher names
|
|
print(f"\n--- All Philosophers ---")
|
|
for sc in sub_chapters:
|
|
print(f" {sc['chapter']:>5}: {sc['title']}")
|
|
|
|
print(f"\nDone! ✨")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|