446 lines
14 KiB
Python
446 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Extract Two Essays (1966) from other.html.
|
|
|
|
The book contains 2 essays:
|
|
1. "Krishna The Reservoir Of Pleasure"
|
|
2. "Who Is Crazy?"
|
|
|
|
These are plain English essays from 1966 — NO ScaGoudy encoding.
|
|
|
|
Output:
|
|
lib/jsons/books/te/intro.json — Book metadata
|
|
lib/jsons/books/te/essays/krishna-reservoir-of-pleasure.json
|
|
lib/jsons/books/te/essays/who-is-crazy.json
|
|
|
|
Usage:
|
|
python3 scripts/te_extract.py
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
from bs4 import BeautifulSoup, Tag
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def get_css_class(tag: Tag) -> str:
|
|
"""Get the first CSS class of a tag, or empty string."""
|
|
classes = tag.get("class", [])
|
|
return classes[0] if classes else ""
|
|
|
|
|
|
def is_empty(tag: Tag) -> bool:
|
|
"""Check if a tag is empty or whitespace-only."""
|
|
text = tag.get_text().strip()
|
|
return not text or text == "\xa0"
|
|
|
|
|
|
def clean_html_content(tag: Tag) -> str:
|
|
"""Extract inner HTML from a tag, strip span class artifacts, preserve <i>, <b>, <br>."""
|
|
inner = "".join(str(child) for child in tag.children)
|
|
# Remove span tags with CSS classes (RTF conversion artifacts)
|
|
inner = re.sub(
|
|
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
|
|
)
|
|
# Clean up Apple-tab-span leftovers (tab characters)
|
|
inner = inner.replace("\t", " ")
|
|
# Normalize whitespace (but preserve <br> tags)
|
|
inner = re.sub(r"\s+", " ", inner).strip()
|
|
return inner
|
|
|
|
|
|
def write_json(data, filepath):
|
|
"""Write data as formatted JSON to filepath, creating directories as needed."""
|
|
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
|
with open(filepath, "w", encoding="utf-8") as f:
|
|
json.dump(data, f, ensure_ascii=False, indent=2)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# CSS classes to SKIP entirely (ad block, structural headers, etc.)
|
|
# ---------------------------------------------------------------------------
|
|
SKIP_CLASSES = {"p12", "p121", "p122", "p13", "p123", "p124", "p23", "p18"}
|
|
|
|
# CSS classes that contain essay body content
|
|
BODY_CLASSES = {"p28", "p6", "p22", "p3"}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Boundary detection
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def find_te_boundaries(all_paragraphs: list[Tag]) -> tuple[int, int]:
|
|
"""Find the start and end paragraph indices for Two Essays content.
|
|
|
|
Start: <p class="p12"><b>TWO ESSAYS</b></p>
|
|
End: Next <p class="p12"> or <p class="p24"> or <p class="p34"> after our content,
|
|
or just past the last essay paragraph.
|
|
|
|
Returns (start_idx, end_idx) — exclusive end.
|
|
"""
|
|
te_start = None
|
|
te_end = None
|
|
|
|
for i, p in enumerate(all_paragraphs):
|
|
css = get_css_class(p)
|
|
text = p.get_text().strip().upper()
|
|
|
|
# Find the TWO ESSAYS header
|
|
if css == "p12" and "TWO ESSAYS" in text:
|
|
te_start = i
|
|
continue
|
|
|
|
# Once we've found the start, look for the end
|
|
if te_start is not None and te_end is None:
|
|
# A new p12 header means we've left Two Essays
|
|
if css == "p12" and i > te_start:
|
|
te_end = i
|
|
break
|
|
# Collier end markers
|
|
if css in ("p24", "p34"):
|
|
te_end = i
|
|
break
|
|
|
|
if te_start is None:
|
|
raise ValueError("Could not find TWO ESSAYS header in HTML")
|
|
|
|
# If we didn't find an explicit end marker, scan for the last body paragraph
|
|
if te_end is None:
|
|
# Walk backwards from the end to find the last relevant paragraph
|
|
for i in range(len(all_paragraphs) - 1, te_start, -1):
|
|
css = get_css_class(all_paragraphs[i])
|
|
if css in BODY_CLASSES:
|
|
te_end = i + 1
|
|
break
|
|
if te_end is None:
|
|
te_end = len(all_paragraphs)
|
|
|
|
return te_start, te_end
|
|
|
|
|
|
def find_essay_headings(
|
|
all_paragraphs: list[Tag], start: int, end: int
|
|
) -> list[tuple[int, str]]:
|
|
"""Find p27 headings within the Two Essays range.
|
|
|
|
Returns list of (paragraph_index, title_text).
|
|
"""
|
|
headings = []
|
|
for i in range(start, end):
|
|
p = all_paragraphs[i]
|
|
css = get_css_class(p)
|
|
if css == "p27":
|
|
title = p.get_text().strip()
|
|
headings.append((i, title))
|
|
return headings
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Paragraph extraction
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def extract_essay_paragraphs(
|
|
all_paragraphs: list[Tag], start: int, end: int
|
|
) -> list[str]:
|
|
"""Extract body paragraphs from the given range.
|
|
|
|
Includes: p28, p6, p22, p3
|
|
Skips: SKIP_CLASSES, p27 (headings), empty paragraphs
|
|
"""
|
|
paragraphs = []
|
|
|
|
for i in range(start, end):
|
|
p = all_paragraphs[i]
|
|
css = get_css_class(p)
|
|
|
|
# Skip classes we don't want
|
|
if css in SKIP_CLASSES:
|
|
continue
|
|
|
|
# Skip headings (already handled)
|
|
if css == "p27":
|
|
continue
|
|
|
|
# Only accept body classes
|
|
if css not in BODY_CLASSES:
|
|
continue
|
|
|
|
# Skip empty paragraphs
|
|
if is_empty(p):
|
|
continue
|
|
|
|
html = clean_html_content(p)
|
|
if html and html != " ":
|
|
paragraphs.append(html)
|
|
|
|
return paragraphs
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main extraction
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def extract_te(soup: BeautifulSoup) -> dict:
|
|
"""Extract Two Essays from the parsed HTML.
|
|
|
|
Returns a dict with:
|
|
"essays": list of essay dicts
|
|
"essay_listing": list of {slug, title, order} for intro.json
|
|
"""
|
|
all_paragraphs = list(soup.find_all("p"))
|
|
|
|
# Step 1: Find Two Essays boundaries
|
|
te_start, te_end = find_te_boundaries(all_paragraphs)
|
|
print(f" Two Essays content: paragraphs {te_start} to {te_end}")
|
|
print(f" Total paragraphs in range: {te_end - te_start}")
|
|
|
|
# Step 2: Find essay headings (p27)
|
|
headings = find_essay_headings(all_paragraphs, te_start, te_end)
|
|
print(f" Found {len(headings)} essay headings")
|
|
for pos, title in headings:
|
|
print(f" paragraph {pos}: {title}")
|
|
|
|
if len(headings) != 2:
|
|
print(f" WARNING: Expected 2 essay headings, found {len(headings)}")
|
|
|
|
# Step 3: Extract essays
|
|
essays = []
|
|
|
|
essay_configs = [
|
|
{
|
|
"slug": "krishna-reservoir-of-pleasure",
|
|
"title": "Krishna The Reservoir Of Pleasure",
|
|
"order": 1,
|
|
},
|
|
{
|
|
"slug": "who-is-crazy",
|
|
"title": "Who Is Crazy?",
|
|
"order": 2,
|
|
},
|
|
]
|
|
|
|
for idx, (heading_pos, heading_title) in enumerate(headings):
|
|
config = essay_configs[idx] if idx < len(essay_configs) else None
|
|
|
|
# Content starts after the heading
|
|
content_start = heading_pos + 1
|
|
|
|
# Content ends at next heading or te_end
|
|
if idx + 1 < len(headings):
|
|
content_end = headings[idx + 1][0]
|
|
else:
|
|
content_end = te_end
|
|
|
|
paragraphs = extract_essay_paragraphs(
|
|
all_paragraphs, content_start, content_end
|
|
)
|
|
|
|
if config:
|
|
slug = config["slug"]
|
|
title = config["title"]
|
|
order = config["order"]
|
|
else:
|
|
# Fallback for unexpected essays
|
|
slug = re.sub(r"[^a-z0-9]+", "-", heading_title.lower()).strip("-")
|
|
title = heading_title
|
|
order = idx + 1
|
|
|
|
essay = {
|
|
"slug": slug,
|
|
"title": title,
|
|
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
|
"year": 1966,
|
|
"group": "two-essays",
|
|
"order": order,
|
|
"paragraphs": paragraphs,
|
|
}
|
|
essays.append(essay)
|
|
|
|
# Step 4: Build essay listing for intro.json
|
|
essay_listing = [
|
|
{"slug": e["slug"], "title": e["title"], "order": e["order"]} for e in essays
|
|
]
|
|
|
|
return {
|
|
"essays": essays,
|
|
"essay_listing": essay_listing,
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def main():
|
|
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
html_path = os.path.join(base_dir, "public", "html", "other.html")
|
|
output_base = os.path.join(base_dir, "lib", "jsons", "books", "te")
|
|
|
|
if not os.path.exists(html_path):
|
|
print(f"ERROR: {html_path} not found!")
|
|
sys.exit(1)
|
|
|
|
print(f"Reading {html_path}...")
|
|
with open(html_path, "r", encoding="utf-8") as f:
|
|
soup = BeautifulSoup(f, "html.parser")
|
|
|
|
print("Extracting Two Essays (1966)...\n")
|
|
result = extract_te(soup)
|
|
|
|
essays = result["essays"]
|
|
essay_listing = result["essay_listing"]
|
|
|
|
# --- Write intro.json ---
|
|
intro_meta = {
|
|
"group": "two-essays",
|
|
"title": "Two Essays",
|
|
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
|
"year": 1966,
|
|
"description": "Two foundational essays: 'Krishna The Reservoir Of Pleasure' and 'Who Is Crazy?'",
|
|
"essays": essay_listing,
|
|
}
|
|
intro_path = os.path.join(output_base, "intro.json")
|
|
write_json(intro_meta, intro_path)
|
|
print(f"\n Written: {intro_path}")
|
|
|
|
# --- Write essay files ---
|
|
print(f"\n--- Essays ---")
|
|
for essay in essays:
|
|
filepath = os.path.join(output_base, "essays", f"{essay['slug']}.json")
|
|
write_json(essay, filepath)
|
|
print(
|
|
f" {essay['order']}. {essay['title']}: "
|
|
f"{len(essay['paragraphs'])} paragraphs → {filepath}"
|
|
)
|
|
|
|
# --- Summary ---
|
|
total_paras = sum(len(e["paragraphs"]) for e in essays)
|
|
|
|
print(f"\n{'=' * 60}")
|
|
print(f" EXTRACTION SUMMARY")
|
|
print(f"{'=' * 60}")
|
|
print(f" Essays: {len(essays)}")
|
|
print(f" Total paragraphs: {total_paras}")
|
|
for e in essays:
|
|
print(f" {e['order']}. {e['title']}: {len(e['paragraphs'])} paragraphs")
|
|
if essays:
|
|
print(f" Avg paras/essay: {total_paras / len(essays):.1f}")
|
|
print(f" Output directory: {output_base}/")
|
|
|
|
# --- Samples ---
|
|
for essay in essays:
|
|
print(f"\n--- Sample: {essay['title']} ---")
|
|
for i, p in enumerate(essay["paragraphs"][:3]):
|
|
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
|
print(f" [{i}] {preview}...")
|
|
|
|
# --- Quality Checks ---
|
|
print(f"\n--- Quality Checks ---")
|
|
|
|
# Check both essays present
|
|
expected_slugs = {"krishna-reservoir-of-pleasure", "who-is-crazy"}
|
|
actual_slugs = {e["slug"] for e in essays}
|
|
missing = expected_slugs - actual_slugs
|
|
extra = actual_slugs - expected_slugs
|
|
if missing:
|
|
print(f" ⚠ Missing essays: {missing}")
|
|
if extra:
|
|
print(f" ⚠ Extra essays: {extra}")
|
|
if not missing and not extra:
|
|
print(f" ✅ Both essays present")
|
|
|
|
# Check empty essays
|
|
empty_essays = [e["slug"] for e in essays if len(e["paragraphs"]) == 0]
|
|
if empty_essays:
|
|
print(f" ⚠ Empty essays (0 paragraphs): {empty_essays}")
|
|
else:
|
|
print(f" ✅ No empty essays")
|
|
|
|
# Check paragraph counts are reasonable
|
|
for e in essays:
|
|
count = len(e["paragraphs"])
|
|
if count < 5:
|
|
print(
|
|
f" ⚠ Essay '{e['slug']}' has only {count} paragraphs (suspiciously low)"
|
|
)
|
|
elif count > 100:
|
|
print(f" ⚠ Essay '{e['slug']}' has {count} paragraphs (suspiciously high)")
|
|
else:
|
|
print(f" ✅ Essay '{e['slug']}' has {count} paragraphs (reasonable)")
|
|
|
|
# Check for residual span class artifacts
|
|
span_count = 0
|
|
for e in essays:
|
|
for p in e["paragraphs"]:
|
|
if 'class="' in p and "<span" in p:
|
|
span_count += 1
|
|
if span_count == 0:
|
|
print(f" ✅ No residual span class artifacts found")
|
|
else:
|
|
print(f" ⚠ {span_count} paragraphs still contain span class artifacts")
|
|
|
|
# Check no ScaGoudy encoding present (these are plain English essays)
|
|
# But verify key words appear as plain English (not IAST)
|
|
all_text = " ".join(" ".join(e["paragraphs"][:10]) for e in essays)
|
|
if "Krishna" in all_text:
|
|
print(f" ✅ 'Krishna' found (plain English, as expected for 1966)")
|
|
else:
|
|
print(f" ⚠ 'Krishna' not found in text")
|
|
|
|
if "Bhagavad Gita" in all_text or "Bhagavad-Gita" in all_text:
|
|
print(f" ✅ 'Bhagavad Gita' found (plain English)")
|
|
|
|
# Check for ad content leaking in
|
|
ad_markers = ["ISKCON", "26 2nd Ave", "SRIMAD-BHAGAWATAM", "$3.25", "$16.80"]
|
|
leaked_ads = []
|
|
for e in essays:
|
|
for p in e["paragraphs"]:
|
|
plain = re.sub(r"<[^>]+>", "", p)
|
|
for marker in ad_markers:
|
|
if marker in plain:
|
|
leaked_ads.append((e["slug"], marker, plain[:80]))
|
|
break
|
|
if leaked_ads:
|
|
print(f" ⚠ {len(leaked_ads)} paragraphs may contain ad content:")
|
|
for slug, marker, preview in leaked_ads[:3]:
|
|
print(f" [{slug}] matched '{marker}': {preview}...")
|
|
else:
|
|
print(f" ✅ No ad content detected in essays")
|
|
|
|
# Check for SKIP_CLASSES content leaking in
|
|
# (p18 paragraphs contain ad text like "BLISS—what was once only a word...")
|
|
bliss_check = False
|
|
for e in essays:
|
|
for p in e["paragraphs"]:
|
|
plain = re.sub(r"<[^>]+>", "", p)
|
|
if plain.startswith("BLISS") or "SOUNDS OF KRISHNA CONSCIOUSNESS" in plain:
|
|
bliss_check = True
|
|
break
|
|
if bliss_check:
|
|
print(f" ⚠ Ad text ('BLISS...') leaked into essay paragraphs")
|
|
else:
|
|
print(f" ✅ Ad text properly excluded")
|
|
|
|
# Show last paragraph of each essay
|
|
print(f"\n--- Last Paragraph of Each Essay ---")
|
|
for e in essays:
|
|
if e["paragraphs"]:
|
|
last = re.sub(r"<[^>]+>", "", e["paragraphs"][-1])[:150]
|
|
print(f" {e['title']}:")
|
|
print(f" ...{last}")
|
|
|
|
print(f"\nDone! ✨")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|