feat: add ssr
This commit is contained in:
gosangam
@@ -0,0 +1,445 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract Two Essays (1966) from other.html.
|
||||
|
||||
The book contains 2 essays:
|
||||
1. "Krishna The Reservoir Of Pleasure"
|
||||
2. "Who Is Crazy?"
|
||||
|
||||
These are plain English essays from 1966 — NO ScaGoudy encoding.
|
||||
|
||||
Output:
|
||||
lib/jsons/books/te/intro.json — Book metadata
|
||||
lib/jsons/books/te/essays/krishna-reservoir-of-pleasure.json
|
||||
lib/jsons/books/te/essays/who-is-crazy.json
|
||||
|
||||
Usage:
|
||||
python3 scripts/te_extract.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def get_css_class(tag: Tag) -> str:
|
||||
"""Get the first CSS class of a tag, or empty string."""
|
||||
classes = tag.get("class", [])
|
||||
return classes[0] if classes else ""
|
||||
|
||||
|
||||
def is_empty(tag: Tag) -> bool:
|
||||
"""Check if a tag is empty or whitespace-only."""
|
||||
text = tag.get_text().strip()
|
||||
return not text or text == "\xa0"
|
||||
|
||||
|
||||
def clean_html_content(tag: Tag) -> str:
|
||||
"""Extract inner HTML from a tag, strip span class artifacts, preserve <i>, <b>, <br>."""
|
||||
inner = "".join(str(child) for child in tag.children)
|
||||
# Remove span tags with CSS classes (RTF conversion artifacts)
|
||||
inner = re.sub(
|
||||
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
|
||||
)
|
||||
# Clean up Apple-tab-span leftovers (tab characters)
|
||||
inner = inner.replace("\t", " ")
|
||||
# Normalize whitespace (but preserve <br> tags)
|
||||
inner = re.sub(r"\s+", " ", inner).strip()
|
||||
return inner
|
||||
|
||||
|
||||
def write_json(data, filepath):
|
||||
"""Write data as formatted JSON to filepath, creating directories as needed."""
|
||||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||||
with open(filepath, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CSS classes to SKIP entirely (ad block, structural headers, etc.)
|
||||
# ---------------------------------------------------------------------------
|
||||
SKIP_CLASSES = {"p12", "p121", "p122", "p13", "p123", "p124", "p23", "p18"}
|
||||
|
||||
# CSS classes that contain essay body content
|
||||
BODY_CLASSES = {"p28", "p6", "p22", "p3"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Boundary detection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def find_te_boundaries(all_paragraphs: list[Tag]) -> tuple[int, int]:
|
||||
"""Find the start and end paragraph indices for Two Essays content.
|
||||
|
||||
Start: <p class="p12"><b>TWO ESSAYS</b></p>
|
||||
End: Next <p class="p12"> or <p class="p24"> or <p class="p34"> after our content,
|
||||
or just past the last essay paragraph.
|
||||
|
||||
Returns (start_idx, end_idx) — exclusive end.
|
||||
"""
|
||||
te_start = None
|
||||
te_end = None
|
||||
|
||||
for i, p in enumerate(all_paragraphs):
|
||||
css = get_css_class(p)
|
||||
text = p.get_text().strip().upper()
|
||||
|
||||
# Find the TWO ESSAYS header
|
||||
if css == "p12" and "TWO ESSAYS" in text:
|
||||
te_start = i
|
||||
continue
|
||||
|
||||
# Once we've found the start, look for the end
|
||||
if te_start is not None and te_end is None:
|
||||
# A new p12 header means we've left Two Essays
|
||||
if css == "p12" and i > te_start:
|
||||
te_end = i
|
||||
break
|
||||
# Collier end markers
|
||||
if css in ("p24", "p34"):
|
||||
te_end = i
|
||||
break
|
||||
|
||||
if te_start is None:
|
||||
raise ValueError("Could not find TWO ESSAYS header in HTML")
|
||||
|
||||
# If we didn't find an explicit end marker, scan for the last body paragraph
|
||||
if te_end is None:
|
||||
# Walk backwards from the end to find the last relevant paragraph
|
||||
for i in range(len(all_paragraphs) - 1, te_start, -1):
|
||||
css = get_css_class(all_paragraphs[i])
|
||||
if css in BODY_CLASSES:
|
||||
te_end = i + 1
|
||||
break
|
||||
if te_end is None:
|
||||
te_end = len(all_paragraphs)
|
||||
|
||||
return te_start, te_end
|
||||
|
||||
|
||||
def find_essay_headings(
|
||||
all_paragraphs: list[Tag], start: int, end: int
|
||||
) -> list[tuple[int, str]]:
|
||||
"""Find p27 headings within the Two Essays range.
|
||||
|
||||
Returns list of (paragraph_index, title_text).
|
||||
"""
|
||||
headings = []
|
||||
for i in range(start, end):
|
||||
p = all_paragraphs[i]
|
||||
css = get_css_class(p)
|
||||
if css == "p27":
|
||||
title = p.get_text().strip()
|
||||
headings.append((i, title))
|
||||
return headings
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Paragraph extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def extract_essay_paragraphs(
|
||||
all_paragraphs: list[Tag], start: int, end: int
|
||||
) -> list[str]:
|
||||
"""Extract body paragraphs from the given range.
|
||||
|
||||
Includes: p28, p6, p22, p3
|
||||
Skips: SKIP_CLASSES, p27 (headings), empty paragraphs
|
||||
"""
|
||||
paragraphs = []
|
||||
|
||||
for i in range(start, end):
|
||||
p = all_paragraphs[i]
|
||||
css = get_css_class(p)
|
||||
|
||||
# Skip classes we don't want
|
||||
if css in SKIP_CLASSES:
|
||||
continue
|
||||
|
||||
# Skip headings (already handled)
|
||||
if css == "p27":
|
||||
continue
|
||||
|
||||
# Only accept body classes
|
||||
if css not in BODY_CLASSES:
|
||||
continue
|
||||
|
||||
# Skip empty paragraphs
|
||||
if is_empty(p):
|
||||
continue
|
||||
|
||||
html = clean_html_content(p)
|
||||
if html and html != " ":
|
||||
paragraphs.append(html)
|
||||
|
||||
return paragraphs
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def extract_te(soup: BeautifulSoup) -> dict:
|
||||
"""Extract Two Essays from the parsed HTML.
|
||||
|
||||
Returns a dict with:
|
||||
"essays": list of essay dicts
|
||||
"essay_listing": list of {slug, title, order} for intro.json
|
||||
"""
|
||||
all_paragraphs = list(soup.find_all("p"))
|
||||
|
||||
# Step 1: Find Two Essays boundaries
|
||||
te_start, te_end = find_te_boundaries(all_paragraphs)
|
||||
print(f" Two Essays content: paragraphs {te_start} to {te_end}")
|
||||
print(f" Total paragraphs in range: {te_end - te_start}")
|
||||
|
||||
# Step 2: Find essay headings (p27)
|
||||
headings = find_essay_headings(all_paragraphs, te_start, te_end)
|
||||
print(f" Found {len(headings)} essay headings")
|
||||
for pos, title in headings:
|
||||
print(f" paragraph {pos}: {title}")
|
||||
|
||||
if len(headings) != 2:
|
||||
print(f" WARNING: Expected 2 essay headings, found {len(headings)}")
|
||||
|
||||
# Step 3: Extract essays
|
||||
essays = []
|
||||
|
||||
essay_configs = [
|
||||
{
|
||||
"slug": "krishna-reservoir-of-pleasure",
|
||||
"title": "Krishna The Reservoir Of Pleasure",
|
||||
"order": 1,
|
||||
},
|
||||
{
|
||||
"slug": "who-is-crazy",
|
||||
"title": "Who Is Crazy?",
|
||||
"order": 2,
|
||||
},
|
||||
]
|
||||
|
||||
for idx, (heading_pos, heading_title) in enumerate(headings):
|
||||
config = essay_configs[idx] if idx < len(essay_configs) else None
|
||||
|
||||
# Content starts after the heading
|
||||
content_start = heading_pos + 1
|
||||
|
||||
# Content ends at next heading or te_end
|
||||
if idx + 1 < len(headings):
|
||||
content_end = headings[idx + 1][0]
|
||||
else:
|
||||
content_end = te_end
|
||||
|
||||
paragraphs = extract_essay_paragraphs(
|
||||
all_paragraphs, content_start, content_end
|
||||
)
|
||||
|
||||
if config:
|
||||
slug = config["slug"]
|
||||
title = config["title"]
|
||||
order = config["order"]
|
||||
else:
|
||||
# Fallback for unexpected essays
|
||||
slug = re.sub(r"[^a-z0-9]+", "-", heading_title.lower()).strip("-")
|
||||
title = heading_title
|
||||
order = idx + 1
|
||||
|
||||
essay = {
|
||||
"slug": slug,
|
||||
"title": title,
|
||||
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
||||
"year": 1966,
|
||||
"group": "two-essays",
|
||||
"order": order,
|
||||
"paragraphs": paragraphs,
|
||||
}
|
||||
essays.append(essay)
|
||||
|
||||
# Step 4: Build essay listing for intro.json
|
||||
essay_listing = [
|
||||
{"slug": e["slug"], "title": e["title"], "order": e["order"]} for e in essays
|
||||
]
|
||||
|
||||
return {
|
||||
"essays": essays,
|
||||
"essay_listing": essay_listing,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "other.html")
|
||||
output_base = os.path.join(base_dir, "lib", "jsons", "books", "te")
|
||||
|
||||
if not os.path.exists(html_path):
|
||||
print(f"ERROR: {html_path} not found!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Reading {html_path}...")
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
soup = BeautifulSoup(f, "html.parser")
|
||||
|
||||
print("Extracting Two Essays (1966)...\n")
|
||||
result = extract_te(soup)
|
||||
|
||||
essays = result["essays"]
|
||||
essay_listing = result["essay_listing"]
|
||||
|
||||
# --- Write intro.json ---
|
||||
intro_meta = {
|
||||
"group": "two-essays",
|
||||
"title": "Two Essays",
|
||||
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
||||
"year": 1966,
|
||||
"description": "Two foundational essays: 'Krishna The Reservoir Of Pleasure' and 'Who Is Crazy?'",
|
||||
"essays": essay_listing,
|
||||
}
|
||||
intro_path = os.path.join(output_base, "intro.json")
|
||||
write_json(intro_meta, intro_path)
|
||||
print(f"\n Written: {intro_path}")
|
||||
|
||||
# --- Write essay files ---
|
||||
print(f"\n--- Essays ---")
|
||||
for essay in essays:
|
||||
filepath = os.path.join(output_base, "essays", f"{essay['slug']}.json")
|
||||
write_json(essay, filepath)
|
||||
print(
|
||||
f" {essay['order']}. {essay['title']}: "
|
||||
f"{len(essay['paragraphs'])} paragraphs → {filepath}"
|
||||
)
|
||||
|
||||
# --- Summary ---
|
||||
total_paras = sum(len(e["paragraphs"]) for e in essays)
|
||||
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f" EXTRACTION SUMMARY")
|
||||
print(f"{'=' * 60}")
|
||||
print(f" Essays: {len(essays)}")
|
||||
print(f" Total paragraphs: {total_paras}")
|
||||
for e in essays:
|
||||
print(f" {e['order']}. {e['title']}: {len(e['paragraphs'])} paragraphs")
|
||||
if essays:
|
||||
print(f" Avg paras/essay: {total_paras / len(essays):.1f}")
|
||||
print(f" Output directory: {output_base}/")
|
||||
|
||||
# --- Samples ---
|
||||
for essay in essays:
|
||||
print(f"\n--- Sample: {essay['title']} ---")
|
||||
for i, p in enumerate(essay["paragraphs"][:3]):
|
||||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
||||
print(f" [{i}] {preview}...")
|
||||
|
||||
# --- Quality Checks ---
|
||||
print(f"\n--- Quality Checks ---")
|
||||
|
||||
# Check both essays present
|
||||
expected_slugs = {"krishna-reservoir-of-pleasure", "who-is-crazy"}
|
||||
actual_slugs = {e["slug"] for e in essays}
|
||||
missing = expected_slugs - actual_slugs
|
||||
extra = actual_slugs - expected_slugs
|
||||
if missing:
|
||||
print(f" ⚠ Missing essays: {missing}")
|
||||
if extra:
|
||||
print(f" ⚠ Extra essays: {extra}")
|
||||
if not missing and not extra:
|
||||
print(f" ✅ Both essays present")
|
||||
|
||||
# Check empty essays
|
||||
empty_essays = [e["slug"] for e in essays if len(e["paragraphs"]) == 0]
|
||||
if empty_essays:
|
||||
print(f" ⚠ Empty essays (0 paragraphs): {empty_essays}")
|
||||
else:
|
||||
print(f" ✅ No empty essays")
|
||||
|
||||
# Check paragraph counts are reasonable
|
||||
for e in essays:
|
||||
count = len(e["paragraphs"])
|
||||
if count < 5:
|
||||
print(
|
||||
f" ⚠ Essay '{e['slug']}' has only {count} paragraphs (suspiciously low)"
|
||||
)
|
||||
elif count > 100:
|
||||
print(f" ⚠ Essay '{e['slug']}' has {count} paragraphs (suspiciously high)")
|
||||
else:
|
||||
print(f" ✅ Essay '{e['slug']}' has {count} paragraphs (reasonable)")
|
||||
|
||||
# Check for residual span class artifacts
|
||||
span_count = 0
|
||||
for e in essays:
|
||||
for p in e["paragraphs"]:
|
||||
if 'class="' in p and "<span" in p:
|
||||
span_count += 1
|
||||
if span_count == 0:
|
||||
print(f" ✅ No residual span class artifacts found")
|
||||
else:
|
||||
print(f" ⚠ {span_count} paragraphs still contain span class artifacts")
|
||||
|
||||
# Check no ScaGoudy encoding present (these are plain English essays)
|
||||
# But verify key words appear as plain English (not IAST)
|
||||
all_text = " ".join(" ".join(e["paragraphs"][:10]) for e in essays)
|
||||
if "Krishna" in all_text:
|
||||
print(f" ✅ 'Krishna' found (plain English, as expected for 1966)")
|
||||
else:
|
||||
print(f" ⚠ 'Krishna' not found in text")
|
||||
|
||||
if "Bhagavad Gita" in all_text or "Bhagavad-Gita" in all_text:
|
||||
print(f" ✅ 'Bhagavad Gita' found (plain English)")
|
||||
|
||||
# Check for ad content leaking in
|
||||
ad_markers = ["ISKCON", "26 2nd Ave", "SRIMAD-BHAGAWATAM", "$3.25", "$16.80"]
|
||||
leaked_ads = []
|
||||
for e in essays:
|
||||
for p in e["paragraphs"]:
|
||||
plain = re.sub(r"<[^>]+>", "", p)
|
||||
for marker in ad_markers:
|
||||
if marker in plain:
|
||||
leaked_ads.append((e["slug"], marker, plain[:80]))
|
||||
break
|
||||
if leaked_ads:
|
||||
print(f" ⚠ {len(leaked_ads)} paragraphs may contain ad content:")
|
||||
for slug, marker, preview in leaked_ads[:3]:
|
||||
print(f" [{slug}] matched '{marker}': {preview}...")
|
||||
else:
|
||||
print(f" ✅ No ad content detected in essays")
|
||||
|
||||
# Check for SKIP_CLASSES content leaking in
|
||||
# (p18 paragraphs contain ad text like "BLISS—what was once only a word...")
|
||||
bliss_check = False
|
||||
for e in essays:
|
||||
for p in e["paragraphs"]:
|
||||
plain = re.sub(r"<[^>]+>", "", p)
|
||||
if plain.startswith("BLISS") or "SOUNDS OF KRISHNA CONSCIOUSNESS" in plain:
|
||||
bliss_check = True
|
||||
break
|
||||
if bliss_check:
|
||||
print(f" ⚠ Ad text ('BLISS...') leaked into essay paragraphs")
|
||||
else:
|
||||
print(f" ✅ Ad text properly excluded")
|
||||
|
||||
# Show last paragraph of each essay
|
||||
print(f"\n--- Last Paragraph of Each Essay ---")
|
||||
for e in essays:
|
||||
if e["paragraphs"]:
|
||||
last = re.sub(r"<[^>]+>", "", e["paragraphs"][-1])[:150]
|
||||
print(f" {e['title']}:")
|
||||
print(f" ...{last}")
|
||||
|
||||
print(f"\nDone! ✨")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user