feat: add journals
This commit is contained in:
gosangam
@@ -0,0 +1,675 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract five "Essays on Śrī Caitanya-caritāmṛta" from others-before-1977.html.
|
||||
|
||||
Essay 1: "The Spiritual Masters" (cc-essays, order 1, 1960)
|
||||
Essay 2: "The Absolute Truth" (cc-essays, order 2, 1957)
|
||||
Essay 3: "The Causes of Lord Caitanya's Incarnation" (cc-essays, order 3, 1957)
|
||||
Essay 4: "The Confidential Reasons for Lord Caitanya's Appearance" (cc-essays, order 4, 1957)
|
||||
Essay 5: "The Glories of Lord Nityānanda" (cc-essays, order 5, 1957)
|
||||
|
||||
All five are plain English (NO ScaGoudy decoding needed).
|
||||
|
||||
Output:
|
||||
lib/jsons/essays/the-spiritual-masters.json
|
||||
lib/jsons/essays/the-absolute-truth.json
|
||||
lib/jsons/essays/causes-of-lord-caitanyas-incarnation.json
|
||||
lib/jsons/essays/confidential-reasons-for-lord-caitanyas-appearance.json
|
||||
lib/jsons/essays/glories-of-lord-nityananda.json
|
||||
|
||||
Usage:
|
||||
python3 scripts/cc_essays_extract.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def get_css_class(tag: Tag) -> str:
|
||||
"""Get the first CSS class of a tag, or empty string."""
|
||||
classes = tag.get("class", [])
|
||||
return classes[0] if classes else ""
|
||||
|
||||
|
||||
def is_empty(tag: Tag) -> bool:
|
||||
"""Check if a tag is empty or whitespace-only."""
|
||||
text = tag.get_text().strip()
|
||||
return not text or text == "\xa0"
|
||||
|
||||
|
||||
def clean_html_content(tag: Tag) -> str:
|
||||
"""Extract inner HTML from a tag, strip span class artifacts, preserve <i>, <b>, <br>."""
|
||||
inner = "".join(str(child) for child in tag.children)
|
||||
# Remove span tags with CSS classes (RTF conversion artifacts) but keep their content
|
||||
inner = re.sub(
|
||||
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
|
||||
)
|
||||
# Remove any remaining bare <span> or </span> tags without classes
|
||||
inner = re.sub(r"</?span[^>]*>", "", inner)
|
||||
# Clean up Apple-tab-span leftovers (tab characters)
|
||||
inner = inner.replace("\t", " ")
|
||||
# Normalize whitespace (but preserve <br> tags)
|
||||
inner = re.sub(r"\s+", " ", inner).strip()
|
||||
return inner
|
||||
|
||||
|
||||
def write_json(data, filepath):
|
||||
"""Write data as formatted JSON to filepath, creating directories as needed."""
|
||||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||||
with open(filepath, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def is_divider(tag: Tag) -> bool:
|
||||
"""Check if a p9 tag is just a divider line (underscores) or a page header."""
|
||||
text = tag.get_text().strip()
|
||||
# Dividers are lines of underscores
|
||||
if re.match(r"^_+$", text):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_page_header(tag: Tag) -> bool:
|
||||
"""Check if a p9 tag is a page header like 'Essays on Shri Chaitanya Charitamrita Page 30'."""
|
||||
text = tag.get_text().strip()
|
||||
if re.search(r"Page\s+\d+", text):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_editor_note(tag: Tag) -> bool:
|
||||
"""Check if a paragraph is an editor note like '[This file has been proofread...]'."""
|
||||
text = tag.get_text().strip()
|
||||
if text.startswith("[") and (
|
||||
"proofread" in text.lower() or "—Ed" in text or "- Ed" in text or ".—Ed" in text
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Essay configurations
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
AUTHOR = "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda"
|
||||
|
||||
ESSAYS = [
|
||||
{
|
||||
"slug": "the-spiritual-masters",
|
||||
"title": "The Spiritual Masters",
|
||||
"author": AUTHOR,
|
||||
"year": 1960,
|
||||
"group": "cc-essays",
|
||||
"order": 1,
|
||||
},
|
||||
{
|
||||
"slug": "the-absolute-truth",
|
||||
"title": "The Absolute Truth",
|
||||
"author": AUTHOR,
|
||||
"year": 1957,
|
||||
"group": "cc-essays",
|
||||
"order": 2,
|
||||
},
|
||||
{
|
||||
"slug": "causes-of-lord-caitanyas-incarnation",
|
||||
"title": "The Causes of Lord Caitanya's Incarnation",
|
||||
"author": AUTHOR,
|
||||
"year": 1957,
|
||||
"group": "cc-essays",
|
||||
"order": 3,
|
||||
},
|
||||
{
|
||||
"slug": "confidential-reasons-for-lord-caitanyas-appearance",
|
||||
"title": "The Confidential Reasons for Lord Caitanya's Appearance",
|
||||
"author": AUTHOR,
|
||||
"year": 1957,
|
||||
"group": "cc-essays",
|
||||
"order": 4,
|
||||
},
|
||||
{
|
||||
"slug": "glories-of-lord-nityananda",
|
||||
"title": "The Glories of Lord Nityānanda",
|
||||
"author": AUTHOR,
|
||||
"year": 1957,
|
||||
"group": "cc-essays",
|
||||
"order": 5,
|
||||
},
|
||||
]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Extraction logic
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def parse_all_paragraphs(html_path: str) -> list[Tag]:
|
||||
"""Parse the entire HTML and return all <p> tags."""
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
soup = BeautifulSoup(content, "html.parser")
|
||||
return list(soup.find_all("p"))
|
||||
|
||||
|
||||
def find_tag_index(tags: list[Tag], start_from: int, predicate) -> int | None:
|
||||
"""Find index of first tag matching predicate, starting from start_from."""
|
||||
for i in range(start_from, len(tags)):
|
||||
if predicate(tags[i]):
|
||||
return i
|
||||
return None
|
||||
|
||||
|
||||
def extract_body_paragraphs(
|
||||
tags: list[Tag], start_idx: int, end_idx: int, include_p9_content: bool = True
|
||||
) -> list[str]:
|
||||
"""Extract body paragraphs from tags[start_idx] to tags[end_idx] (exclusive).
|
||||
|
||||
Includes p6 paragraphs (main body) and optionally p9 paragraphs that
|
||||
contain actual content (Sanskrit verses, section markers) but skips
|
||||
p9 paragraphs that are dividers or page headers.
|
||||
"""
|
||||
paragraphs = []
|
||||
body_classes = {"p6"}
|
||||
if include_p9_content:
|
||||
body_classes.add("p9")
|
||||
|
||||
for i in range(start_idx, end_idx):
|
||||
tag = tags[i]
|
||||
css = get_css_class(tag)
|
||||
|
||||
if css not in body_classes:
|
||||
continue
|
||||
|
||||
if is_empty(tag):
|
||||
continue
|
||||
|
||||
# For p9 tags, skip dividers and page headers
|
||||
if css == "p9":
|
||||
if is_divider(tag) or is_page_header(tag):
|
||||
continue
|
||||
|
||||
html = clean_html_content(tag)
|
||||
if html and html != " ":
|
||||
paragraphs.append(html)
|
||||
|
||||
return paragraphs
|
||||
|
||||
|
||||
def extract_body_paragraphs_extended(
|
||||
tags: list[Tag],
|
||||
start_idx: int,
|
||||
end_idx: int,
|
||||
body_classes: set[str] | None = None,
|
||||
skip_editor_notes: bool = False,
|
||||
) -> list[str]:
|
||||
"""Extract body paragraphs from tags[start_idx] to tags[end_idx] (exclusive).
|
||||
|
||||
Like extract_body_paragraphs but with configurable body classes and
|
||||
optional editor-note skipping.
|
||||
"""
|
||||
if body_classes is None:
|
||||
body_classes = {"p6"}
|
||||
|
||||
paragraphs = []
|
||||
|
||||
for i in range(start_idx, end_idx):
|
||||
tag = tags[i]
|
||||
css = get_css_class(tag)
|
||||
|
||||
if css not in body_classes:
|
||||
continue
|
||||
|
||||
if is_empty(tag):
|
||||
continue
|
||||
|
||||
# For p9 tags, skip dividers and page headers
|
||||
if css == "p9":
|
||||
if is_divider(tag) or is_page_header(tag):
|
||||
continue
|
||||
|
||||
# Skip editor notes if requested
|
||||
if skip_editor_notes and is_editor_note(tag):
|
||||
continue
|
||||
|
||||
html = clean_html_content(tag)
|
||||
if html and html != " ":
|
||||
paragraphs.append(html)
|
||||
|
||||
return paragraphs
|
||||
|
||||
|
||||
def extract_essay_1(tags: list[Tag]) -> dict:
|
||||
"""Extract 'The Spiritual Masters' essay.
|
||||
|
||||
Start: <p class="p8"><b>(The Spiritual Masters).</b></p>
|
||||
Body: p6 and p9 paragraphs after start
|
||||
End: Next <p class="p10"> (contains [BTG 1960])
|
||||
"""
|
||||
# Find the title tag
|
||||
start_idx = find_tag_index(
|
||||
tags,
|
||||
0,
|
||||
lambda t: get_css_class(t) == "p8" and "Spiritual Masters" in t.get_text(),
|
||||
)
|
||||
if start_idx is None:
|
||||
raise ValueError("Could not find Essay 1 start marker (p8 'Spiritual Masters')")
|
||||
|
||||
# Find the end marker: next p10
|
||||
end_idx = find_tag_index(tags, start_idx + 1, lambda t: get_css_class(t) == "p10")
|
||||
if end_idx is None:
|
||||
raise ValueError("Could not find Essay 1 end marker (p10)")
|
||||
|
||||
print(f" Essay 1: tags[{start_idx}] → tags[{end_idx}] (exclusive)")
|
||||
|
||||
# Body starts after the p8 title tag
|
||||
paragraphs = extract_body_paragraphs(tags, start_idx + 1, end_idx)
|
||||
|
||||
essay = dict(ESSAYS[0])
|
||||
essay["paragraphs"] = paragraphs
|
||||
return essay
|
||||
|
||||
|
||||
def extract_essay_2(tags: list[Tag]) -> dict:
|
||||
"""Extract 'The Absolute Truth' essay.
|
||||
|
||||
Start: <p class="p9">(Second Part)</p>
|
||||
Body: p6 paragraphs after start
|
||||
End: Next <p class="p11"> after start
|
||||
"""
|
||||
# Find the start marker
|
||||
start_idx = find_tag_index(
|
||||
tags,
|
||||
0,
|
||||
lambda t: get_css_class(t) == "p9" and "(Second Part)" in t.get_text(),
|
||||
)
|
||||
if start_idx is None:
|
||||
raise ValueError("Could not find Essay 2 start marker (p9 '(Second Part)')")
|
||||
|
||||
# Find the end marker: next p11
|
||||
end_idx = find_tag_index(tags, start_idx + 1, lambda t: get_css_class(t) == "p11")
|
||||
if end_idx is None:
|
||||
raise ValueError("Could not find Essay 2 end marker (p11)")
|
||||
|
||||
print(f" Essay 2: tags[{start_idx}] → tags[{end_idx}] (exclusive)")
|
||||
|
||||
# Body starts after the "(Second Part)" header
|
||||
paragraphs = extract_body_paragraphs(tags, start_idx + 1, end_idx)
|
||||
|
||||
essay = dict(ESSAYS[1])
|
||||
essay["paragraphs"] = paragraphs
|
||||
return essay
|
||||
|
||||
|
||||
def extract_essay_3(tags: list[Tag]) -> dict:
|
||||
"""Extract 'The Causes of Lord Caitanya's Incarnation' essay.
|
||||
|
||||
Start: <p class="p9"> containing "Essays on Shri Chaitanya Charitamrita. III"
|
||||
Body: p6 paragraphs after start
|
||||
End: Next <p class="p11"> after start.
|
||||
The <p class="p9">____</p> divider is skipped.
|
||||
But the <p class="p6">Thus ends the III part...</p> IS included.
|
||||
"""
|
||||
# Find the start marker — note the ". III" with period
|
||||
start_idx = find_tag_index(
|
||||
tags,
|
||||
0,
|
||||
lambda t: (
|
||||
get_css_class(t) == "p9"
|
||||
and "Essays on Shri Chaitanya Charitamrita. III" in t.get_text()
|
||||
),
|
||||
)
|
||||
if start_idx is None:
|
||||
raise ValueError(
|
||||
"Could not find Essay 3 start marker (p9 'Essays on Shri Chaitanya Charitamrita. III')"
|
||||
)
|
||||
|
||||
# Find the end marker: next p11
|
||||
end_idx = find_tag_index(tags, start_idx + 1, lambda t: get_css_class(t) == "p11")
|
||||
if end_idx is None:
|
||||
raise ValueError("Could not find Essay 3 end marker (p11)")
|
||||
|
||||
print(f" Essay 3: tags[{start_idx}] → tags[{end_idx}] (exclusive)")
|
||||
|
||||
# Body starts after the header tag.
|
||||
# We collect p6 paragraphs. The divider (p9 with underscores) is skipped
|
||||
# automatically by extract_body_paragraphs. The "Thus ends the III part..."
|
||||
# paragraph is a p6, so it will be included.
|
||||
paragraphs = extract_body_paragraphs(tags, start_idx + 1, end_idx)
|
||||
|
||||
essay = dict(ESSAYS[2])
|
||||
essay["paragraphs"] = paragraphs
|
||||
return essay
|
||||
|
||||
|
||||
def extract_essay_4(tags: list[Tag]) -> dict:
|
||||
"""Extract 'The Confidential Reasons for Lord Caitanya's Appearance' essay.
|
||||
|
||||
Start: <p class="p30"> containing "4th Chapter"
|
||||
Body: p6 paragraphs after start (skip editor note, skip p4/p5/p29/p30 headers)
|
||||
End: Next <p class="p4"> after start (next chapter header "Ädi-lélä, Chapter 5")
|
||||
"""
|
||||
# Find the start marker: p30 containing "4th Chapter"
|
||||
start_idx = find_tag_index(
|
||||
tags,
|
||||
0,
|
||||
lambda t: get_css_class(t) == "p30" and "4th Chapter" in t.get_text(),
|
||||
)
|
||||
if start_idx is None:
|
||||
raise ValueError("Could not find Essay 4 start marker (p30 '4th Chapter')")
|
||||
|
||||
# Find the end marker: next p4 after the start (next chapter header)
|
||||
end_idx = find_tag_index(tags, start_idx + 1, lambda t: get_css_class(t) == "p4")
|
||||
if end_idx is None:
|
||||
raise ValueError("Could not find Essay 4 end marker (p4)")
|
||||
|
||||
print(f" Essay 4: tags[{start_idx}] → tags[{end_idx}] (exclusive)")
|
||||
|
||||
# Collect p6 paragraphs, skipping editor notes
|
||||
paragraphs = extract_body_paragraphs_extended(
|
||||
tags,
|
||||
start_idx + 1,
|
||||
end_idx,
|
||||
body_classes={"p6"},
|
||||
skip_editor_notes=True,
|
||||
)
|
||||
|
||||
essay = dict(ESSAYS[3])
|
||||
essay["paragraphs"] = paragraphs
|
||||
return essay
|
||||
|
||||
|
||||
def extract_essay_5(tags: list[Tag]) -> dict:
|
||||
"""Extract 'The Glories of Lord Nityānanda' essay.
|
||||
|
||||
Start: <p class="p31"> containing "1.V" (i.e. "Shi Chaitanya Charitamrita 1.V")
|
||||
Body: p6 and p34 paragraphs after start (skip editor note, skip p4/p5/p31 headers,
|
||||
skip p9 divider underscores)
|
||||
End: Next <p class="p35"> after start ("Easy Journey To Other Planets")
|
||||
"""
|
||||
# Find the start marker: p31 containing "1.V"
|
||||
start_idx = find_tag_index(
|
||||
tags,
|
||||
0,
|
||||
lambda t: get_css_class(t) == "p31" and "1.V" in t.get_text(),
|
||||
)
|
||||
if start_idx is None:
|
||||
raise ValueError("Could not find Essay 5 start marker (p31 containing '1.V')")
|
||||
|
||||
# Find the end marker: next p35 after the start
|
||||
end_idx = find_tag_index(tags, start_idx + 1, lambda t: get_css_class(t) == "p35")
|
||||
if end_idx is None:
|
||||
raise ValueError("Could not find Essay 5 end marker (p35)")
|
||||
|
||||
print(f" Essay 5: tags[{start_idx}] → tags[{end_idx}] (exclusive)")
|
||||
|
||||
# Collect p6 and p34 paragraphs, skipping editor notes and p9 dividers
|
||||
paragraphs = extract_body_paragraphs_extended(
|
||||
tags,
|
||||
start_idx + 1,
|
||||
end_idx,
|
||||
body_classes={"p6", "p34"},
|
||||
skip_editor_notes=True,
|
||||
)
|
||||
|
||||
essay = dict(ESSAYS[4])
|
||||
essay["paragraphs"] = paragraphs
|
||||
return essay
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Quality checks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def quality_check(essay: dict, essay_num: int) -> None:
|
||||
"""Run quality checks on an extracted essay."""
|
||||
paragraphs = essay["paragraphs"]
|
||||
title = essay["title"]
|
||||
|
||||
print(f"\n--- Quality Checks: Essay {essay_num} ({title}) ---")
|
||||
|
||||
# Check paragraph count is reasonable
|
||||
if len(paragraphs) < 5:
|
||||
print(f" ⚠ Only {len(paragraphs)} paragraphs (suspiciously low)")
|
||||
elif len(paragraphs) > 150:
|
||||
print(f" ⚠ {len(paragraphs)} paragraphs (suspiciously high)")
|
||||
else:
|
||||
print(f" ✅ {len(paragraphs)} paragraphs (reasonable)")
|
||||
|
||||
# Check for residual span class artifacts
|
||||
span_count = sum(1 for p in paragraphs if 'class="' in p and "<span" in p)
|
||||
if span_count == 0:
|
||||
print(f" ✅ No residual span class artifacts found")
|
||||
else:
|
||||
print(f" ⚠ {span_count} paragraphs still contain span class artifacts")
|
||||
|
||||
# Check key terms
|
||||
all_text = " ".join(paragraphs)
|
||||
if "Krishna" in all_text or "Chaitanya" in all_text:
|
||||
print(f" ✅ 'Krishna'/'Chaitanya' found (plain English, as expected)")
|
||||
else:
|
||||
print(f" ⚠ Key terms not found")
|
||||
|
||||
# Check that editor notes were excluded
|
||||
for p in paragraphs:
|
||||
plain = re.sub(r"<[^>]+>", "", p).strip()
|
||||
if plain.startswith("[") and ("proofread" in plain.lower() or "—Ed" in plain):
|
||||
print(f" ⚠ Editor note leaked into paragraphs")
|
||||
break
|
||||
else:
|
||||
print(f" ✅ No editor notes leaked into paragraphs")
|
||||
|
||||
# Check that divider lines were excluded
|
||||
for p in paragraphs:
|
||||
plain = re.sub(r"<[^>]+>", "", p).strip()
|
||||
if re.match(r"^_+$", plain):
|
||||
print(f" ⚠ Divider line leaked into paragraphs")
|
||||
break
|
||||
else:
|
||||
print(f" ✅ No divider lines leaked into paragraphs")
|
||||
|
||||
# Essay-specific checks
|
||||
if essay_num == 1:
|
||||
if "spiritual master" in all_text.lower():
|
||||
print(f" ✅ 'spiritual master' found in text")
|
||||
else:
|
||||
print(f" ⚠ 'spiritual master' not found")
|
||||
|
||||
# Check that reference line [BTG 1960] was excluded
|
||||
for p in paragraphs:
|
||||
plain = re.sub(r"<[^>]+>", "", p).strip()
|
||||
if "BTG 1960" in plain and plain.startswith("["):
|
||||
print(f" ⚠ Reference line '[BTG 1960]' leaked into paragraphs")
|
||||
break
|
||||
else:
|
||||
print(f" ✅ Reference line properly excluded")
|
||||
|
||||
# Check that title was excluded
|
||||
for p in paragraphs:
|
||||
plain = re.sub(r"<[^>]+>", "", p).strip()
|
||||
if (
|
||||
plain == "(The Spiritual Masters)."
|
||||
or plain == "(The Spiritual Masters)."
|
||||
):
|
||||
print(f" ⚠ Title line leaked into paragraphs")
|
||||
break
|
||||
else:
|
||||
print(f" ✅ Title line properly excluded")
|
||||
|
||||
elif essay_num == 2:
|
||||
if "impersonal Brahman" in all_text or "Absolute Truth" in all_text:
|
||||
print(f" ✅ 'Absolute Truth' / 'impersonal Brahman' found")
|
||||
else:
|
||||
print(f" ⚠ Key terms for Essay 2 not found")
|
||||
|
||||
# Check "(Second Part)" header was excluded
|
||||
for p in paragraphs:
|
||||
plain = re.sub(r"<[^>]+>", "", p).strip()
|
||||
if plain == "(Second Part)":
|
||||
print(f" ⚠ Header '(Second Part)' leaked into paragraphs")
|
||||
break
|
||||
else:
|
||||
print(f" ✅ Header '(Second Part)' properly excluded")
|
||||
|
||||
elif essay_num == 3:
|
||||
if "incarnation" in all_text.lower() or "descent" in all_text.lower():
|
||||
print(f" ✅ 'incarnation'/'descent' found")
|
||||
else:
|
||||
print(f" ⚠ Key terms for Essay 3 not found")
|
||||
|
||||
# Check that "Thus ends the III part..." is included
|
||||
found_ending = False
|
||||
for p in paragraphs:
|
||||
if "Thus ends the III part" in p:
|
||||
found_ending = True
|
||||
break
|
||||
if found_ending:
|
||||
print(f" ✅ 'Thus ends the III part...' paragraph included")
|
||||
else:
|
||||
print(f" ⚠ 'Thus ends the III part...' paragraph missing")
|
||||
|
||||
elif essay_num == 4:
|
||||
if "Radharani" in all_text or "confidential" in all_text.lower():
|
||||
print(f" ✅ 'Radharani'/'confidential' found")
|
||||
else:
|
||||
print(f" ⚠ Key terms for Essay 4 not found")
|
||||
|
||||
# Check that the summary paragraph is included (last paragraph)
|
||||
found_summary = False
|
||||
for p in paragraphs:
|
||||
if "author then summarize" in p.lower():
|
||||
found_summary = True
|
||||
break
|
||||
if found_summary:
|
||||
print(f" ✅ Summary paragraph ('The author then summarize...') included")
|
||||
else:
|
||||
print(f" ⚠ Summary paragraph missing")
|
||||
|
||||
# Check that the first body paragraph mentions "three principal purposes"
|
||||
if paragraphs:
|
||||
if "three principal purposes" in paragraphs[0]:
|
||||
print(f" ✅ First paragraph mentions 'three principal purposes'")
|
||||
else:
|
||||
print(f" ⚠ First paragraph doesn't mention 'three principal purposes'")
|
||||
|
||||
# Check that p4/p5/p29/p30 header lines were excluded
|
||||
for p in paragraphs:
|
||||
plain = re.sub(r"<[^>]+>", "", p).strip()
|
||||
if "4th Chapter" in plain and len(plain) < 50:
|
||||
print(f" ⚠ Header '4th Chapter' leaked into paragraphs")
|
||||
break
|
||||
else:
|
||||
print(f" ✅ Header lines properly excluded")
|
||||
|
||||
elif essay_num == 5:
|
||||
if "Nityananda" in all_text:
|
||||
print(f" ✅ 'Nityananda' found")
|
||||
else:
|
||||
print(f" ⚠ 'Nityananda' not found")
|
||||
|
||||
# Check that the final p34 description paragraph is included
|
||||
if paragraphs:
|
||||
last_plain = re.sub(r"<[^>]+>", "", paragraphs[-1]).strip()
|
||||
if (
|
||||
"lusture" in last_plain
|
||||
or "turban" in last_plain
|
||||
or "wonderful feature" in last_plain
|
||||
):
|
||||
print(f" ✅ Final description paragraph (p34) included")
|
||||
else:
|
||||
print(f" ⚠ Final description paragraph (p34) may be missing")
|
||||
print(f" Last paragraph preview: {last_plain[:80]}...")
|
||||
|
||||
# Check that the first body paragraph mentions "5th chapter"
|
||||
if paragraphs:
|
||||
if "5th chapter" in paragraphs[0].lower():
|
||||
print(f" ✅ First paragraph mentions '5th chapter'")
|
||||
else:
|
||||
print(f" ⚠ First paragraph doesn't mention '5th chapter'")
|
||||
|
||||
# Check that p31 header was excluded
|
||||
for p in paragraphs:
|
||||
plain = re.sub(r"<[^>]+>", "", p).strip()
|
||||
if "Shi Chaitanya Charitamrita 1.V" in plain:
|
||||
print(
|
||||
f" ⚠ Header 'Shi Chaitanya Charitamrita 1.V' leaked into paragraphs"
|
||||
)
|
||||
break
|
||||
else:
|
||||
print(f" ✅ Header lines properly excluded")
|
||||
|
||||
# Sample output
|
||||
print(f"\n First paragraph (preview):")
|
||||
if paragraphs:
|
||||
preview = re.sub(r"<[^>]+>", "", paragraphs[0])[:120]
|
||||
print(f" {preview}...")
|
||||
print(f" Last paragraph (preview):")
|
||||
if paragraphs:
|
||||
last = re.sub(r"<[^>]+>", "", paragraphs[-1])[:120]
|
||||
print(f" {last}...")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html")
|
||||
output_dir = os.path.join(base_dir, "lib", "jsons", "essays")
|
||||
|
||||
if not os.path.exists(html_path):
|
||||
print(f"ERROR: {html_path} not found!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Reading {html_path}...")
|
||||
tags = parse_all_paragraphs(html_path)
|
||||
print(f" Parsed {len(tags)} <p> tags total\n")
|
||||
|
||||
# Extract all five essays
|
||||
print("Locating essays...")
|
||||
essay_1 = extract_essay_1(tags)
|
||||
essay_2 = extract_essay_2(tags)
|
||||
essay_3 = extract_essay_3(tags)
|
||||
essay_4 = extract_essay_4(tags)
|
||||
essay_5 = extract_essay_5(tags)
|
||||
|
||||
all_essays = [essay_1, essay_2, essay_3, essay_4, essay_5]
|
||||
|
||||
# Write JSON files
|
||||
print(f"\nWriting JSON files...")
|
||||
for essay in all_essays:
|
||||
output_path = os.path.join(output_dir, f"{essay['slug']}.json")
|
||||
write_json(essay, output_path)
|
||||
print(f" Written: {output_path}")
|
||||
|
||||
# Quality checks
|
||||
for i, essay in enumerate(all_essays, 1):
|
||||
quality_check(essay, i)
|
||||
|
||||
# Summary
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f" EXTRACTION SUMMARY")
|
||||
print(f"{'=' * 60}")
|
||||
for i, essay in enumerate(all_essays, 1):
|
||||
print(
|
||||
f" Essay {i}: {essay['title']}"
|
||||
f" ({len(essay['paragraphs'])} paragraphs, {essay['year']})"
|
||||
)
|
||||
print(
|
||||
f" slug={essay['slug']}, group={essay['group']}, order={essay['order']}"
|
||||
)
|
||||
print(f"{'=' * 60}")
|
||||
print(f" Total essays: {len(all_essays)}")
|
||||
print(f" Total paragraphs: {sum(len(e['paragraphs']) for e in all_essays)}")
|
||||
print(f"\nDone! ✨")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user