feat: add journals
This commit is contained in:
gosangam
@@ -0,0 +1,303 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract "An Analysis of Bhagwat Geeta" from others-before-1977.html.
|
||||
|
||||
This document is a structured analysis organized into three numbered sections
|
||||
(topics) with numbered points and verse references. It is plain English text
|
||||
(no ScaGoudy decoding needed).
|
||||
|
||||
Source: public/html/others-before-1977.html
|
||||
- Title: <p class="p116"><b>An Analysis of Bhagwat Geeta</b></p> (line ~15182)
|
||||
- Section 1: "Mans eternal relation with Godhead" — 45 <p class="p6"> items
|
||||
- Section 2: "Man's Duty Toward Godhead" — 34 <p class="p6"> items
|
||||
- Section 3: "Man's ultimate necessity and perfection of life" — 3 <p class="p6"> items
|
||||
- End boundary: <p class="p11">Solution of Present Crisises... (line ~15276)
|
||||
|
||||
CSS classes:
|
||||
p116 — Title (bold)
|
||||
p39 — Section headers ("1: Mans eternal relation with Godhead")
|
||||
p5 — Duplicate section title (bold, with <span class="s3">) — SKIP
|
||||
p6 — Body paragraphs (plain English)
|
||||
p34 — Math calculation at the end — SKIP
|
||||
p11 — Next document title (end boundary)
|
||||
|
||||
Output (prose book structure):
|
||||
lib/jsons/other-documents/analysis-of-bhagavad-gita/
|
||||
intro.json # metadata
|
||||
chapters/
|
||||
01.json # Chapter 1
|
||||
02.json # Chapter 2
|
||||
03.json # Chapter 3
|
||||
|
||||
Usage:
|
||||
python3 scripts/od_analysis_bg_extract.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
BOOK_CODE = "od-analysis-bg"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def get_css_class(tag: Tag) -> str:
|
||||
"""Get the first CSS class of a tag, or empty string."""
|
||||
classes = tag.get("class", [])
|
||||
return classes[0] if classes else ""
|
||||
|
||||
|
||||
def is_empty(tag: Tag) -> bool:
|
||||
"""Check if a tag is empty or whitespace-only."""
|
||||
text = tag.get_text().strip()
|
||||
return not text or text == "\xa0"
|
||||
|
||||
|
||||
def clean_text(tag: Tag) -> str:
|
||||
"""Extract inner HTML from a tag, strip span class artifacts, normalize whitespace."""
|
||||
inner = "".join(str(child) for child in tag.children)
|
||||
# Remove span tags with CSS classes (RTF conversion artifacts) but keep their content
|
||||
inner = re.sub(
|
||||
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
|
||||
)
|
||||
# Remove any remaining bare <span> or </span> tags
|
||||
inner = re.sub(r"</?span[^>]*>", "", inner)
|
||||
# Remove <b> and </b> tags (not needed for body paragraphs)
|
||||
inner = re.sub(r"</?b>", "", inner)
|
||||
# Clean up Apple-tab-span leftovers (tab characters)
|
||||
inner = inner.replace("\t", " ")
|
||||
# Strip <br> tags
|
||||
inner = re.sub(r"<br\s*/?>", " ", inner)
|
||||
# Normalize whitespace
|
||||
inner = re.sub(r"\s+", " ", inner).strip()
|
||||
# Fix HTML entities
|
||||
inner = inner.replace("&", "&")
|
||||
return inner
|
||||
|
||||
|
||||
def write_json(data, filepath):
|
||||
"""Write data as formatted JSON to filepath, creating directories as needed."""
|
||||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||||
with open(filepath, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
print(f" ✅ Wrote {filepath}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def extract_chapters(soup: BeautifulSoup) -> list[dict]:
|
||||
"""Extract the chapter data from 'An Analysis of Bhagwat Geeta'."""
|
||||
|
||||
# Find the title tag: <p class="p116"><b>An Analysis of Bhagwat Geeta</b></p>
|
||||
title_tag = None
|
||||
for p in soup.find_all("p", class_="p116"):
|
||||
text = p.get_text().strip()
|
||||
if "Analysis of Bhagwat Geeta" in text:
|
||||
title_tag = p
|
||||
break
|
||||
|
||||
if not title_tag:
|
||||
print("❌ Could not find title 'An Analysis of Bhagwat Geeta'")
|
||||
sys.exit(1)
|
||||
|
||||
print(f" 📍 Found title: {title_tag.get_text().strip()}")
|
||||
|
||||
# Collect all <p> tags after the title until we hit the end boundary
|
||||
chapters = []
|
||||
current_chapter = None
|
||||
|
||||
tag = title_tag.find_next_sibling()
|
||||
while tag:
|
||||
if not isinstance(tag, Tag) or tag.name != "p":
|
||||
tag = tag.find_next_sibling()
|
||||
continue
|
||||
|
||||
css = get_css_class(tag)
|
||||
text = tag.get_text().strip()
|
||||
|
||||
# End boundary: next document starts
|
||||
if css == "p11" and "Solution of Present Crisises" in text:
|
||||
break
|
||||
|
||||
# Skip empty tags
|
||||
if is_empty(tag):
|
||||
tag = tag.find_next_sibling()
|
||||
continue
|
||||
|
||||
# Section header: <p class="p39">1: Mans eternal relation with Godhead</p>
|
||||
if css == "p39":
|
||||
# Parse section number and title
|
||||
match = re.match(r"(\d+):\s*(.+)", text)
|
||||
if match:
|
||||
num = int(match.group(1))
|
||||
title = match.group(2).strip()
|
||||
current_chapter = {
|
||||
"number": num,
|
||||
"title": title,
|
||||
"paragraphs": [],
|
||||
}
|
||||
chapters.append(current_chapter)
|
||||
print(f" 📖 Section {num}: {title}")
|
||||
tag = tag.find_next_sibling()
|
||||
continue
|
||||
|
||||
# Skip p5 (duplicate section title in bold with span)
|
||||
if css == "p5":
|
||||
tag = tag.find_next_sibling()
|
||||
continue
|
||||
|
||||
# Skip p34 (math calculation at the end)
|
||||
if css == "p34":
|
||||
tag = tag.find_next_sibling()
|
||||
continue
|
||||
|
||||
# Body paragraph: <p class="p6">
|
||||
if css == "p6" and current_chapter is not None:
|
||||
cleaned = clean_text(tag)
|
||||
if cleaned:
|
||||
current_chapter["paragraphs"].append(cleaned)
|
||||
tag = tag.find_next_sibling()
|
||||
continue
|
||||
|
||||
# Any other class — skip
|
||||
tag = tag.find_next_sibling()
|
||||
|
||||
return chapters
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Quality check
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def quality_check(chapters: list[dict]):
|
||||
"""Validate the extracted chapters."""
|
||||
print("\n🔍 Quality check:\n")
|
||||
|
||||
print(f" Chapters: {len(chapters)}")
|
||||
assert len(chapters) == 3, f"Expected 3 chapters, got {len(chapters)}"
|
||||
|
||||
total_paragraphs = 0
|
||||
for ch in chapters:
|
||||
count = len(ch["paragraphs"])
|
||||
total_paragraphs += count
|
||||
print(f' Chapter {ch["number"]}: "{ch["title"]}" — {count} paragraphs')
|
||||
|
||||
# Spot-check first paragraph
|
||||
if ch["paragraphs"]:
|
||||
first = ch["paragraphs"][0]
|
||||
print(f" First: {first[:80]}...")
|
||||
|
||||
# Spot-check last paragraph
|
||||
if len(ch["paragraphs"]) > 1:
|
||||
last = ch["paragraphs"][-1]
|
||||
print(f" Last: {last[:80]}...")
|
||||
|
||||
print(f"\n Total paragraphs: {total_paragraphs}")
|
||||
|
||||
# Expected counts (from manual inspection):
|
||||
# Section 1: 45 items (numbered 1-45)
|
||||
# Section 2: 34 items (numbered 1-34)
|
||||
# Section 3: 3 items (numbered 1-3)
|
||||
expected_total = 82 # 45 + 34 + 3
|
||||
if total_paragraphs == expected_total:
|
||||
print(f" ✅ Paragraph count matches expected: {expected_total}")
|
||||
else:
|
||||
print(f" ⚠️ Expected {expected_total} paragraphs, got {total_paragraphs}")
|
||||
# This is a warning, not a hard failure — some items may have sub-items
|
||||
# or the HTML structure may slightly differ
|
||||
|
||||
# Check that paragraphs are not empty
|
||||
for ch in chapters:
|
||||
for i, p in enumerate(ch["paragraphs"]):
|
||||
assert p.strip(), f"Empty paragraph in chapter {ch['number']}, index {i}"
|
||||
|
||||
print("\n ✅ All checks passed!")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Output
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def write_output(chapters: list[dict], output_dir: str):
|
||||
"""Write intro.json and per-chapter JSON files."""
|
||||
|
||||
# Build intro.json
|
||||
intro = {
|
||||
"book": BOOK_CODE,
|
||||
"title": "An Analysis of Bhagwat Geeta",
|
||||
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
||||
"year": None,
|
||||
"description": "A structured analysis of Bhagavad-gītā organized into three topics with numbered points and verse references.",
|
||||
"totalChapters": len(chapters),
|
||||
"chapters": [{"number": ch["number"], "title": ch["title"]} for ch in chapters],
|
||||
}
|
||||
|
||||
write_json(intro, os.path.join(output_dir, "intro.json"))
|
||||
|
||||
# Write per-chapter files
|
||||
chapters_dir = os.path.join(output_dir, "chapters")
|
||||
for ch in chapters:
|
||||
chapter_data = {
|
||||
"book": BOOK_CODE,
|
||||
"chapter": ch["number"],
|
||||
"title": ch["title"],
|
||||
"type": "chapter",
|
||||
"paragraphs": ch["paragraphs"],
|
||||
}
|
||||
filename = f"{ch['number']:02d}.json"
|
||||
write_json(chapter_data, os.path.join(chapters_dir, filename))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html")
|
||||
output_dir = os.path.join(
|
||||
base_dir, "lib", "jsons", "other-documents", "analysis-of-bhagavad-gita"
|
||||
)
|
||||
|
||||
if not os.path.exists(html_path):
|
||||
print(f"❌ HTML file not found: {html_path}")
|
||||
sys.exit(1)
|
||||
|
||||
# Clean up old single-file JSON if it exists
|
||||
old_json = os.path.join(
|
||||
base_dir, "lib", "jsons", "other-documents", "analysis-of-bhagavad-gita.json"
|
||||
)
|
||||
if os.path.exists(old_json):
|
||||
os.remove(old_json)
|
||||
print(f" 🗑️ Removed old single-file JSON: {old_json}")
|
||||
|
||||
print(f"📂 Reading {html_path}...")
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
soup = BeautifulSoup(f, "html.parser")
|
||||
|
||||
print("\n📝 Extracting 'An Analysis of Bhagwat Geeta'...\n")
|
||||
chapters = extract_chapters(soup)
|
||||
|
||||
quality_check(chapters)
|
||||
|
||||
write_output(chapters, output_dir)
|
||||
print(f"\n🎉 Done! Output directory: {output_dir}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user