feat: add journals
This commit is contained in:
gosangam
@@ -0,0 +1,590 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract Bhagavāner Kathā (part of Renunciation through Wisdom) from others-before-1977.html.
|
||||
|
||||
The book has 2 chapters (no intro segments):
|
||||
1. This World of Misery
|
||||
2. The Cause of Misery
|
||||
|
||||
Source: public/html/others-before-1977.html, lines ~4182–4289.
|
||||
Start: <p class="p56"> containing "Bhagaväner kathä"
|
||||
End: <p class="p35"> containing "Indian Bhagavatams" (stop before)
|
||||
|
||||
Chapter boundaries: <p class="p17"> tags containing "Bk:Chapter"
|
||||
|
||||
IMPORTANT: Unlike MOG/CBHC, this book's body text uses ScaGoudy font encoding.
|
||||
Characters like ä, å, ñ, ö, ë, ò, ç, ì, ï, ü, é, à, ù must be decoded to
|
||||
proper IAST Unicode (ā, ṛ, ṣ, ṭ, ṇ, ḍ, ś, ṅ, ñ, ū, ī, ṁ, ḥ).
|
||||
|
||||
CSS classes:
|
||||
p56 — Book title header (skip)
|
||||
p17 — Chapter bookmark headers ("Bk:Chapter N") — boundary detection, skip
|
||||
p5 — Chapter title headings (skip — we derive titles ourselves)
|
||||
p6 — Body paragraphs (MAIN content) — HAS ScaGoudy encoding
|
||||
p12 — Body paragraph variant (longer text) — HAS ScaGoudy encoding
|
||||
p38 — Body paragraph variant — HAS ScaGoudy encoding
|
||||
p9 — Sanskrit verse lines (italic, with <span class="s3">) — centered italic
|
||||
p10 — Reference markers like [RTW 1.1] — include as paragraphs
|
||||
p61 — Reference marker variant [RTW 1.2] — include
|
||||
p57 — Footnote header ("Footnote") — include
|
||||
p58 — Footnote text — include
|
||||
p19 — Sanskrit verse block (multi-line) — centered italic
|
||||
p59 — Sanskrit verse block variant — centered italic
|
||||
p23 — Sanskrit verse line variant — centered italic
|
||||
p36 — Sanskrit verse line variant — centered italic
|
||||
p60 — Sanskrit verse line variant — centered italic
|
||||
|
||||
Output:
|
||||
lib/jsons/books/bk/intro.json — Book metadata
|
||||
lib/jsons/books/bk/chapters/01.json — Chapter 1: This World of Misery
|
||||
lib/jsons/books/bk/chapters/02.json — Chapter 2: The Cause of Misery
|
||||
|
||||
Usage:
|
||||
python3 scripts/bk_extract.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ScaGoudy → IAST Unicode mapping
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
SCAGOUDY_TO_IAST = {
|
||||
"ä": "ā",
|
||||
"Ä": "Ā",
|
||||
"å": "ṛ",
|
||||
"Å": "Ṛ",
|
||||
"ñ": "ṣ",
|
||||
"Ñ": "Ṣ",
|
||||
"ö": "ṭ",
|
||||
"Ö": "Ṭ",
|
||||
"ë": "ṇ",
|
||||
"Ë": "Ṇ",
|
||||
"ò": "ḍ",
|
||||
"Ò": "Ḍ",
|
||||
"ç": "ś",
|
||||
"Ç": "Ś",
|
||||
"ì": "ṅ",
|
||||
"Ì": "Ṅ",
|
||||
"ï": "ñ",
|
||||
"Ï": "Ñ",
|
||||
"ü": "ū",
|
||||
"Ü": "Ū",
|
||||
"é": "ī",
|
||||
"É": "Ī",
|
||||
"à": "ṁ",
|
||||
"À": "Ṁ",
|
||||
"ù": "ḥ",
|
||||
"Ù": "Ḥ",
|
||||
"è": "ṝ",
|
||||
"û": "ṁl",
|
||||
"ÿ": "ḷ",
|
||||
}
|
||||
|
||||
# Build a translation table for fast character-by-character replacement
|
||||
_SCAGOUDY_TRANS = str.maketrans(
|
||||
{k: v for k, v in SCAGOUDY_TO_IAST.items() if len(k) == 1}
|
||||
)
|
||||
|
||||
|
||||
def scagoudy_decode(text: str) -> str:
|
||||
"""Decode ScaGoudy-encoded text to proper IAST Unicode."""
|
||||
return text.translate(_SCAGOUDY_TRANS)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def decode_html_content(tag: Tag, apply_scagoudy: bool = True) -> str:
|
||||
"""Extract inner HTML from a tag, preserve formatting tags, strip spans,
|
||||
and optionally apply ScaGoudy decoding.
|
||||
|
||||
Steps:
|
||||
1. Get inner HTML
|
||||
2. Strip <span class="sN"> wrappers (keep content)
|
||||
3. Apply ScaGoudy decode to the entire result (if requested)
|
||||
4. Normalize whitespace
|
||||
"""
|
||||
inner = "".join(str(child) for child in tag.children)
|
||||
# Remove span tags with CSS classes (RTF conversion artifacts) — keep content
|
||||
inner = re.sub(
|
||||
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
|
||||
)
|
||||
# Clean up Apple-tab-span leftovers (tab characters)
|
||||
inner = inner.replace("\t", " ")
|
||||
# Normalize whitespace
|
||||
inner = re.sub(r"\s+", " ", inner).strip()
|
||||
# Apply ScaGoudy decoding
|
||||
if apply_scagoudy:
|
||||
inner = scagoudy_decode(inner)
|
||||
return inner
|
||||
|
||||
|
||||
def decode_html_content_as_verse(tag: Tag) -> str:
|
||||
"""Extract inner HTML, strip spans, ScaGoudy decode, wrap as centered italic.
|
||||
|
||||
For Sanskrit verse lines (p9, p19, p59, p23, p36, p60).
|
||||
Preserves <br> and reference tags like [Bg. 7.14].
|
||||
"""
|
||||
inner = "".join(str(child) for child in tag.children)
|
||||
# Remove span tags with CSS classes — keep content
|
||||
inner = re.sub(
|
||||
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
|
||||
)
|
||||
# Clean up tab characters
|
||||
inner = inner.replace("\t", " ")
|
||||
# Normalize spaces (but preserve <br> tags)
|
||||
# Replace multiple spaces with single space
|
||||
inner = re.sub(r"[ ]+", " ", inner).strip()
|
||||
# Remove trailing <br> tags
|
||||
inner = re.sub(r"(<br\s*/?\s*>)+\s*$", "", inner).strip()
|
||||
# Apply ScaGoudy decoding
|
||||
inner = scagoudy_decode(inner)
|
||||
# Wrap as centered italic
|
||||
if inner:
|
||||
return f'<div style="text-align:center"><i>{inner}</i></div>'
|
||||
return ""
|
||||
|
||||
|
||||
def decode_text(tag: Tag) -> str:
|
||||
"""Get plain text from a tag."""
|
||||
return tag.get_text().strip()
|
||||
|
||||
|
||||
def get_css_class(tag: Tag) -> str:
|
||||
"""Get the first CSS class of a tag, or empty string."""
|
||||
classes = tag.get("class", [])
|
||||
return classes[0] if classes else ""
|
||||
|
||||
|
||||
def is_empty(tag: Tag) -> bool:
|
||||
"""Check if a tag is empty or whitespace-only."""
|
||||
text = tag.get_text().strip()
|
||||
return not text or text == "\xa0"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CSS class categorization for BK
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Book title header
|
||||
BOOK_TITLE_CLASS = "p56"
|
||||
|
||||
# Chapter bookmark headers (used for boundary detection)
|
||||
CHAPTER_BOOKMARK_CLASS = "p17"
|
||||
|
||||
# Chapter title (bold) — skip from body
|
||||
CHAPTER_TITLE_CLASS = "p5"
|
||||
|
||||
# Body paragraph classes (all need ScaGoudy decoding)
|
||||
BODY_CLASSES = frozenset(["p6", "p12", "p38"])
|
||||
|
||||
# Sanskrit verse line classes (wrap as centered italic)
|
||||
VERSE_CLASSES = frozenset(["p9", "p19", "p59", "p23", "p36", "p60"])
|
||||
|
||||
# Reference marker classes — include as paragraphs
|
||||
REFERENCE_CLASSES = frozenset(["p10", "p61"])
|
||||
|
||||
# Footnote classes — include as paragraphs
|
||||
FOOTNOTE_CLASSES = frozenset(["p57", "p58"])
|
||||
|
||||
# Classes to skip entirely
|
||||
SKIP_CLASSES = frozenset(["p43", "p24"])
|
||||
|
||||
# TOC entry class
|
||||
TOC_CLASS = "p2"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Boundary detection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def find_bk_boundaries(all_paragraphs: list) -> tuple:
|
||||
"""Find the start and end paragraph indices of the BK content.
|
||||
|
||||
Start: <p class="p56"> containing "Bhagaväner kathä"
|
||||
End: <p class="p35"> containing "Indian Bhagavatams" (stop before)
|
||||
"""
|
||||
start_idx = None
|
||||
end_idx = len(all_paragraphs)
|
||||
|
||||
for i, p in enumerate(all_paragraphs):
|
||||
cls = get_css_class(p)
|
||||
text = decode_text(p)
|
||||
|
||||
# Find start: p56 containing "Bhagaväner" or p2 TOC entry
|
||||
if start_idx is None:
|
||||
if cls == BOOK_TITLE_CLASS and "Bhag" in text and "kath" in text:
|
||||
start_idx = i
|
||||
continue
|
||||
continue
|
||||
|
||||
# Find end: p35 containing "Indian Bhagavatams"
|
||||
if cls == "p35" and "Indian" in text and "Bhagavatam" in text:
|
||||
end_idx = i
|
||||
break
|
||||
|
||||
if start_idx is None:
|
||||
print("ERROR: Could not find 'Bhagaväner kathä' in others-before-1977.html!")
|
||||
sys.exit(1)
|
||||
|
||||
return start_idx, end_idx
|
||||
|
||||
|
||||
def find_chapter_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
|
||||
"""Find all BK chapter bookmark tags (p17) in the BK range.
|
||||
|
||||
Returns list of (paragraph_index, raw_title_text).
|
||||
"""
|
||||
bookmarks = []
|
||||
for i in range(start_idx, end_idx):
|
||||
p = all_paragraphs[i]
|
||||
cls = get_css_class(p)
|
||||
if cls == CHAPTER_BOOKMARK_CLASS:
|
||||
text = decode_text(p).strip()
|
||||
if text and "Bk:" in text:
|
||||
bookmarks.append((i, text))
|
||||
return bookmarks
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Paragraph extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def extract_paragraphs(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
|
||||
"""Extract body paragraphs for a chapter section.
|
||||
|
||||
Collects:
|
||||
- p6, p12, p38 (body paragraphs — ScaGoudy decoded)
|
||||
- p9, p19, p59, p23, p36, p60 (verse lines — centered italic, ScaGoudy decoded)
|
||||
- p10, p61 (reference markers — ScaGoudy decoded)
|
||||
- p57, p58 (footnotes — ScaGoudy decoded)
|
||||
Skips:
|
||||
- p56 (book title), p17 (bookmarks), p5 (chapter titles), p2 (TOC), empties
|
||||
"""
|
||||
result = []
|
||||
|
||||
for i in range(start_idx, end_idx):
|
||||
p = all_paragraphs[i]
|
||||
cls = get_css_class(p)
|
||||
|
||||
# Skip empty paragraphs
|
||||
if is_empty(p):
|
||||
continue
|
||||
|
||||
# Skip known skip classes
|
||||
if cls in SKIP_CLASSES:
|
||||
continue
|
||||
|
||||
# Skip book title header
|
||||
if cls == BOOK_TITLE_CLASS:
|
||||
continue
|
||||
|
||||
# Skip chapter bookmark headers
|
||||
if cls == CHAPTER_BOOKMARK_CLASS:
|
||||
continue
|
||||
|
||||
# Skip chapter title headings (p5)
|
||||
if cls == CHAPTER_TITLE_CLASS:
|
||||
continue
|
||||
|
||||
# Skip TOC entries (p2)
|
||||
if cls == TOC_CLASS:
|
||||
continue
|
||||
|
||||
# Body paragraphs (ScaGoudy decoded)
|
||||
if cls in BODY_CLASSES:
|
||||
html = decode_html_content(p, apply_scagoudy=True)
|
||||
if html:
|
||||
result.append(html)
|
||||
continue
|
||||
|
||||
# Sanskrit verse lines (centered italic, ScaGoudy decoded)
|
||||
if cls in VERSE_CLASSES:
|
||||
html = decode_html_content_as_verse(p)
|
||||
if html:
|
||||
result.append(html)
|
||||
continue
|
||||
|
||||
# Reference markers (ScaGoudy decoded)
|
||||
if cls in REFERENCE_CLASSES:
|
||||
html = decode_html_content(p, apply_scagoudy=True)
|
||||
if html:
|
||||
result.append(html)
|
||||
continue
|
||||
|
||||
# Footnote classes (ScaGoudy decoded)
|
||||
if cls in FOOTNOTE_CLASSES:
|
||||
html = decode_html_content(p, apply_scagoudy=True)
|
||||
if html:
|
||||
result.append(html)
|
||||
continue
|
||||
|
||||
# Fallback: try to extract content from unknown classes
|
||||
html = decode_html_content(p, apply_scagoudy=True)
|
||||
if html:
|
||||
print(f" ⚠ Unknown class '{cls}' at paragraph {i}: {html[:80]}...")
|
||||
result.append(html)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main extraction logic
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def extract_bk(soup: BeautifulSoup) -> dict:
|
||||
"""Extract the entire Bhagavāner Kathā from the parsed HTML.
|
||||
|
||||
Returns a dict with:
|
||||
"chapters": list of chapter dicts
|
||||
"chapter_listing": list of {number, title} for intro.json
|
||||
"""
|
||||
all_paragraphs = list(soup.find_all("p"))
|
||||
|
||||
# Step 1: Find BK boundaries
|
||||
bk_start, bk_end = find_bk_boundaries(all_paragraphs)
|
||||
print(f" BK content: paragraphs {bk_start} to {bk_end}")
|
||||
print(f" Total paragraphs in range: {bk_end - bk_start}")
|
||||
|
||||
# Step 2: Find all chapter bookmarks (p17 with "Bk:")
|
||||
bookmarks = find_chapter_bookmarks(all_paragraphs, bk_start, bk_end)
|
||||
print(f" Found {len(bookmarks)} BK chapter bookmarks:")
|
||||
for pos, title in bookmarks:
|
||||
print(f" Paragraph {pos}: {title}")
|
||||
|
||||
# Step 3: Map bookmarks to chapters with proper titles
|
||||
# Expected: 2 chapters
|
||||
chapter_titles_map = {
|
||||
1: "This World of Misery",
|
||||
2: "The Cause of Misery",
|
||||
}
|
||||
|
||||
chapter_bookmarks = [] # (para_idx, chapter_num, normalized_title)
|
||||
for idx, (pos, _raw_title) in enumerate(bookmarks):
|
||||
ch_num = idx + 1
|
||||
ch_title = chapter_titles_map.get(ch_num, f"Chapter {ch_num}")
|
||||
chapter_bookmarks.append((pos, ch_num, ch_title))
|
||||
|
||||
print(f"\n Mapped {len(chapter_bookmarks)} chapters:")
|
||||
for pos, num, title in chapter_bookmarks:
|
||||
print(f" Chapter {num}: {title} (paragraph {pos})")
|
||||
|
||||
# Step 4: Extract chapters
|
||||
chapters = []
|
||||
for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_bookmarks):
|
||||
# Content starts after the bookmark line (skip p17 and p5 heading)
|
||||
content_start = ch_pos + 1
|
||||
|
||||
# Content ends at next chapter bookmark or bk_end
|
||||
if idx + 1 < len(chapter_bookmarks):
|
||||
content_end = chapter_bookmarks[idx + 1][0]
|
||||
else:
|
||||
content_end = bk_end
|
||||
|
||||
paragraphs = extract_paragraphs(all_paragraphs, content_start, content_end)
|
||||
|
||||
ch = {
|
||||
"book": "bk",
|
||||
"chapter": ch_num,
|
||||
"title": ch_title,
|
||||
"type": "chapter",
|
||||
"paragraphs": paragraphs,
|
||||
}
|
||||
chapters.append(ch)
|
||||
|
||||
# Step 5: Build chapter listing for intro.json
|
||||
chapter_listing = [
|
||||
{"number": ch["chapter"], "title": ch["title"]} for ch in chapters
|
||||
]
|
||||
|
||||
return {
|
||||
"chapters": chapters,
|
||||
"chapter_listing": chapter_listing,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Output
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def write_json(data, filepath):
|
||||
"""Write data as formatted JSON to filepath, creating directories as needed."""
|
||||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||||
with open(filepath, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html")
|
||||
output_base = os.path.join(base_dir, "lib", "jsons", "books", "bk")
|
||||
|
||||
if not os.path.exists(html_path):
|
||||
print(f"ERROR: {html_path} not found!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Reading {html_path}...")
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
soup = BeautifulSoup(f, "html.parser")
|
||||
|
||||
print("Extracting Bhagavāner Kathā...\n")
|
||||
result = extract_bk(soup)
|
||||
|
||||
chapters = result["chapters"]
|
||||
chapter_listing = result["chapter_listing"]
|
||||
|
||||
# --- Write intro.json ---
|
||||
intro_meta = {
|
||||
"book": "bk",
|
||||
"title": "Bhagavāner Kathā",
|
||||
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
||||
"year": None,
|
||||
"description": (
|
||||
"Early Bengali writings on the topics of Godhead "
|
||||
"— part of 'Renunciation through Wisdom' (Vairāgya-vidyā)."
|
||||
),
|
||||
"totalChapters": 2,
|
||||
"chapters": chapter_listing,
|
||||
}
|
||||
intro_path = os.path.join(output_base, "intro.json")
|
||||
write_json(intro_meta, intro_path)
|
||||
print(f"\n Written: {intro_path}")
|
||||
|
||||
# --- Write chapters ---
|
||||
print(f"\n--- Chapters ---")
|
||||
for ch in chapters:
|
||||
ch_num_str = str(ch["chapter"]).zfill(2)
|
||||
filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
|
||||
write_json(ch, filepath)
|
||||
print(
|
||||
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
|
||||
f"{ch['title']}"
|
||||
)
|
||||
|
||||
# --- Summary ---
|
||||
total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
|
||||
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f" EXTRACTION SUMMARY")
|
||||
print(f"{'=' * 60}")
|
||||
print(f" Chapters: {len(chapters)}")
|
||||
for ch in chapters:
|
||||
print(
|
||||
f" Ch {ch['chapter']}: {len(ch['paragraphs'])} paragraphs — {ch['title']}"
|
||||
)
|
||||
print(f" Total paragraphs: {total_chapter_paras}")
|
||||
if chapters:
|
||||
print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
|
||||
print(f" Output directory: {output_base}/")
|
||||
|
||||
# --- Samples ---
|
||||
for ch in chapters:
|
||||
ch_num = ch["chapter"]
|
||||
print(f"\n--- Sample: Chapter {ch_num} ({ch['title']}) ---")
|
||||
for i, p in enumerate(ch["paragraphs"][:3]):
|
||||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
||||
print(f" [{i}] {preview}...")
|
||||
if len(ch["paragraphs"]) > 3:
|
||||
print(f" ... ({len(ch['paragraphs']) - 3} more paragraphs)")
|
||||
|
||||
# --- Quality Checks ---
|
||||
print(f"\n--- Quality Checks ---")
|
||||
|
||||
# Check all 2 chapters present
|
||||
expected_chapters = set(range(1, 3))
|
||||
actual_chapters = set(ch["chapter"] for ch in chapters)
|
||||
missing = expected_chapters - actual_chapters
|
||||
extra = actual_chapters - expected_chapters
|
||||
if missing:
|
||||
print(f" ⚠ Missing chapters: {sorted(missing)}")
|
||||
if extra:
|
||||
print(f" ⚠ Extra chapters: {sorted(extra)}")
|
||||
if not missing and not extra:
|
||||
print(f" ✅ All 2 chapters present")
|
||||
|
||||
# Check empty chapters
|
||||
empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
|
||||
if empty_chapters:
|
||||
print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
|
||||
else:
|
||||
print(f" ✅ No empty chapters")
|
||||
|
||||
# Check ScaGoudy decoding worked — look for common ScaGoudy characters
|
||||
scagoudy_chars = set("äåñöëòçìïüéàù")
|
||||
all_text = ""
|
||||
for ch in chapters:
|
||||
all_text += " ".join(ch["paragraphs"])
|
||||
remaining_scagoudy = [c for c in all_text if c in scagoudy_chars]
|
||||
if remaining_scagoudy:
|
||||
# Count occurrences
|
||||
from collections import Counter
|
||||
|
||||
counts = Counter(remaining_scagoudy)
|
||||
print(
|
||||
f" ⚠ Remaining ScaGoudy characters found (may be false positives): {dict(counts)}"
|
||||
)
|
||||
else:
|
||||
print(f" ✅ No ScaGoudy characters remaining — decoding looks clean")
|
||||
|
||||
# Check for IAST characters (positive signal)
|
||||
iast_chars = set("āṛṣṭṇḍśṅñūīṁḥ")
|
||||
iast_found = [c for c in all_text if c in iast_chars]
|
||||
if iast_found:
|
||||
from collections import Counter
|
||||
|
||||
counts = Counter(iast_found)
|
||||
print(f" ✅ IAST Unicode characters present: {dict(counts)}")
|
||||
else:
|
||||
print(f" ⚠ No IAST Unicode characters found — decoding may have failed")
|
||||
|
||||
# Check chapter 1 has [RTW 1.1] reference
|
||||
if chapters:
|
||||
ch1_text = " ".join(chapters[0]["paragraphs"])
|
||||
if "RTW" in ch1_text and "1.1" in ch1_text:
|
||||
print(f" ✅ '[RTW 1.1]' reference found in Chapter 1")
|
||||
else:
|
||||
print(f" ⚠ '[RTW 1.1]' reference NOT found in Chapter 1")
|
||||
|
||||
# Check chapter 2 has [RTW 1.2] reference
|
||||
if len(chapters) >= 2:
|
||||
ch2_text = " ".join(chapters[1]["paragraphs"])
|
||||
if "RTW" in ch2_text and "1.2" in ch2_text:
|
||||
print(f" ✅ '[RTW 1.2]' reference found in Chapter 2")
|
||||
else:
|
||||
print(f" ⚠ '[RTW 1.2]' reference NOT found in Chapter 2")
|
||||
|
||||
# Check for content leak from next work (Indian Bhagavatams)
|
||||
if "Indian Bhagavatam" in all_text:
|
||||
print(f" ⚠ Content from next work (Indian Bhagavatams) may have leaked!")
|
||||
else:
|
||||
print(f" ✅ No content leak from next work detected")
|
||||
|
||||
# Check Kṛṣṇa appears (decoded from Kåñëa)
|
||||
if "Kṛṣṇa" in all_text:
|
||||
print(f" ✅ 'Kṛṣṇa' properly decoded (from 'Kåñëa')")
|
||||
else:
|
||||
print(f" ⚠ 'Kṛṣṇa' not found — ScaGoudy decoding may need review")
|
||||
|
||||
# Check last chapter ends cleanly
|
||||
if chapters:
|
||||
last_ch = chapters[-1]
|
||||
last_para = last_ch["paragraphs"][-1] if last_ch["paragraphs"] else ""
|
||||
last_preview = re.sub(r"<[^>]+>", "", last_para)[:100]
|
||||
print(f' ℹ Last paragraph of Ch {last_ch["chapter"]}: "{last_preview}..."')
|
||||
|
||||
print(f"\nDone! ✨")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,444 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract Śrī Caitanya-bhāgavata Commentary from others-before-1977.html.
|
||||
|
||||
The book has 4 chapters (no intro segments):
|
||||
1. Chapter 1
|
||||
2. Lord Caitanya's Appearance
|
||||
3. Chapter 3
|
||||
4. Chapter 4
|
||||
|
||||
Source: public/html/others-before-1977.html, lines ~4139–4180.
|
||||
Start: <p class="p3"><b>Çré Caitanya-bhägavata-Commentary</b></p>
|
||||
End: <p class="p2">Bhagaväner kathä [Original Documents]</p> (stop before)
|
||||
|
||||
Chapter boundaries: <p class="p17"> tags containing "Caitanya-bhägavata"
|
||||
|
||||
CSS classes:
|
||||
p3 — Book title header (skip)
|
||||
p17 — Chapter bookmark headers (boundary detection, skip from output)
|
||||
p5 — Chapter title headings (skip — we derive titles ourselves)
|
||||
p6 — Body paragraphs (MAIN content)
|
||||
p38 — Body paragraph variant
|
||||
p55 — Body paragraph variant (last paragraph of chapter 4)
|
||||
p9 — End-of-chapter markers like "(End Chapter One)" — include
|
||||
|
||||
Output:
|
||||
lib/jsons/books/cbhc/intro.json — Book metadata
|
||||
lib/jsons/books/cbhc/chapters/01.json — Chapter 1
|
||||
lib/jsons/books/cbhc/chapters/02.json — Chapter 2
|
||||
lib/jsons/books/cbhc/chapters/03.json — Chapter 3
|
||||
lib/jsons/books/cbhc/chapters/04.json — Chapter 4
|
||||
|
||||
Usage:
|
||||
python3 scripts/cbhc_extract.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def decode_html_content(tag: Tag) -> str:
|
||||
"""Extract inner HTML from a tag, preserve formatting tags, strip spans."""
|
||||
inner = "".join(str(child) for child in tag.children)
|
||||
# Remove span tags with CSS classes (RTF conversion artifacts)
|
||||
inner = re.sub(
|
||||
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
|
||||
)
|
||||
# Clean up Apple-tab-span leftovers (tab characters)
|
||||
inner = inner.replace("\t", " ")
|
||||
# Normalize whitespace
|
||||
inner = re.sub(r"\s+", " ", inner).strip()
|
||||
return inner
|
||||
|
||||
|
||||
def decode_text(tag: Tag) -> str:
|
||||
"""Get plain text from a tag."""
|
||||
return tag.get_text().strip()
|
||||
|
||||
|
||||
def get_css_class(tag: Tag) -> str:
|
||||
"""Get the first CSS class of a tag, or empty string."""
|
||||
classes = tag.get("class", [])
|
||||
return classes[0] if classes else ""
|
||||
|
||||
|
||||
def is_empty(tag: Tag) -> bool:
|
||||
"""Check if a tag is empty or whitespace-only."""
|
||||
text = tag.get_text().strip()
|
||||
return not text or text == "\xa0"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CSS class categorization for CBHC
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Book title header
|
||||
BOOK_TITLE_CLASS = "p3"
|
||||
|
||||
# Chapter bookmark headers (used for boundary detection)
|
||||
CHAPTER_BOOKMARK_CLASS = "p17"
|
||||
|
||||
# Chapter title (bold, e.g., "Chapter 1") — skip from body
|
||||
CHAPTER_TITLE_CLASS = "p5"
|
||||
|
||||
# Body paragraph classes
|
||||
BODY_CLASSES = frozenset(["p6", "p38", "p55"])
|
||||
|
||||
# End-of-chapter markers — include as a paragraph
|
||||
END_MARKER_CLASS = "p9"
|
||||
|
||||
# Classes to skip entirely
|
||||
SKIP_CLASSES = frozenset(["p43", "p24"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Boundary detection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def find_cbhc_boundaries(all_paragraphs: list) -> tuple:
|
||||
"""Find the start and end paragraph indices of the CBHC content.
|
||||
|
||||
Start: <p class="p3"> containing "Caitanya-bhägavata-Commentary"
|
||||
End: <p class="p2"> containing "Bhagaväner kathä" (next work)
|
||||
"""
|
||||
start_idx = None
|
||||
end_idx = len(all_paragraphs)
|
||||
|
||||
for i, p in enumerate(all_paragraphs):
|
||||
cls = get_css_class(p)
|
||||
text = decode_text(p)
|
||||
|
||||
# Find start: p3 containing "Caitanya-bhägavata"
|
||||
if start_idx is None:
|
||||
if cls == BOOK_TITLE_CLASS and "Caitanya-bh" in text:
|
||||
start_idx = i
|
||||
continue
|
||||
continue
|
||||
|
||||
# Find end: p2 containing "Bhagaväner" (next work boundary)
|
||||
if cls == "p2" and "Bhag" in text and "kath" in text:
|
||||
end_idx = i
|
||||
break
|
||||
|
||||
if start_idx is None:
|
||||
print(
|
||||
"ERROR: Could not find 'Çré Caitanya-bhägavata-Commentary' "
|
||||
"in others-before-1977.html!"
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
return start_idx, end_idx
|
||||
|
||||
|
||||
def find_chapter_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
|
||||
"""Find all CBHC chapter bookmark tags (p17) in the CBHC range.
|
||||
|
||||
Returns list of (paragraph_index, raw_title_text).
|
||||
"""
|
||||
bookmarks = []
|
||||
for i in range(start_idx, end_idx):
|
||||
p = all_paragraphs[i]
|
||||
cls = get_css_class(p)
|
||||
if cls == CHAPTER_BOOKMARK_CLASS:
|
||||
text = decode_text(p).strip()
|
||||
if text and "Caitanya-bh" in text:
|
||||
bookmarks.append((i, text))
|
||||
return bookmarks
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Paragraph extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def extract_paragraphs(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
|
||||
"""Extract body paragraphs for a chapter section.
|
||||
|
||||
Collects p6, p38, p55, and p9 (end markers).
|
||||
Skips p3 (book header), p17 (bookmarks), p5 (chapter titles), p2, empties.
|
||||
"""
|
||||
result = []
|
||||
|
||||
for i in range(start_idx, end_idx):
|
||||
p = all_paragraphs[i]
|
||||
cls = get_css_class(p)
|
||||
|
||||
# Skip empty paragraphs
|
||||
if is_empty(p):
|
||||
continue
|
||||
|
||||
# Skip known skip classes
|
||||
if cls in SKIP_CLASSES:
|
||||
continue
|
||||
|
||||
# Skip book title header
|
||||
if cls == BOOK_TITLE_CLASS:
|
||||
continue
|
||||
|
||||
# Skip chapter bookmark headers
|
||||
if cls == CHAPTER_BOOKMARK_CLASS:
|
||||
continue
|
||||
|
||||
# Skip chapter title headings (p5)
|
||||
if cls == CHAPTER_TITLE_CLASS:
|
||||
continue
|
||||
|
||||
# Skip p2 (TOC entries)
|
||||
if cls == "p2":
|
||||
continue
|
||||
|
||||
# Body paragraphs
|
||||
if cls in BODY_CLASSES:
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
result.append(html)
|
||||
continue
|
||||
|
||||
# End-of-chapter markers (p9) — include as a paragraph
|
||||
if cls == END_MARKER_CLASS:
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
result.append(html)
|
||||
continue
|
||||
|
||||
# Fallback: try to extract content from unknown classes
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
print(f" ⚠ Unknown class '{cls}' at paragraph {i}: {html[:80]}...")
|
||||
result.append(html)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main extraction logic
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def extract_cbhc(soup: BeautifulSoup) -> dict:
|
||||
"""Extract the entire Śrī Caitanya-bhāgavata Commentary from the parsed HTML.
|
||||
|
||||
Returns a dict with:
|
||||
"chapters": list of chapter dicts
|
||||
"chapter_listing": list of {number, title} for intro.json
|
||||
"""
|
||||
all_paragraphs = list(soup.find_all("p"))
|
||||
|
||||
# Step 1: Find CBHC boundaries
|
||||
cbhc_start, cbhc_end = find_cbhc_boundaries(all_paragraphs)
|
||||
print(f" CBHC content: paragraphs {cbhc_start} to {cbhc_end}")
|
||||
print(f" Total paragraphs in range: {cbhc_end - cbhc_start}")
|
||||
|
||||
# Step 2: Find all chapter bookmarks (p17)
|
||||
bookmarks = find_chapter_bookmarks(all_paragraphs, cbhc_start, cbhc_end)
|
||||
print(f" Found {len(bookmarks)} CBHC chapter bookmarks:")
|
||||
for pos, title in bookmarks:
|
||||
print(f" Paragraph {pos}: {title}")
|
||||
|
||||
# Step 3: Map bookmarks to chapters with proper titles
|
||||
# Expected: 4 chapters
|
||||
# We derive proper titles from the p5 heading that follows each bookmark
|
||||
chapter_titles_map = {
|
||||
1: "Chapter 1",
|
||||
2: "Lord Caitanya's Appearance",
|
||||
3: "Chapter 3",
|
||||
4: "Chapter 4",
|
||||
}
|
||||
|
||||
chapter_bookmarks = [] # (para_idx, chapter_num, normalized_title)
|
||||
for idx, (pos, _raw_title) in enumerate(bookmarks):
|
||||
ch_num = idx + 1
|
||||
ch_title = chapter_titles_map.get(ch_num, f"Chapter {ch_num}")
|
||||
chapter_bookmarks.append((pos, ch_num, ch_title))
|
||||
|
||||
print(f"\n Mapped {len(chapter_bookmarks)} chapters:")
|
||||
for pos, num, title in chapter_bookmarks:
|
||||
print(f" Chapter {num}: {title} (paragraph {pos})")
|
||||
|
||||
# Step 4: Extract chapters
|
||||
chapters = []
|
||||
for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_bookmarks):
|
||||
# Content starts after the bookmark line (skip p17 and p5 heading)
|
||||
content_start = ch_pos + 1
|
||||
|
||||
# Content ends at next chapter bookmark or cbhc_end
|
||||
if idx + 1 < len(chapter_bookmarks):
|
||||
content_end = chapter_bookmarks[idx + 1][0]
|
||||
else:
|
||||
content_end = cbhc_end
|
||||
|
||||
paragraphs = extract_paragraphs(all_paragraphs, content_start, content_end)
|
||||
|
||||
ch = {
|
||||
"book": "cbhc",
|
||||
"chapter": ch_num,
|
||||
"title": ch_title,
|
||||
"type": "chapter",
|
||||
"paragraphs": paragraphs,
|
||||
}
|
||||
chapters.append(ch)
|
||||
|
||||
# Step 5: Build chapter listing for intro.json
|
||||
chapter_listing = [
|
||||
{"number": ch["chapter"], "title": ch["title"]} for ch in chapters
|
||||
]
|
||||
|
||||
return {
|
||||
"chapters": chapters,
|
||||
"chapter_listing": chapter_listing,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Output
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def write_json(data, filepath):
|
||||
"""Write data as formatted JSON to filepath, creating directories as needed."""
|
||||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||||
with open(filepath, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html")
|
||||
output_base = os.path.join(base_dir, "lib", "jsons", "books", "cbhc")
|
||||
|
||||
if not os.path.exists(html_path):
|
||||
print(f"ERROR: {html_path} not found!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Reading {html_path}...")
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
soup = BeautifulSoup(f, "html.parser")
|
||||
|
||||
print("Extracting Śrī Caitanya-bhāgavata Commentary...\n")
|
||||
result = extract_cbhc(soup)
|
||||
|
||||
chapters = result["chapters"]
|
||||
chapter_listing = result["chapter_listing"]
|
||||
|
||||
# --- Write intro.json ---
|
||||
intro_meta = {
|
||||
"book": "cbhc",
|
||||
"title": "Śrī Caitanya-bhāgavata Commentary",
|
||||
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
||||
"year": None,
|
||||
"description": (
|
||||
"Commentary on Śrī Caitanya-bhāgavata by Vṛndāvana dāsa Ṭhākura "
|
||||
"— covering the first four chapters of Lord Caitanya's pastimes."
|
||||
),
|
||||
"totalChapters": 4,
|
||||
"chapters": chapter_listing,
|
||||
}
|
||||
intro_path = os.path.join(output_base, "intro.json")
|
||||
write_json(intro_meta, intro_path)
|
||||
print(f"\n Written: {intro_path}")
|
||||
|
||||
# --- Write chapters ---
|
||||
print(f"\n--- Chapters ---")
|
||||
for ch in chapters:
|
||||
ch_num_str = str(ch["chapter"]).zfill(2)
|
||||
filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
|
||||
write_json(ch, filepath)
|
||||
print(
|
||||
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
|
||||
f"{ch['title']}"
|
||||
)
|
||||
|
||||
# --- Summary ---
|
||||
total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
|
||||
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f" EXTRACTION SUMMARY")
|
||||
print(f"{'=' * 60}")
|
||||
print(f" Chapters: {len(chapters)}")
|
||||
for ch in chapters:
|
||||
print(
|
||||
f" Ch {ch['chapter']}: {len(ch['paragraphs'])} paragraphs — {ch['title']}"
|
||||
)
|
||||
print(f" Total paragraphs: {total_chapter_paras}")
|
||||
if chapters:
|
||||
print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
|
||||
print(f" Output directory: {output_base}/")
|
||||
|
||||
# --- Samples ---
|
||||
for ch in chapters:
|
||||
ch_num = ch["chapter"]
|
||||
print(f"\n--- Sample: Chapter {ch_num} ({ch['title']}) ---")
|
||||
for i, p in enumerate(ch["paragraphs"][:3]):
|
||||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
||||
print(f" [{i}] {preview}...")
|
||||
if len(ch["paragraphs"]) > 3:
|
||||
print(f" ... ({len(ch['paragraphs']) - 3} more paragraphs)")
|
||||
|
||||
# --- Quality Checks ---
|
||||
print(f"\n--- Quality Checks ---")
|
||||
|
||||
# Check all 4 chapters present
|
||||
expected_chapters = set(range(1, 5))
|
||||
actual_chapters = set(ch["chapter"] for ch in chapters)
|
||||
missing = expected_chapters - actual_chapters
|
||||
extra = actual_chapters - expected_chapters
|
||||
if missing:
|
||||
print(f" ⚠ Missing chapters: {sorted(missing)}")
|
||||
if extra:
|
||||
print(f" ⚠ Extra chapters: {sorted(extra)}")
|
||||
if not missing and not extra:
|
||||
print(f" ✅ All 4 chapters present")
|
||||
|
||||
# Check empty chapters
|
||||
empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
|
||||
if empty_chapters:
|
||||
print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
|
||||
else:
|
||||
print(f" ✅ No empty chapters")
|
||||
|
||||
# Check chapter 1 has "(End Chapter One)"
|
||||
if chapters:
|
||||
ch1_text = " ".join(chapters[0]["paragraphs"])
|
||||
if "End Chapter One" in ch1_text:
|
||||
print(f" ✅ '(End Chapter One)' end marker found in Chapter 1")
|
||||
else:
|
||||
print(f" ⚠ '(End Chapter One)' end marker NOT found in Chapter 1")
|
||||
|
||||
# Check chapter 2 mentions Lord Caitanya's appearance
|
||||
if len(chapters) >= 2:
|
||||
ch2_text = " ".join(chapters[1]["paragraphs"])
|
||||
if "appearance" in ch2_text.lower() or "born" in ch2_text.lower():
|
||||
print(f" ✅ Chapter 2 contains appearance/birth content")
|
||||
else:
|
||||
print(f" ⚠ Chapter 2 may not contain appearance content")
|
||||
|
||||
# Check for content leak from next work (Bhagaväner kathä)
|
||||
all_text = ""
|
||||
for ch in chapters:
|
||||
all_text += " ".join(ch["paragraphs"])
|
||||
if "Original Documents" in all_text:
|
||||
print(f" ⚠ Content from next work (Bhagaväner kathä) may have leaked!")
|
||||
else:
|
||||
print(f" ✅ No content leak from next work detected")
|
||||
|
||||
# Check last chapter ends cleanly
|
||||
if chapters:
|
||||
last_ch = chapters[-1]
|
||||
last_para = last_ch["paragraphs"][-1] if last_ch["paragraphs"] else ""
|
||||
last_preview = re.sub(r"<[^>]+>", "", last_para)[:100]
|
||||
print(f' ℹ Last paragraph of Ch {last_ch["chapter"]}: "{last_preview}..."')
|
||||
|
||||
print(f"\nDone! ✨")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,675 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract five "Essays on Śrī Caitanya-caritāmṛta" from others-before-1977.html.
|
||||
|
||||
Essay 1: "The Spiritual Masters" (cc-essays, order 1, 1960)
|
||||
Essay 2: "The Absolute Truth" (cc-essays, order 2, 1957)
|
||||
Essay 3: "The Causes of Lord Caitanya's Incarnation" (cc-essays, order 3, 1957)
|
||||
Essay 4: "The Confidential Reasons for Lord Caitanya's Appearance" (cc-essays, order 4, 1957)
|
||||
Essay 5: "The Glories of Lord Nityānanda" (cc-essays, order 5, 1957)
|
||||
|
||||
All five are plain English (NO ScaGoudy decoding needed).
|
||||
|
||||
Output:
|
||||
lib/jsons/essays/the-spiritual-masters.json
|
||||
lib/jsons/essays/the-absolute-truth.json
|
||||
lib/jsons/essays/causes-of-lord-caitanyas-incarnation.json
|
||||
lib/jsons/essays/confidential-reasons-for-lord-caitanyas-appearance.json
|
||||
lib/jsons/essays/glories-of-lord-nityananda.json
|
||||
|
||||
Usage:
|
||||
python3 scripts/cc_essays_extract.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def get_css_class(tag: Tag) -> str:
|
||||
"""Get the first CSS class of a tag, or empty string."""
|
||||
classes = tag.get("class", [])
|
||||
return classes[0] if classes else ""
|
||||
|
||||
|
||||
def is_empty(tag: Tag) -> bool:
|
||||
"""Check if a tag is empty or whitespace-only."""
|
||||
text = tag.get_text().strip()
|
||||
return not text or text == "\xa0"
|
||||
|
||||
|
||||
def clean_html_content(tag: Tag) -> str:
|
||||
"""Extract inner HTML from a tag, strip span class artifacts, preserve <i>, <b>, <br>."""
|
||||
inner = "".join(str(child) for child in tag.children)
|
||||
# Remove span tags with CSS classes (RTF conversion artifacts) but keep their content
|
||||
inner = re.sub(
|
||||
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
|
||||
)
|
||||
# Remove any remaining bare <span> or </span> tags without classes
|
||||
inner = re.sub(r"</?span[^>]*>", "", inner)
|
||||
# Clean up Apple-tab-span leftovers (tab characters)
|
||||
inner = inner.replace("\t", " ")
|
||||
# Normalize whitespace (but preserve <br> tags)
|
||||
inner = re.sub(r"\s+", " ", inner).strip()
|
||||
return inner
|
||||
|
||||
|
||||
def write_json(data, filepath):
|
||||
"""Write data as formatted JSON to filepath, creating directories as needed."""
|
||||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||||
with open(filepath, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def is_divider(tag: Tag) -> bool:
|
||||
"""Check if a p9 tag is just a divider line (underscores) or a page header."""
|
||||
text = tag.get_text().strip()
|
||||
# Dividers are lines of underscores
|
||||
if re.match(r"^_+$", text):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_page_header(tag: Tag) -> bool:
|
||||
"""Check if a p9 tag is a page header like 'Essays on Shri Chaitanya Charitamrita Page 30'."""
|
||||
text = tag.get_text().strip()
|
||||
if re.search(r"Page\s+\d+", text):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_editor_note(tag: Tag) -> bool:
|
||||
"""Check if a paragraph is an editor note like '[This file has been proofread...]'."""
|
||||
text = tag.get_text().strip()
|
||||
if text.startswith("[") and (
|
||||
"proofread" in text.lower() or "—Ed" in text or "- Ed" in text or ".—Ed" in text
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Essay configurations
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
AUTHOR = "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda"
|
||||
|
||||
ESSAYS = [
|
||||
{
|
||||
"slug": "the-spiritual-masters",
|
||||
"title": "The Spiritual Masters",
|
||||
"author": AUTHOR,
|
||||
"year": 1960,
|
||||
"group": "cc-essays",
|
||||
"order": 1,
|
||||
},
|
||||
{
|
||||
"slug": "the-absolute-truth",
|
||||
"title": "The Absolute Truth",
|
||||
"author": AUTHOR,
|
||||
"year": 1957,
|
||||
"group": "cc-essays",
|
||||
"order": 2,
|
||||
},
|
||||
{
|
||||
"slug": "causes-of-lord-caitanyas-incarnation",
|
||||
"title": "The Causes of Lord Caitanya's Incarnation",
|
||||
"author": AUTHOR,
|
||||
"year": 1957,
|
||||
"group": "cc-essays",
|
||||
"order": 3,
|
||||
},
|
||||
{
|
||||
"slug": "confidential-reasons-for-lord-caitanyas-appearance",
|
||||
"title": "The Confidential Reasons for Lord Caitanya's Appearance",
|
||||
"author": AUTHOR,
|
||||
"year": 1957,
|
||||
"group": "cc-essays",
|
||||
"order": 4,
|
||||
},
|
||||
{
|
||||
"slug": "glories-of-lord-nityananda",
|
||||
"title": "The Glories of Lord Nityānanda",
|
||||
"author": AUTHOR,
|
||||
"year": 1957,
|
||||
"group": "cc-essays",
|
||||
"order": 5,
|
||||
},
|
||||
]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Extraction logic
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def parse_all_paragraphs(html_path: str) -> list[Tag]:
|
||||
"""Parse the entire HTML and return all <p> tags."""
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
soup = BeautifulSoup(content, "html.parser")
|
||||
return list(soup.find_all("p"))
|
||||
|
||||
|
||||
def find_tag_index(tags: list[Tag], start_from: int, predicate) -> int | None:
|
||||
"""Find index of first tag matching predicate, starting from start_from."""
|
||||
for i in range(start_from, len(tags)):
|
||||
if predicate(tags[i]):
|
||||
return i
|
||||
return None
|
||||
|
||||
|
||||
def extract_body_paragraphs(
|
||||
tags: list[Tag], start_idx: int, end_idx: int, include_p9_content: bool = True
|
||||
) -> list[str]:
|
||||
"""Extract body paragraphs from tags[start_idx] to tags[end_idx] (exclusive).
|
||||
|
||||
Includes p6 paragraphs (main body) and optionally p9 paragraphs that
|
||||
contain actual content (Sanskrit verses, section markers) but skips
|
||||
p9 paragraphs that are dividers or page headers.
|
||||
"""
|
||||
paragraphs = []
|
||||
body_classes = {"p6"}
|
||||
if include_p9_content:
|
||||
body_classes.add("p9")
|
||||
|
||||
for i in range(start_idx, end_idx):
|
||||
tag = tags[i]
|
||||
css = get_css_class(tag)
|
||||
|
||||
if css not in body_classes:
|
||||
continue
|
||||
|
||||
if is_empty(tag):
|
||||
continue
|
||||
|
||||
# For p9 tags, skip dividers and page headers
|
||||
if css == "p9":
|
||||
if is_divider(tag) or is_page_header(tag):
|
||||
continue
|
||||
|
||||
html = clean_html_content(tag)
|
||||
if html and html != " ":
|
||||
paragraphs.append(html)
|
||||
|
||||
return paragraphs
|
||||
|
||||
|
||||
def extract_body_paragraphs_extended(
|
||||
tags: list[Tag],
|
||||
start_idx: int,
|
||||
end_idx: int,
|
||||
body_classes: set[str] | None = None,
|
||||
skip_editor_notes: bool = False,
|
||||
) -> list[str]:
|
||||
"""Extract body paragraphs from tags[start_idx] to tags[end_idx] (exclusive).
|
||||
|
||||
Like extract_body_paragraphs but with configurable body classes and
|
||||
optional editor-note skipping.
|
||||
"""
|
||||
if body_classes is None:
|
||||
body_classes = {"p6"}
|
||||
|
||||
paragraphs = []
|
||||
|
||||
for i in range(start_idx, end_idx):
|
||||
tag = tags[i]
|
||||
css = get_css_class(tag)
|
||||
|
||||
if css not in body_classes:
|
||||
continue
|
||||
|
||||
if is_empty(tag):
|
||||
continue
|
||||
|
||||
# For p9 tags, skip dividers and page headers
|
||||
if css == "p9":
|
||||
if is_divider(tag) or is_page_header(tag):
|
||||
continue
|
||||
|
||||
# Skip editor notes if requested
|
||||
if skip_editor_notes and is_editor_note(tag):
|
||||
continue
|
||||
|
||||
html = clean_html_content(tag)
|
||||
if html and html != " ":
|
||||
paragraphs.append(html)
|
||||
|
||||
return paragraphs
|
||||
|
||||
|
||||
def extract_essay_1(tags: list[Tag]) -> dict:
|
||||
"""Extract 'The Spiritual Masters' essay.
|
||||
|
||||
Start: <p class="p8"><b>(The Spiritual Masters).</b></p>
|
||||
Body: p6 and p9 paragraphs after start
|
||||
End: Next <p class="p10"> (contains [BTG 1960])
|
||||
"""
|
||||
# Find the title tag
|
||||
start_idx = find_tag_index(
|
||||
tags,
|
||||
0,
|
||||
lambda t: get_css_class(t) == "p8" and "Spiritual Masters" in t.get_text(),
|
||||
)
|
||||
if start_idx is None:
|
||||
raise ValueError("Could not find Essay 1 start marker (p8 'Spiritual Masters')")
|
||||
|
||||
# Find the end marker: next p10
|
||||
end_idx = find_tag_index(tags, start_idx + 1, lambda t: get_css_class(t) == "p10")
|
||||
if end_idx is None:
|
||||
raise ValueError("Could not find Essay 1 end marker (p10)")
|
||||
|
||||
print(f" Essay 1: tags[{start_idx}] → tags[{end_idx}] (exclusive)")
|
||||
|
||||
# Body starts after the p8 title tag
|
||||
paragraphs = extract_body_paragraphs(tags, start_idx + 1, end_idx)
|
||||
|
||||
essay = dict(ESSAYS[0])
|
||||
essay["paragraphs"] = paragraphs
|
||||
return essay
|
||||
|
||||
|
||||
def extract_essay_2(tags: list[Tag]) -> dict:
|
||||
"""Extract 'The Absolute Truth' essay.
|
||||
|
||||
Start: <p class="p9">(Second Part)</p>
|
||||
Body: p6 paragraphs after start
|
||||
End: Next <p class="p11"> after start
|
||||
"""
|
||||
# Find the start marker
|
||||
start_idx = find_tag_index(
|
||||
tags,
|
||||
0,
|
||||
lambda t: get_css_class(t) == "p9" and "(Second Part)" in t.get_text(),
|
||||
)
|
||||
if start_idx is None:
|
||||
raise ValueError("Could not find Essay 2 start marker (p9 '(Second Part)')")
|
||||
|
||||
# Find the end marker: next p11
|
||||
end_idx = find_tag_index(tags, start_idx + 1, lambda t: get_css_class(t) == "p11")
|
||||
if end_idx is None:
|
||||
raise ValueError("Could not find Essay 2 end marker (p11)")
|
||||
|
||||
print(f" Essay 2: tags[{start_idx}] → tags[{end_idx}] (exclusive)")
|
||||
|
||||
# Body starts after the "(Second Part)" header
|
||||
paragraphs = extract_body_paragraphs(tags, start_idx + 1, end_idx)
|
||||
|
||||
essay = dict(ESSAYS[1])
|
||||
essay["paragraphs"] = paragraphs
|
||||
return essay
|
||||
|
||||
|
||||
def extract_essay_3(tags: list[Tag]) -> dict:
|
||||
"""Extract 'The Causes of Lord Caitanya's Incarnation' essay.
|
||||
|
||||
Start: <p class="p9"> containing "Essays on Shri Chaitanya Charitamrita. III"
|
||||
Body: p6 paragraphs after start
|
||||
End: Next <p class="p11"> after start.
|
||||
The <p class="p9">____</p> divider is skipped.
|
||||
But the <p class="p6">Thus ends the III part...</p> IS included.
|
||||
"""
|
||||
# Find the start marker — note the ". III" with period
|
||||
start_idx = find_tag_index(
|
||||
tags,
|
||||
0,
|
||||
lambda t: (
|
||||
get_css_class(t) == "p9"
|
||||
and "Essays on Shri Chaitanya Charitamrita. III" in t.get_text()
|
||||
),
|
||||
)
|
||||
if start_idx is None:
|
||||
raise ValueError(
|
||||
"Could not find Essay 3 start marker (p9 'Essays on Shri Chaitanya Charitamrita. III')"
|
||||
)
|
||||
|
||||
# Find the end marker: next p11
|
||||
end_idx = find_tag_index(tags, start_idx + 1, lambda t: get_css_class(t) == "p11")
|
||||
if end_idx is None:
|
||||
raise ValueError("Could not find Essay 3 end marker (p11)")
|
||||
|
||||
print(f" Essay 3: tags[{start_idx}] → tags[{end_idx}] (exclusive)")
|
||||
|
||||
# Body starts after the header tag.
|
||||
# We collect p6 paragraphs. The divider (p9 with underscores) is skipped
|
||||
# automatically by extract_body_paragraphs. The "Thus ends the III part..."
|
||||
# paragraph is a p6, so it will be included.
|
||||
paragraphs = extract_body_paragraphs(tags, start_idx + 1, end_idx)
|
||||
|
||||
essay = dict(ESSAYS[2])
|
||||
essay["paragraphs"] = paragraphs
|
||||
return essay
|
||||
|
||||
|
||||
def extract_essay_4(tags: list[Tag]) -> dict:
|
||||
"""Extract 'The Confidential Reasons for Lord Caitanya's Appearance' essay.
|
||||
|
||||
Start: <p class="p30"> containing "4th Chapter"
|
||||
Body: p6 paragraphs after start (skip editor note, skip p4/p5/p29/p30 headers)
|
||||
End: Next <p class="p4"> after start (next chapter header "Ädi-lélä, Chapter 5")
|
||||
"""
|
||||
# Find the start marker: p30 containing "4th Chapter"
|
||||
start_idx = find_tag_index(
|
||||
tags,
|
||||
0,
|
||||
lambda t: get_css_class(t) == "p30" and "4th Chapter" in t.get_text(),
|
||||
)
|
||||
if start_idx is None:
|
||||
raise ValueError("Could not find Essay 4 start marker (p30 '4th Chapter')")
|
||||
|
||||
# Find the end marker: next p4 after the start (next chapter header)
|
||||
end_idx = find_tag_index(tags, start_idx + 1, lambda t: get_css_class(t) == "p4")
|
||||
if end_idx is None:
|
||||
raise ValueError("Could not find Essay 4 end marker (p4)")
|
||||
|
||||
print(f" Essay 4: tags[{start_idx}] → tags[{end_idx}] (exclusive)")
|
||||
|
||||
# Collect p6 paragraphs, skipping editor notes
|
||||
paragraphs = extract_body_paragraphs_extended(
|
||||
tags,
|
||||
start_idx + 1,
|
||||
end_idx,
|
||||
body_classes={"p6"},
|
||||
skip_editor_notes=True,
|
||||
)
|
||||
|
||||
essay = dict(ESSAYS[3])
|
||||
essay["paragraphs"] = paragraphs
|
||||
return essay
|
||||
|
||||
|
||||
def extract_essay_5(tags: list[Tag]) -> dict:
|
||||
"""Extract 'The Glories of Lord Nityānanda' essay.
|
||||
|
||||
Start: <p class="p31"> containing "1.V" (i.e. "Shi Chaitanya Charitamrita 1.V")
|
||||
Body: p6 and p34 paragraphs after start (skip editor note, skip p4/p5/p31 headers,
|
||||
skip p9 divider underscores)
|
||||
End: Next <p class="p35"> after start ("Easy Journey To Other Planets")
|
||||
"""
|
||||
# Find the start marker: p31 containing "1.V"
|
||||
start_idx = find_tag_index(
|
||||
tags,
|
||||
0,
|
||||
lambda t: get_css_class(t) == "p31" and "1.V" in t.get_text(),
|
||||
)
|
||||
if start_idx is None:
|
||||
raise ValueError("Could not find Essay 5 start marker (p31 containing '1.V')")
|
||||
|
||||
# Find the end marker: next p35 after the start
|
||||
end_idx = find_tag_index(tags, start_idx + 1, lambda t: get_css_class(t) == "p35")
|
||||
if end_idx is None:
|
||||
raise ValueError("Could not find Essay 5 end marker (p35)")
|
||||
|
||||
print(f" Essay 5: tags[{start_idx}] → tags[{end_idx}] (exclusive)")
|
||||
|
||||
# Collect p6 and p34 paragraphs, skipping editor notes and p9 dividers
|
||||
paragraphs = extract_body_paragraphs_extended(
|
||||
tags,
|
||||
start_idx + 1,
|
||||
end_idx,
|
||||
body_classes={"p6", "p34"},
|
||||
skip_editor_notes=True,
|
||||
)
|
||||
|
||||
essay = dict(ESSAYS[4])
|
||||
essay["paragraphs"] = paragraphs
|
||||
return essay
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Quality checks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def quality_check(essay: dict, essay_num: int) -> None:
|
||||
"""Run quality checks on an extracted essay."""
|
||||
paragraphs = essay["paragraphs"]
|
||||
title = essay["title"]
|
||||
|
||||
print(f"\n--- Quality Checks: Essay {essay_num} ({title}) ---")
|
||||
|
||||
# Check paragraph count is reasonable
|
||||
if len(paragraphs) < 5:
|
||||
print(f" ⚠ Only {len(paragraphs)} paragraphs (suspiciously low)")
|
||||
elif len(paragraphs) > 150:
|
||||
print(f" ⚠ {len(paragraphs)} paragraphs (suspiciously high)")
|
||||
else:
|
||||
print(f" ✅ {len(paragraphs)} paragraphs (reasonable)")
|
||||
|
||||
# Check for residual span class artifacts
|
||||
span_count = sum(1 for p in paragraphs if 'class="' in p and "<span" in p)
|
||||
if span_count == 0:
|
||||
print(f" ✅ No residual span class artifacts found")
|
||||
else:
|
||||
print(f" ⚠ {span_count} paragraphs still contain span class artifacts")
|
||||
|
||||
# Check key terms
|
||||
all_text = " ".join(paragraphs)
|
||||
if "Krishna" in all_text or "Chaitanya" in all_text:
|
||||
print(f" ✅ 'Krishna'/'Chaitanya' found (plain English, as expected)")
|
||||
else:
|
||||
print(f" ⚠ Key terms not found")
|
||||
|
||||
# Check that editor notes were excluded
|
||||
for p in paragraphs:
|
||||
plain = re.sub(r"<[^>]+>", "", p).strip()
|
||||
if plain.startswith("[") and ("proofread" in plain.lower() or "—Ed" in plain):
|
||||
print(f" ⚠ Editor note leaked into paragraphs")
|
||||
break
|
||||
else:
|
||||
print(f" ✅ No editor notes leaked into paragraphs")
|
||||
|
||||
# Check that divider lines were excluded
|
||||
for p in paragraphs:
|
||||
plain = re.sub(r"<[^>]+>", "", p).strip()
|
||||
if re.match(r"^_+$", plain):
|
||||
print(f" ⚠ Divider line leaked into paragraphs")
|
||||
break
|
||||
else:
|
||||
print(f" ✅ No divider lines leaked into paragraphs")
|
||||
|
||||
# Essay-specific checks
|
||||
if essay_num == 1:
|
||||
if "spiritual master" in all_text.lower():
|
||||
print(f" ✅ 'spiritual master' found in text")
|
||||
else:
|
||||
print(f" ⚠ 'spiritual master' not found")
|
||||
|
||||
# Check that reference line [BTG 1960] was excluded
|
||||
for p in paragraphs:
|
||||
plain = re.sub(r"<[^>]+>", "", p).strip()
|
||||
if "BTG 1960" in plain and plain.startswith("["):
|
||||
print(f" ⚠ Reference line '[BTG 1960]' leaked into paragraphs")
|
||||
break
|
||||
else:
|
||||
print(f" ✅ Reference line properly excluded")
|
||||
|
||||
# Check that title was excluded
|
||||
for p in paragraphs:
|
||||
plain = re.sub(r"<[^>]+>", "", p).strip()
|
||||
if (
|
||||
plain == "(The Spiritual Masters)."
|
||||
or plain == "(The Spiritual Masters)."
|
||||
):
|
||||
print(f" ⚠ Title line leaked into paragraphs")
|
||||
break
|
||||
else:
|
||||
print(f" ✅ Title line properly excluded")
|
||||
|
||||
elif essay_num == 2:
|
||||
if "impersonal Brahman" in all_text or "Absolute Truth" in all_text:
|
||||
print(f" ✅ 'Absolute Truth' / 'impersonal Brahman' found")
|
||||
else:
|
||||
print(f" ⚠ Key terms for Essay 2 not found")
|
||||
|
||||
# Check "(Second Part)" header was excluded
|
||||
for p in paragraphs:
|
||||
plain = re.sub(r"<[^>]+>", "", p).strip()
|
||||
if plain == "(Second Part)":
|
||||
print(f" ⚠ Header '(Second Part)' leaked into paragraphs")
|
||||
break
|
||||
else:
|
||||
print(f" ✅ Header '(Second Part)' properly excluded")
|
||||
|
||||
elif essay_num == 3:
|
||||
if "incarnation" in all_text.lower() or "descent" in all_text.lower():
|
||||
print(f" ✅ 'incarnation'/'descent' found")
|
||||
else:
|
||||
print(f" ⚠ Key terms for Essay 3 not found")
|
||||
|
||||
# Check that "Thus ends the III part..." is included
|
||||
found_ending = False
|
||||
for p in paragraphs:
|
||||
if "Thus ends the III part" in p:
|
||||
found_ending = True
|
||||
break
|
||||
if found_ending:
|
||||
print(f" ✅ 'Thus ends the III part...' paragraph included")
|
||||
else:
|
||||
print(f" ⚠ 'Thus ends the III part...' paragraph missing")
|
||||
|
||||
elif essay_num == 4:
|
||||
if "Radharani" in all_text or "confidential" in all_text.lower():
|
||||
print(f" ✅ 'Radharani'/'confidential' found")
|
||||
else:
|
||||
print(f" ⚠ Key terms for Essay 4 not found")
|
||||
|
||||
# Check that the summary paragraph is included (last paragraph)
|
||||
found_summary = False
|
||||
for p in paragraphs:
|
||||
if "author then summarize" in p.lower():
|
||||
found_summary = True
|
||||
break
|
||||
if found_summary:
|
||||
print(f" ✅ Summary paragraph ('The author then summarize...') included")
|
||||
else:
|
||||
print(f" ⚠ Summary paragraph missing")
|
||||
|
||||
# Check that the first body paragraph mentions "three principal purposes"
|
||||
if paragraphs:
|
||||
if "three principal purposes" in paragraphs[0]:
|
||||
print(f" ✅ First paragraph mentions 'three principal purposes'")
|
||||
else:
|
||||
print(f" ⚠ First paragraph doesn't mention 'three principal purposes'")
|
||||
|
||||
# Check that p4/p5/p29/p30 header lines were excluded
|
||||
for p in paragraphs:
|
||||
plain = re.sub(r"<[^>]+>", "", p).strip()
|
||||
if "4th Chapter" in plain and len(plain) < 50:
|
||||
print(f" ⚠ Header '4th Chapter' leaked into paragraphs")
|
||||
break
|
||||
else:
|
||||
print(f" ✅ Header lines properly excluded")
|
||||
|
||||
elif essay_num == 5:
|
||||
if "Nityananda" in all_text:
|
||||
print(f" ✅ 'Nityananda' found")
|
||||
else:
|
||||
print(f" ⚠ 'Nityananda' not found")
|
||||
|
||||
# Check that the final p34 description paragraph is included
|
||||
if paragraphs:
|
||||
last_plain = re.sub(r"<[^>]+>", "", paragraphs[-1]).strip()
|
||||
if (
|
||||
"lusture" in last_plain
|
||||
or "turban" in last_plain
|
||||
or "wonderful feature" in last_plain
|
||||
):
|
||||
print(f" ✅ Final description paragraph (p34) included")
|
||||
else:
|
||||
print(f" ⚠ Final description paragraph (p34) may be missing")
|
||||
print(f" Last paragraph preview: {last_plain[:80]}...")
|
||||
|
||||
# Check that the first body paragraph mentions "5th chapter"
|
||||
if paragraphs:
|
||||
if "5th chapter" in paragraphs[0].lower():
|
||||
print(f" ✅ First paragraph mentions '5th chapter'")
|
||||
else:
|
||||
print(f" ⚠ First paragraph doesn't mention '5th chapter'")
|
||||
|
||||
# Check that p31 header was excluded
|
||||
for p in paragraphs:
|
||||
plain = re.sub(r"<[^>]+>", "", p).strip()
|
||||
if "Shi Chaitanya Charitamrita 1.V" in plain:
|
||||
print(
|
||||
f" ⚠ Header 'Shi Chaitanya Charitamrita 1.V' leaked into paragraphs"
|
||||
)
|
||||
break
|
||||
else:
|
||||
print(f" ✅ Header lines properly excluded")
|
||||
|
||||
# Sample output
|
||||
print(f"\n First paragraph (preview):")
|
||||
if paragraphs:
|
||||
preview = re.sub(r"<[^>]+>", "", paragraphs[0])[:120]
|
||||
print(f" {preview}...")
|
||||
print(f" Last paragraph (preview):")
|
||||
if paragraphs:
|
||||
last = re.sub(r"<[^>]+>", "", paragraphs[-1])[:120]
|
||||
print(f" {last}...")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html")
|
||||
output_dir = os.path.join(base_dir, "lib", "jsons", "essays")
|
||||
|
||||
if not os.path.exists(html_path):
|
||||
print(f"ERROR: {html_path} not found!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Reading {html_path}...")
|
||||
tags = parse_all_paragraphs(html_path)
|
||||
print(f" Parsed {len(tags)} <p> tags total\n")
|
||||
|
||||
# Extract all five essays
|
||||
print("Locating essays...")
|
||||
essay_1 = extract_essay_1(tags)
|
||||
essay_2 = extract_essay_2(tags)
|
||||
essay_3 = extract_essay_3(tags)
|
||||
essay_4 = extract_essay_4(tags)
|
||||
essay_5 = extract_essay_5(tags)
|
||||
|
||||
all_essays = [essay_1, essay_2, essay_3, essay_4, essay_5]
|
||||
|
||||
# Write JSON files
|
||||
print(f"\nWriting JSON files...")
|
||||
for essay in all_essays:
|
||||
output_path = os.path.join(output_dir, f"{essay['slug']}.json")
|
||||
write_json(essay, output_path)
|
||||
print(f" Written: {output_path}")
|
||||
|
||||
# Quality checks
|
||||
for i, essay in enumerate(all_essays, 1):
|
||||
quality_check(essay, i)
|
||||
|
||||
# Summary
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f" EXTRACTION SUMMARY")
|
||||
print(f"{'=' * 60}")
|
||||
for i, essay in enumerate(all_essays, 1):
|
||||
print(
|
||||
f" Essay {i}: {essay['title']}"
|
||||
f" ({len(essay['paragraphs'])} paragraphs, {essay['year']})"
|
||||
)
|
||||
print(
|
||||
f" slug={essay['slug']}, group={essay['group']}, order={essay['order']}"
|
||||
)
|
||||
print(f"{'=' * 60}")
|
||||
print(f" Total essays: {len(all_essays)}")
|
||||
print(f" Total paragraphs: {sum(len(e['paragraphs']) for e in all_essays)}")
|
||||
print(f"\nDone! ✨")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,688 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract The Jaladuta Diary (1965) from others-before-1977.html.
|
||||
|
||||
Each diary day entry becomes its own chapter file.
|
||||
Chapters are grouped by month in the intro.json metadata.
|
||||
|
||||
Structure:
|
||||
- Preface and Introduction are standalone chapters (slugs "preface", "introduction")
|
||||
- Day entries use slugs like "apr-3", "may-2", "aug-13", "sep-1", etc.
|
||||
- The "Memoranda" entry at the end uses slug "memoranda"
|
||||
- Months are parent groups in intro.json (non-clickable headings)
|
||||
|
||||
Output:
|
||||
lib/jsons/journals/jaladuta-diary/intro.json
|
||||
lib/jsons/journals/jaladuta-diary/chapters/preface.json
|
||||
lib/jsons/journals/jaladuta-diary/chapters/introduction.json
|
||||
lib/jsons/journals/jaladuta-diary/chapters/apr-3.json
|
||||
lib/jsons/journals/jaladuta-diary/chapters/apr-5.json
|
||||
...
|
||||
lib/jsons/journals/jaladuta-diary/chapters/nov-28.json
|
||||
lib/jsons/journals/jaladuta-diary/chapters/memoranda.json
|
||||
|
||||
Usage:
|
||||
python3 scripts/jaladuta_diary_extract.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ScaGoudy → IAST Unicode mapping
|
||||
# ---------------------------------------------------------------------------
|
||||
SCAGOUDY_TO_IAST = {
|
||||
"ä": "ā",
|
||||
"Ä": "Ā",
|
||||
"å": "ṛ",
|
||||
"Å": "Ṛ",
|
||||
"ñ": "ṣ",
|
||||
"Ñ": "Ṣ",
|
||||
"ö": "ṭ",
|
||||
"Ö": "Ṭ",
|
||||
"ë": "ṇ",
|
||||
"Ë": "Ṇ",
|
||||
"ò": "ḍ",
|
||||
"Ò": "Ḍ",
|
||||
"ç": "ś",
|
||||
"Ç": "Ś",
|
||||
"ì": "ṅ",
|
||||
"Ì": "Ṅ",
|
||||
"ï": "ñ",
|
||||
"Ï": "Ñ",
|
||||
"ü": "ū",
|
||||
"Ü": "Ū",
|
||||
"é": "ī",
|
||||
"É": "Ī",
|
||||
"à": "ṁ",
|
||||
"ù": "ḥ",
|
||||
"è": "ṝ",
|
||||
"ÿ": "ḷ",
|
||||
}
|
||||
|
||||
|
||||
def decode_scagoudy(text: str) -> str:
|
||||
"""Replace ScaGoudy-encoded characters with proper IAST Unicode."""
|
||||
for src, dst in SCAGOUDY_TO_IAST.items():
|
||||
text = text.replace(src, dst)
|
||||
return text
|
||||
|
||||
|
||||
def get_css_class(tag: Tag) -> str:
|
||||
classes = tag.get("class", [])
|
||||
return classes[0] if classes else ""
|
||||
|
||||
|
||||
def inner_html(tag: Tag) -> str:
|
||||
"""Get inner HTML of a tag, decode ScaGoudy, clean up spans."""
|
||||
html = "".join(str(child) for child in tag.children)
|
||||
html = decode_scagoudy(html)
|
||||
# <span class="s3">text</span> → <i>text</i>
|
||||
html = re.sub(
|
||||
r'<span[^>]*class="s3"[^>]*>(.*?)</span>', r"<i>\1</i>", html, flags=re.DOTALL
|
||||
)
|
||||
# <span class="s5">...</span> → keep inner text
|
||||
html = re.sub(
|
||||
r'<span[^>]*class="s5"[^>]*>(.*?)</span>', r"\1", html, flags=re.DOTALL
|
||||
)
|
||||
# <span class="s13">...</span> → keep inner text
|
||||
html = re.sub(
|
||||
r'<span[^>]*class="s1[0-9]+"[^>]*>(.*?)</span>', r"\1", html, flags=re.DOTALL
|
||||
)
|
||||
# <span class="s23">...</span> → keep inner text
|
||||
html = re.sub(
|
||||
r'<span[^>]*class="s2[0-9]+"[^>]*>(.*?)</span>', r"\1", html, flags=re.DOTALL
|
||||
)
|
||||
# Remove any remaining spans but keep content
|
||||
html = re.sub(r"<span[^>]*>(.*?)</span>", r"\1", html, flags=re.DOTALL)
|
||||
# Clean up Apple-tab-span leftovers
|
||||
html = html.replace('<span class="Apple-tab-span">\t</span>', "\t")
|
||||
# Collapse multiple spaces (but not around <br>)
|
||||
html = re.sub(r"[ \t]+", " ", html)
|
||||
html = html.strip()
|
||||
# Remove leading <br> tags
|
||||
html = re.sub(r"^(<br\s*/?>[\s]*)+", "", html)
|
||||
return html
|
||||
|
||||
|
||||
def write_json(data: dict, path: str):
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
f.write("\n")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Month abbreviation helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
MONTH_ABBR = {
|
||||
"april": "apr",
|
||||
"may": "may",
|
||||
"june": "jun",
|
||||
"july": "jul",
|
||||
"august": "aug",
|
||||
"september": "sep",
|
||||
"october": "oct",
|
||||
"november": "nov",
|
||||
}
|
||||
|
||||
MONTH_ORDER = ["apr", "may", "jun", "jul", "aug", "sep", "oct", "nov"]
|
||||
|
||||
DAY_NAMES = {
|
||||
"monday",
|
||||
"tuesday",
|
||||
"wednesday",
|
||||
"thursday",
|
||||
"friday",
|
||||
"saturday",
|
||||
"sunday",
|
||||
}
|
||||
|
||||
|
||||
def parse_day_header(bold_text: str):
|
||||
"""Parse the bold header of a diary entry to extract day name and date number.
|
||||
|
||||
Handles formats like:
|
||||
"SATURDAY 3"
|
||||
"13 FRIDAY"
|
||||
"SATURDAY" (date-less, like Aug 14)
|
||||
"MEMORANDA"
|
||||
Returns (day_name, date_number) or (None, None) if not a day entry.
|
||||
"""
|
||||
text = bold_text.strip().split("\n")[0].strip()
|
||||
text = re.sub(r"<br\s*/?>", " ", text).strip()
|
||||
|
||||
if text.upper() == "MEMORANDA":
|
||||
return ("MEMORANDA", None)
|
||||
|
||||
# Try "DAY NUMBER" pattern
|
||||
m = re.match(r"([A-Za-z]+)\s+(\d+)", text)
|
||||
if m:
|
||||
day = m.group(1).upper()
|
||||
num = int(m.group(2))
|
||||
if day in {d.upper() for d in DAY_NAMES}:
|
||||
return (day, num)
|
||||
|
||||
# Try "NUMBER DAY" pattern
|
||||
m = re.match(r"(\d+)\s+([A-Za-z]+)", text)
|
||||
if m:
|
||||
num = int(m.group(1))
|
||||
day = m.group(2).upper()
|
||||
if day in {d.upper() for d in DAY_NAMES}:
|
||||
return (day, num)
|
||||
|
||||
# Try just "DAY" (no number — like SATURDAY 14 where 14 is on next line)
|
||||
word = text.split()[0].upper() if text.split() else ""
|
||||
if word in {d.upper() for d in DAY_NAMES}:
|
||||
# Try to find number after
|
||||
rest = text[len(word) :].strip()
|
||||
m2 = re.match(r"(\d+)", rest)
|
||||
if m2:
|
||||
return (word, int(m2.group(1)))
|
||||
return (word, None)
|
||||
|
||||
return (None, None)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html")
|
||||
output_dir = os.path.join(base_dir, "lib", "jsons", "journals", "jaladuta-diary")
|
||||
chapters_dir = os.path.join(output_dir, "chapters")
|
||||
|
||||
if not os.path.exists(html_path):
|
||||
print(f"ERROR: {html_path} not found!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Reading {html_path}...")
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
|
||||
soup = BeautifulSoup(content, "html.parser")
|
||||
all_tags = soup.find_all("p")
|
||||
print(f" Parsed {len(all_tags)} <p> tags total")
|
||||
|
||||
# Find the Jaladuta Diary section boundaries
|
||||
start_idx = None
|
||||
end_idx = None
|
||||
for i, tag in enumerate(all_tags):
|
||||
text = tag.get_text().strip()
|
||||
cls = get_css_class(tag)
|
||||
if cls == "p35" and "Jaladuta Diary" in text:
|
||||
start_idx = i
|
||||
if start_idx is not None and i > start_idx:
|
||||
# End at "Prayers Written by Śrīla Prabhupāda aboard the Jaladuta" or the next p4 bookmark after JD
|
||||
if cls == "p4" and "JD 8" in text:
|
||||
end_idx = i
|
||||
break
|
||||
|
||||
if start_idx is None:
|
||||
print("ERROR: Could not find Jaladuta Diary start")
|
||||
sys.exit(1)
|
||||
if end_idx is None:
|
||||
# Fallback: use the memoranda end
|
||||
for i, tag in enumerate(all_tags):
|
||||
if (
|
||||
i > start_idx
|
||||
and "MEMORANDA" in tag.get_text()
|
||||
and get_css_class(tag) == "p110"
|
||||
):
|
||||
end_idx = i + 1
|
||||
break
|
||||
|
||||
print(f" Section: tags[{start_idx}] → tags[{end_idx}]")
|
||||
|
||||
section_tags = all_tags[start_idx:end_idx]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 1: Extract Preface and Introduction
|
||||
# ---------------------------------------------------------------------------
|
||||
preface_paragraphs = []
|
||||
intro_paragraphs = []
|
||||
current_section = None # "preface" | "introduction" | None
|
||||
|
||||
# Also track where months/diary entries begin
|
||||
diary_start_idx = None
|
||||
|
||||
for i, tag in enumerate(section_tags):
|
||||
cls = get_css_class(tag)
|
||||
text = tag.get_text().strip()
|
||||
|
||||
# Detect Preface header
|
||||
if cls == "p3" and text == "Preface":
|
||||
current_section = "preface"
|
||||
continue
|
||||
if cls == "p153": # bookmark
|
||||
continue
|
||||
|
||||
# Detect Introduction header
|
||||
if cls in ("p11", "p157") and "Introduction" in text:
|
||||
if cls == "p157":
|
||||
current_section = "introduction"
|
||||
continue
|
||||
elif cls == "p11":
|
||||
current_section = "introduction"
|
||||
continue
|
||||
|
||||
# Detect month start (diary entries begin)
|
||||
if cls == "p11" and re.match(r"JD \d+:", text):
|
||||
diary_start_idx = i
|
||||
current_section = None
|
||||
break
|
||||
|
||||
if cls == "p29":
|
||||
# Bold month header like "APRIL 1965" — this is diary territory
|
||||
diary_start_idx = i
|
||||
current_section = None
|
||||
break
|
||||
|
||||
# Collect preface/intro paragraphs
|
||||
if current_section == "preface" and cls in ("p154", "p155", "p156"):
|
||||
html = inner_html(tag)
|
||||
if html:
|
||||
preface_paragraphs.append(html)
|
||||
elif current_section == "introduction" and cls in ("p154", "p155", "p156"):
|
||||
html = inner_html(tag)
|
||||
if html:
|
||||
intro_paragraphs.append(html)
|
||||
|
||||
print(f"\n Preface: {len(preface_paragraphs)} paragraphs")
|
||||
print(f" Introduction: {len(intro_paragraphs)} paragraphs")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 2: Extract diary entries grouped by month
|
||||
# ---------------------------------------------------------------------------
|
||||
# Walk through remaining tags, tracking current month
|
||||
|
||||
current_month = None # "apr", "may", etc.
|
||||
current_month_full = None # "April 1965", etc.
|
||||
|
||||
# Section headers that appear mid-month
|
||||
section_headers = {} # Maps tag index to header text
|
||||
|
||||
# All day entries: list of { month, day, date, slug, title, paragraphs }
|
||||
day_entries = []
|
||||
current_entry = None
|
||||
|
||||
MONTH_MAP = {
|
||||
"APRIL": ("apr", "April 1965"),
|
||||
"MAY": ("may", "May 1965"),
|
||||
"JUNE": ("jun", "June 1965"),
|
||||
"JULY": ("jul", "July 1965"),
|
||||
"AUGUST": ("aug", "August 1965"),
|
||||
"SEPTEMBER": ("sep", "September 1965"),
|
||||
"OCTOBER": ("oct", "October 1965"),
|
||||
"NOVEMBER": ("nov", "November 1965"),
|
||||
}
|
||||
|
||||
def finalize_entry():
|
||||
nonlocal current_entry
|
||||
if current_entry and current_entry["paragraphs"]:
|
||||
day_entries.append(current_entry)
|
||||
current_entry = None
|
||||
|
||||
diary_tags = section_tags[diary_start_idx:] if diary_start_idx else []
|
||||
|
||||
for i, tag in enumerate(diary_tags):
|
||||
cls = get_css_class(tag)
|
||||
text = tag.get_text().strip()
|
||||
|
||||
# Skip bookmarks
|
||||
if cls in ("p11", "p4", "p39", "p153"):
|
||||
continue
|
||||
|
||||
# Detect month headers (p29 bold)
|
||||
if cls == "p29":
|
||||
html = inner_html(tag)
|
||||
# Check if this is a month header like "APRIL 1965" or "MAY 1965"
|
||||
for month_key, (abbr, full) in MONTH_MAP.items():
|
||||
if month_key in text.upper():
|
||||
finalize_entry()
|
||||
current_month = abbr
|
||||
current_month_full = full
|
||||
break
|
||||
else:
|
||||
# Could be a section header like "Verse 1." — skip if not month
|
||||
# Or it could be a section header within a month
|
||||
pass
|
||||
continue
|
||||
|
||||
# Detect section headers (p68 class — "The Journey", "America")
|
||||
if cls == "p68":
|
||||
finalize_entry()
|
||||
html = inner_html(tag)
|
||||
# Create a section-header entry
|
||||
if html and current_month:
|
||||
section_entry = {
|
||||
"month": current_month,
|
||||
"month_full": current_month_full,
|
||||
"day": None,
|
||||
"date": None,
|
||||
"slug": None,
|
||||
"title": None,
|
||||
"paragraphs": [f"<b>{re.sub('<[^>]+>', ' ', html).strip()}</b>"],
|
||||
"is_section_header": True,
|
||||
}
|
||||
# Extract title text
|
||||
plain = re.sub("<[^>]+>", " ", html)
|
||||
plain = re.sub(r"\s+", " ", plain).strip()
|
||||
section_entry["title"] = plain
|
||||
day_entries.append(section_entry)
|
||||
continue
|
||||
|
||||
# Detect diary entries (p110 class)
|
||||
if cls == "p110":
|
||||
finalize_entry()
|
||||
|
||||
html = inner_html(tag)
|
||||
if not html:
|
||||
continue
|
||||
|
||||
# Parse the bold header
|
||||
b_tag = tag.find("b")
|
||||
if b_tag:
|
||||
bold_text = b_tag.get_text().strip()
|
||||
day_name, date_num = parse_day_header(bold_text)
|
||||
else:
|
||||
day_name, date_num = (None, None)
|
||||
|
||||
if day_name == "MEMORANDA":
|
||||
current_entry = {
|
||||
"month": current_month,
|
||||
"month_full": current_month_full,
|
||||
"day": "MEMORANDA",
|
||||
"date": None,
|
||||
"slug": "memoranda",
|
||||
"title": "Memoranda",
|
||||
"paragraphs": [html],
|
||||
"is_section_header": False,
|
||||
}
|
||||
elif day_name and date_num and current_month:
|
||||
slug = f"{current_month}-{date_num}"
|
||||
title = f"{day_name.capitalize()} {date_num}"
|
||||
current_entry = {
|
||||
"month": current_month,
|
||||
"month_full": current_month_full,
|
||||
"day": day_name,
|
||||
"date": date_num,
|
||||
"slug": slug,
|
||||
"title": title,
|
||||
"paragraphs": [html],
|
||||
"is_section_header": False,
|
||||
}
|
||||
elif day_name and current_month:
|
||||
# Day without date number (rare — like "SATURDAY" alone on Aug 14)
|
||||
# Try to infer from context or just skip
|
||||
slug = f"{current_month}-{day_name.lower()}"
|
||||
title = day_name.capitalize()
|
||||
current_entry = {
|
||||
"month": current_month,
|
||||
"month_full": current_month_full,
|
||||
"day": day_name,
|
||||
"date": None,
|
||||
"slug": slug,
|
||||
"title": title,
|
||||
"paragraphs": [html],
|
||||
"is_section_header": False,
|
||||
}
|
||||
else:
|
||||
# Can't parse — treat as continuation or standalone
|
||||
current_entry = {
|
||||
"month": current_month,
|
||||
"month_full": current_month_full,
|
||||
"day": None,
|
||||
"date": None,
|
||||
"slug": None,
|
||||
"title": text[:50],
|
||||
"paragraphs": [html],
|
||||
"is_section_header": False,
|
||||
}
|
||||
continue
|
||||
|
||||
# Detect p12 paragraphs (continuation text in some entries like Sep 16)
|
||||
if cls == "p12" and current_entry:
|
||||
html = inner_html(tag)
|
||||
if html:
|
||||
current_entry["paragraphs"].append(html)
|
||||
continue
|
||||
|
||||
# p155 (source citations within diary section)
|
||||
if cls == "p155" and current_entry:
|
||||
html = inner_html(tag)
|
||||
if html:
|
||||
current_entry["paragraphs"].append(html)
|
||||
continue
|
||||
|
||||
finalize_entry()
|
||||
|
||||
print(f"\n Total day entries extracted: {len(day_entries)}")
|
||||
|
||||
# Count by month
|
||||
month_counts = {}
|
||||
for entry in day_entries:
|
||||
m = entry["month"]
|
||||
month_counts[m] = month_counts.get(m, 0) + 1
|
||||
for m in MONTH_ORDER:
|
||||
if m in month_counts:
|
||||
print(f" {m}: {month_counts[m]} entries")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 3: Fix up entries — handle dateless entries, dedup slugs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Some entries have no date — try to fix known cases
|
||||
# Aug "SATURDAY" without number is Aug 14 (follows Aug 13 FRIDAY)
|
||||
for i, entry in enumerate(day_entries):
|
||||
if entry.get("is_section_header"):
|
||||
continue
|
||||
if entry["date"] is None and entry["day"] and entry["day"] != "MEMORANDA":
|
||||
# Try to infer from previous entry
|
||||
if i > 0:
|
||||
prev = day_entries[i - 1]
|
||||
if prev["date"] and prev["month"] == entry["month"]:
|
||||
inferred = prev["date"] + 1
|
||||
entry["date"] = inferred
|
||||
entry["slug"] = f"{entry['month']}-{inferred}"
|
||||
entry["title"] = f"{entry['day'].capitalize()} {inferred}"
|
||||
print(
|
||||
f" Inferred date: {entry['slug']} (from previous {prev['slug']})"
|
||||
)
|
||||
|
||||
# Ensure unique slugs
|
||||
seen_slugs = set()
|
||||
for entry in day_entries:
|
||||
if entry.get("is_section_header"):
|
||||
continue
|
||||
slug = entry["slug"]
|
||||
if slug and slug in seen_slugs:
|
||||
# Append suffix
|
||||
suffix = 2
|
||||
while f"{slug}-{suffix}" in seen_slugs:
|
||||
suffix += 1
|
||||
entry["slug"] = f"{slug}-{suffix}"
|
||||
print(f" Deduped slug: {slug} → {entry['slug']}")
|
||||
if slug:
|
||||
seen_slugs.add(entry["slug"])
|
||||
|
||||
# Merge section headers into the next entry's paragraphs (prepend)
|
||||
merged_entries = []
|
||||
pending_header = None
|
||||
for entry in day_entries:
|
||||
if entry.get("is_section_header"):
|
||||
pending_header = entry["paragraphs"][0]
|
||||
continue
|
||||
if pending_header:
|
||||
entry["paragraphs"] = [pending_header] + entry["paragraphs"]
|
||||
pending_header = None
|
||||
merged_entries.append(entry)
|
||||
day_entries = merged_entries
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 4: Build intro.json with hierarchical month groups
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Assign chapter numbers: preface=1, introduction=2, then day entries sequentially
|
||||
chapter_num = 1
|
||||
|
||||
all_chapters = []
|
||||
|
||||
# Preface
|
||||
preface_chapter = {
|
||||
"slug": "preface",
|
||||
"title": "Preface",
|
||||
"number": chapter_num,
|
||||
}
|
||||
all_chapters.append(preface_chapter)
|
||||
chapter_num += 1
|
||||
|
||||
# Introduction
|
||||
intro_chapter = {
|
||||
"slug": "introduction",
|
||||
"title": "Introduction",
|
||||
"number": chapter_num,
|
||||
}
|
||||
all_chapters.append(intro_chapter)
|
||||
chapter_num += 1
|
||||
|
||||
# Group day entries by month
|
||||
months_data = [] # List of { month, title, subChapters: [...] }
|
||||
current_month_group = None
|
||||
|
||||
for entry in day_entries:
|
||||
month = entry["month"]
|
||||
month_full = entry["month_full"]
|
||||
|
||||
if current_month_group is None or current_month_group["month"] != month:
|
||||
current_month_group = {
|
||||
"month": month,
|
||||
"title": month_full,
|
||||
"subChapters": [],
|
||||
}
|
||||
months_data.append(current_month_group)
|
||||
|
||||
entry["number"] = chapter_num
|
||||
current_month_group["subChapters"].append(
|
||||
{
|
||||
"slug": entry["slug"],
|
||||
"title": entry["title"],
|
||||
}
|
||||
)
|
||||
chapter_num += 1
|
||||
|
||||
# Build the intro.json chapters array
|
||||
# Front matter chapters (preface, introduction) are top-level
|
||||
# Month groups use the hierarchical sub-chapter pattern
|
||||
intro_chapters = []
|
||||
|
||||
# Add preface and introduction as simple chapters
|
||||
intro_chapters.append({"slug": "preface", "title": "Preface"})
|
||||
intro_chapters.append({"slug": "introduction", "title": "Introduction"})
|
||||
|
||||
# Add month groups
|
||||
for mg in months_data:
|
||||
intro_chapters.append(
|
||||
{
|
||||
"title": mg["title"],
|
||||
"subChapters": mg["subChapters"],
|
||||
}
|
||||
)
|
||||
|
||||
intro_json = {
|
||||
"book": "jaladuta-diary",
|
||||
"title": "The Jaladuta Diary, 1965 — Journey to the U.S.A.",
|
||||
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
||||
"year": 1965,
|
||||
"description": "Śrīla Prabhupāda's personal diary kept during his historic voyage from Calcutta to New York aboard the cargo ship Jaladuta in 1965.",
|
||||
"totalChapters": chapter_num - 1,
|
||||
"chapters": intro_chapters,
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 5: Write JSON files
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Clean output directory
|
||||
if os.path.exists(chapters_dir):
|
||||
for f in os.listdir(chapters_dir):
|
||||
os.remove(os.path.join(chapters_dir, f))
|
||||
|
||||
# Write intro.json
|
||||
write_json(intro_json, os.path.join(output_dir, "intro.json"))
|
||||
print(f"\n Written: intro.json ({len(intro_chapters)} top-level entries)")
|
||||
|
||||
# Write preface
|
||||
write_json(
|
||||
{
|
||||
"book": "jaladuta-diary",
|
||||
"chapter": 1,
|
||||
"title": "Preface",
|
||||
"type": "preface",
|
||||
"slug": "preface",
|
||||
"paragraphs": preface_paragraphs,
|
||||
},
|
||||
os.path.join(chapters_dir, "preface.json"),
|
||||
)
|
||||
print(f" Written: preface.json ({len(preface_paragraphs)} paragraphs)")
|
||||
|
||||
# Write introduction
|
||||
write_json(
|
||||
{
|
||||
"book": "jaladuta-diary",
|
||||
"chapter": 2,
|
||||
"title": "Introduction",
|
||||
"type": "introduction",
|
||||
"slug": "introduction",
|
||||
"paragraphs": intro_paragraphs,
|
||||
},
|
||||
os.path.join(chapters_dir, "introduction.json"),
|
||||
)
|
||||
print(f" Written: introduction.json ({len(intro_paragraphs)} paragraphs)")
|
||||
|
||||
# Write day entries
|
||||
ch_num = 3
|
||||
for entry in day_entries:
|
||||
slug = entry["slug"]
|
||||
if not slug:
|
||||
print(f" ⚠ Skipping entry with no slug: {entry.get('title', '?')}")
|
||||
continue
|
||||
|
||||
chapter_data = {
|
||||
"book": "jaladuta-diary",
|
||||
"chapter": ch_num,
|
||||
"title": entry["title"],
|
||||
"type": "chapter",
|
||||
"slug": slug,
|
||||
"paragraphs": entry["paragraphs"],
|
||||
}
|
||||
write_json(chapter_data, os.path.join(chapters_dir, f"{slug}.json"))
|
||||
ch_num += 1
|
||||
|
||||
total_files = 2 + len([e for e in day_entries if e.get("slug")])
|
||||
print(f"\n Total chapter files written: {total_files}")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Summary
|
||||
# ---------------------------------------------------------------------------
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f" EXTRACTION SUMMARY")
|
||||
print(f"{'=' * 60}")
|
||||
print(f" Preface: {len(preface_paragraphs)} paragraphs")
|
||||
print(f" Introduction: {len(intro_paragraphs)} paragraphs")
|
||||
print(f" Day entries: {len(day_entries)}")
|
||||
for mg in months_data:
|
||||
print(f" {mg['title']}: {len(mg['subChapters'])} entries")
|
||||
for sc in mg["subChapters"]:
|
||||
print(f" - {sc['slug']}: {sc['title']}")
|
||||
print(f" Total chapters: {ch_num - 1}")
|
||||
print(f"{'=' * 60}")
|
||||
print(f"\nDone! ✨")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,539 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract Message of Godhead (1955) from others-before-1977.html.
|
||||
|
||||
The book has 1 intro segment (front-pages) and 3 chapters:
|
||||
1. Message of Godhead (Introductory)
|
||||
2. Knowledge Transcendental
|
||||
3. Work with Transcendental Effect (Karma Yoga)
|
||||
|
||||
Output:
|
||||
lib/jsons/books/mog/intro.json — Book metadata
|
||||
lib/jsons/books/mog/intro/*.json — Intro segments (front-pages)
|
||||
lib/jsons/books/mog/chapters/*.json — Chapters (01–03)
|
||||
|
||||
Usage:
|
||||
python3 scripts/mog_extract.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def decode_html_content(tag: Tag) -> str:
|
||||
"""Extract inner HTML from a tag, preserve formatting tags, strip spans."""
|
||||
inner = "".join(str(child) for child in tag.children)
|
||||
# Remove span tags with CSS classes (RTF conversion artifacts)
|
||||
inner = re.sub(
|
||||
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
|
||||
)
|
||||
# Clean up Apple-tab-span leftovers (tab characters)
|
||||
inner = inner.replace("\t", " ")
|
||||
# Normalize whitespace
|
||||
inner = re.sub(r"\s+", " ", inner).strip()
|
||||
return inner
|
||||
|
||||
|
||||
def decode_text(tag: Tag) -> str:
|
||||
"""Get plain text from a tag."""
|
||||
return tag.get_text().strip()
|
||||
|
||||
|
||||
def get_css_class(tag: Tag) -> str:
|
||||
"""Get the first CSS class of a tag, or empty string."""
|
||||
classes = tag.get("class", [])
|
||||
return classes[0] if classes else ""
|
||||
|
||||
|
||||
def is_empty(tag: Tag) -> bool:
|
||||
"""Check if a tag is empty or whitespace-only."""
|
||||
text = tag.get_text().strip()
|
||||
return not text or text == "\xa0"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CSS class categorization for MOG
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Front page classes
|
||||
FRONT_TITLE_CLASS = "p3" # Book title header (bold)
|
||||
FRONT_INFO_CLASS = "p19" # Author/address info
|
||||
FRONT_DATE_CLASS = "p15" # Date line ("From the 30/8/55")
|
||||
|
||||
# Chapter heading classes
|
||||
CHAPTER_BOOKMARK_CLASS = (
|
||||
"p11" # Chapter bookmark headers (like "MESSAGE OF GODHEAD (Introductory)")
|
||||
)
|
||||
CHAPTER_TITLE_CLASS = "p5" # Chapter title (bold, duplicate of p11 — skip)
|
||||
|
||||
# Body paragraph classes
|
||||
BODY_CLASSES = frozenset(["p6", "p38", "p44", "p45", "p46", "p47"])
|
||||
|
||||
# Classes to skip entirely (empty/decorative/break)
|
||||
SKIP_CLASSES = frozenset(["p43", "p24"])
|
||||
|
||||
# Boundary detection: classes that mark the start of a different work after MOG
|
||||
NEXT_WORK_BOUNDARY_CLASSES = frozenset(["p2", "p3"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Boundary detection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def find_mog_boundaries(all_paragraphs: list) -> tuple:
|
||||
"""Find the start and end paragraph indices of the MOG content.
|
||||
|
||||
Start: <p class="p3"> containing "Message of Godhead"
|
||||
End: next <p class="p2"> or <p class="p3"> that starts a different book,
|
||||
or <p class="p6"> containing "Bhakti-rasāmṛta-sindhu" (the next work)
|
||||
"""
|
||||
start_idx = None
|
||||
end_idx = len(all_paragraphs)
|
||||
|
||||
for i, p in enumerate(all_paragraphs):
|
||||
cls = get_css_class(p)
|
||||
text = decode_text(p)
|
||||
|
||||
# Find start: p3 containing "Message of Godhead"
|
||||
if start_idx is None:
|
||||
if cls == "p2" and "Message of Godhead" in text:
|
||||
# The p2 is the TOC entry; the actual start is the next p3
|
||||
start_idx = i
|
||||
continue
|
||||
continue
|
||||
|
||||
# Find end: look for the next work boundary
|
||||
# The p6 containing "Bhakti-rasāmṛta-sindhu" starts a different work
|
||||
if cls == "p6" and "Bhakti-ras" in text and "sindhu" in text:
|
||||
end_idx = i
|
||||
break
|
||||
|
||||
# Also check for p2 that starts a different book section
|
||||
if cls == "p2" and start_idx is not None and i > start_idx + 5:
|
||||
text_lower = text.lower()
|
||||
if "message of godhead" not in text_lower:
|
||||
end_idx = i
|
||||
break
|
||||
|
||||
if start_idx is None:
|
||||
print("ERROR: Could not find 'Message of Godhead' in others-before-1977.html!")
|
||||
sys.exit(1)
|
||||
|
||||
return start_idx, end_idx
|
||||
|
||||
|
||||
def find_chapter_bookmarks(all_paragraphs: list, start_idx: int, end_idx: int) -> list:
|
||||
"""Find all MOG chapter bookmark tags (p11) in the MOG range.
|
||||
|
||||
Returns list of (paragraph_index, chapter_title).
|
||||
"""
|
||||
bookmarks = []
|
||||
for i in range(start_idx, end_idx):
|
||||
p = all_paragraphs[i]
|
||||
cls = get_css_class(p)
|
||||
if cls == CHAPTER_BOOKMARK_CLASS:
|
||||
text = decode_text(p).strip()
|
||||
if text:
|
||||
bookmarks.append((i, text))
|
||||
return bookmarks
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Paragraph extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def extract_paragraphs(
|
||||
all_paragraphs: list,
|
||||
start_idx: int,
|
||||
end_idx: int,
|
||||
is_front_pages: bool = False,
|
||||
) -> list:
|
||||
"""Extract paragraphs for a section (front pages or chapter body).
|
||||
|
||||
For front pages: collect p3, p19, p15 lines as paragraphs.
|
||||
For chapters: collect body paragraphs (p6, p38, p44, p45, p46, p47).
|
||||
Always skip p43, p24 (empty/break), p11 (bookmark), p5 (duplicate heading).
|
||||
"""
|
||||
result = []
|
||||
|
||||
for i in range(start_idx, end_idx):
|
||||
p = all_paragraphs[i]
|
||||
cls = get_css_class(p)
|
||||
|
||||
# Skip empty paragraphs
|
||||
if is_empty(p):
|
||||
continue
|
||||
|
||||
# Skip known empty/break classes
|
||||
if cls in SKIP_CLASSES:
|
||||
continue
|
||||
|
||||
# Skip chapter bookmark headers (p11) — used for boundary detection only
|
||||
if cls == CHAPTER_BOOKMARK_CLASS:
|
||||
continue
|
||||
|
||||
# Skip chapter title duplicate (p5)
|
||||
if cls == CHAPTER_TITLE_CLASS:
|
||||
continue
|
||||
|
||||
# Skip p2 (TOC entry at start)
|
||||
if cls == "p2":
|
||||
continue
|
||||
|
||||
if is_front_pages:
|
||||
# For front pages, collect title, author/address, and date lines
|
||||
if cls in (FRONT_TITLE_CLASS, FRONT_INFO_CLASS, FRONT_DATE_CLASS):
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
result.append(html)
|
||||
continue
|
||||
# Also include any other non-skip class if we encounter it
|
||||
# (shouldn't happen much for front pages, but be permissive)
|
||||
if cls in BODY_CLASSES:
|
||||
# Body paragraph before first chapter = still front pages
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
result.append(html)
|
||||
continue
|
||||
else:
|
||||
# For chapter body, collect all body paragraph classes
|
||||
if cls in BODY_CLASSES:
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
result.append(html)
|
||||
continue
|
||||
|
||||
# p19 can appear inside chapters (e.g., "(Devanāgarī)" in chapter 2)
|
||||
if cls == FRONT_INFO_CLASS:
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
result.append(html)
|
||||
continue
|
||||
|
||||
# Title page class inside chapter body (shouldn't happen, but be safe)
|
||||
if cls == FRONT_TITLE_CLASS:
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
result.append(html)
|
||||
continue
|
||||
|
||||
# Fallback: try to extract content from unknown classes
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
result.append(html)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main extraction logic
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def extract_mog(soup: BeautifulSoup) -> dict:
|
||||
"""Extract the entire Message of Godhead from the parsed HTML.
|
||||
|
||||
Returns a dict with:
|
||||
"intro_segments": list of intro segment dicts
|
||||
"chapters": list of chapter dicts
|
||||
"chapter_listing": list of {number, title} for intro.json
|
||||
"""
|
||||
all_paragraphs = list(soup.find_all("p"))
|
||||
|
||||
# Step 1: Find MOG boundaries
|
||||
mog_start, mog_end = find_mog_boundaries(all_paragraphs)
|
||||
print(f" MOG content: paragraphs {mog_start} to {mog_end}")
|
||||
print(f" Total paragraphs in range: {mog_end - mog_start}")
|
||||
|
||||
# Step 2: Find all chapter bookmarks (p11)
|
||||
bookmarks = find_chapter_bookmarks(all_paragraphs, mog_start, mog_end)
|
||||
print(f" Found {len(bookmarks)} MOG chapter bookmarks:")
|
||||
for pos, title in bookmarks:
|
||||
print(f" Paragraph {pos}: {title}")
|
||||
|
||||
# Step 3: Map bookmarks to chapters with proper titles
|
||||
# Expected: 3 chapters
|
||||
# Map raw bookmark text (case-insensitive) to normalized titles
|
||||
chapter_titles = {
|
||||
"message of godhead": "Message of Godhead (Introductory)",
|
||||
"knowledge transcendental": "Knowledge Transcendental",
|
||||
"work with transcendental effect": "Work with Transcendental Effect (Karma Yoga)",
|
||||
}
|
||||
|
||||
chapter_bookmarks = [] # (para_idx, chapter_num, normalized_title)
|
||||
ch_num = 1
|
||||
for pos, raw_title in bookmarks:
|
||||
# Try to match known chapter titles (case-insensitive)
|
||||
matched_title = None
|
||||
raw_lower = raw_title.lower().strip()
|
||||
for key, nice_title in chapter_titles.items():
|
||||
if key in raw_lower:
|
||||
matched_title = nice_title
|
||||
break
|
||||
|
||||
if matched_title is None:
|
||||
# Fallback: use the raw title with title case
|
||||
matched_title = raw_title.strip()
|
||||
|
||||
chapter_bookmarks.append((pos, ch_num, matched_title))
|
||||
ch_num += 1
|
||||
|
||||
print(f"\n Mapped {len(chapter_bookmarks)} chapters:")
|
||||
for pos, num, title in chapter_bookmarks:
|
||||
print(f" Chapter {num}: {title} (paragraph {pos})")
|
||||
|
||||
# Step 4: Extract front pages
|
||||
# From MOG start to first chapter bookmark
|
||||
intro_segments = []
|
||||
|
||||
if chapter_bookmarks:
|
||||
front_pages_end = chapter_bookmarks[0][0]
|
||||
else:
|
||||
front_pages_end = mog_end
|
||||
|
||||
front_pages_paragraphs = extract_paragraphs(
|
||||
all_paragraphs, mog_start, front_pages_end, is_front_pages=True
|
||||
)
|
||||
intro_segments.append(
|
||||
{
|
||||
"slug": "front-pages",
|
||||
"title": "Front Pages",
|
||||
"type": "intro-segment",
|
||||
"book": "mog",
|
||||
"order": 1,
|
||||
"paragraphs": front_pages_paragraphs,
|
||||
}
|
||||
)
|
||||
|
||||
# Step 5: Extract chapters
|
||||
chapters = []
|
||||
for idx, (ch_pos, ch_num, ch_title) in enumerate(chapter_bookmarks):
|
||||
# Content starts after the bookmark line
|
||||
# But we also need to skip the p5 duplicate heading that follows p11
|
||||
content_start = ch_pos + 1
|
||||
|
||||
# Content ends at next chapter bookmark or mog_end
|
||||
if idx + 1 < len(chapter_bookmarks):
|
||||
content_end = chapter_bookmarks[idx + 1][0]
|
||||
else:
|
||||
content_end = mog_end
|
||||
|
||||
paragraphs = extract_paragraphs(all_paragraphs, content_start, content_end)
|
||||
|
||||
ch = {
|
||||
"book": "mog",
|
||||
"chapter": ch_num,
|
||||
"title": ch_title,
|
||||
"type": "chapter",
|
||||
"paragraphs": paragraphs,
|
||||
}
|
||||
chapters.append(ch)
|
||||
|
||||
# Step 6: Build chapter listing for intro.json
|
||||
chapter_listing = [
|
||||
{"number": ch["chapter"], "title": ch["title"]} for ch in chapters
|
||||
]
|
||||
|
||||
return {
|
||||
"intro_segments": intro_segments,
|
||||
"chapters": chapters,
|
||||
"chapter_listing": chapter_listing,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Output
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def write_json(data, filepath):
|
||||
"""Write data as formatted JSON to filepath, creating directories as needed."""
|
||||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||||
with open(filepath, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html")
|
||||
output_base = os.path.join(base_dir, "lib", "jsons", "books", "mog")
|
||||
|
||||
if not os.path.exists(html_path):
|
||||
print(f"ERROR: {html_path} not found!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Reading {html_path}...")
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
soup = BeautifulSoup(f, "html.parser")
|
||||
|
||||
print("Extracting Message of Godhead (1955)...\n")
|
||||
result = extract_mog(soup)
|
||||
|
||||
intro_segments = result["intro_segments"]
|
||||
chapters = result["chapters"]
|
||||
chapter_listing = result["chapter_listing"]
|
||||
|
||||
# --- Write intro.json ---
|
||||
intro_meta = {
|
||||
"book": "mog",
|
||||
"title": "Message of Godhead",
|
||||
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
||||
"year": 1955,
|
||||
"description": "An early handwritten work presenting the essence of Bhagavad-gītā's teachings on transcendental knowledge, karma-yoga, and the Personality of Godhead.",
|
||||
"totalChapters": 3,
|
||||
"chapters": chapter_listing,
|
||||
}
|
||||
intro_path = os.path.join(output_base, "intro.json")
|
||||
write_json(intro_meta, intro_path)
|
||||
print(f"\n Written: {intro_path}")
|
||||
|
||||
# --- Write intro segments ---
|
||||
print(f"\n--- Intro Segments ---")
|
||||
for seg in intro_segments:
|
||||
filepath = os.path.join(output_base, "intro", f"{seg['slug']}.json")
|
||||
write_json(seg, filepath)
|
||||
print(
|
||||
f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs → {filepath}"
|
||||
)
|
||||
|
||||
# --- Write chapters ---
|
||||
print(f"\n--- Chapters ---")
|
||||
for ch in chapters:
|
||||
ch_num_str = str(ch["chapter"]).zfill(2)
|
||||
filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
|
||||
write_json(ch, filepath)
|
||||
print(
|
||||
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
|
||||
f"{ch['title']}"
|
||||
)
|
||||
|
||||
# --- Summary ---
|
||||
total_intro_paras = sum(len(s["paragraphs"]) for s in intro_segments)
|
||||
total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
|
||||
total_paras = total_intro_paras + total_chapter_paras
|
||||
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f" EXTRACTION SUMMARY")
|
||||
print(f"{'=' * 60}")
|
||||
print(f" Intro segments: {len(intro_segments)}")
|
||||
for seg in intro_segments:
|
||||
print(
|
||||
f" {seg['order']}. {seg['title']}: {len(seg['paragraphs'])} paragraphs"
|
||||
)
|
||||
print(f" Chapters: {len(chapters)}")
|
||||
for ch in chapters:
|
||||
print(
|
||||
f" Ch {ch['chapter']}: {len(ch['paragraphs'])} paragraphs — {ch['title']}"
|
||||
)
|
||||
print(f" Total paragraphs: {total_paras}")
|
||||
print(f" Intro: {total_intro_paras}")
|
||||
print(f" Chapters: {total_chapter_paras}")
|
||||
if chapters:
|
||||
print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
|
||||
print(f" Output directory: {output_base}/")
|
||||
|
||||
# --- Samples ---
|
||||
print(f"\n--- Sample: Front Pages ---")
|
||||
if intro_segments:
|
||||
front = intro_segments[0]
|
||||
for i, p in enumerate(front["paragraphs"][:5]):
|
||||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
||||
print(f" [{i}] {preview}")
|
||||
|
||||
if chapters:
|
||||
print(f"\n--- Sample: Chapter 1 ({chapters[0]['title']}) ---")
|
||||
ch1 = chapters[0]
|
||||
for i, p in enumerate(ch1["paragraphs"][:3]):
|
||||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
||||
print(f" [{i}] {preview}...")
|
||||
|
||||
if len(chapters) >= 2:
|
||||
print(f"\n--- Sample: Chapter 2 ({chapters[1]['title']}) ---")
|
||||
ch2 = chapters[1]
|
||||
for i, p in enumerate(ch2["paragraphs"][:3]):
|
||||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
||||
print(f" [{i}] {preview}...")
|
||||
|
||||
if len(chapters) >= 3:
|
||||
print(f"\n--- Sample: Chapter 3 ({chapters[2]['title']}) ---")
|
||||
ch3 = chapters[2]
|
||||
for i, p in enumerate(ch3["paragraphs"][:3]):
|
||||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
||||
print(f" [{i}] {preview}...")
|
||||
|
||||
# --- Quality Checks ---
|
||||
print(f"\n--- Quality Checks ---")
|
||||
|
||||
# Check all 3 chapters present
|
||||
expected_chapters = set(range(1, 4))
|
||||
actual_chapters = set(ch["chapter"] for ch in chapters)
|
||||
missing = expected_chapters - actual_chapters
|
||||
extra = actual_chapters - expected_chapters
|
||||
if missing:
|
||||
print(f" ⚠ Missing chapters: {sorted(missing)}")
|
||||
if extra:
|
||||
print(f" ⚠ Extra chapters: {sorted(extra)}")
|
||||
if not missing and not extra:
|
||||
print(f" ✅ All 3 chapters present")
|
||||
|
||||
# Check intro segments
|
||||
if intro_segments:
|
||||
print(
|
||||
f" ✅ Front pages segment present ({len(intro_segments[0]['paragraphs'])} paragraphs)"
|
||||
)
|
||||
else:
|
||||
print(f" ⚠ Front pages segment missing!")
|
||||
|
||||
# Check empty chapters
|
||||
empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
|
||||
if empty_chapters:
|
||||
print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
|
||||
else:
|
||||
print(f" ✅ No empty chapters")
|
||||
|
||||
# Check for "——The Author." in chapter 1
|
||||
if chapters:
|
||||
ch1_text = " ".join(chapters[0]["paragraphs"])
|
||||
if "The Author" in ch1_text:
|
||||
print(f" ✅ '——The Author.' attribution found in Chapter 1")
|
||||
else:
|
||||
print(f" ⚠ '——The Author.' attribution NOT found in Chapter 1")
|
||||
|
||||
# Check chapter 2 has "(Devanāgarī)" line
|
||||
if len(chapters) >= 2:
|
||||
ch2_text = " ".join(chapters[1]["paragraphs"])
|
||||
if "Devan" in ch2_text:
|
||||
print(f" ✅ '(Devanāgarī)' line found in Chapter 2")
|
||||
else:
|
||||
print(f" ℹ '(Devanāgarī)' line not found in Chapter 2 (may be normal)")
|
||||
|
||||
# Verify no Bhakti-rasāmṛta-sindhu content leaked in
|
||||
all_text = ""
|
||||
for ch in chapters:
|
||||
all_text += " ".join(ch["paragraphs"])
|
||||
if "Text 22" in all_text or "BHAKTI RASAMRITA" in all_text.upper():
|
||||
print(f" ⚠ Content from Bhakti-rasāmṛta-sindhu may have leaked into MOG!")
|
||||
else:
|
||||
print(f" ✅ No content leak from next work detected")
|
||||
|
||||
# Check last chapter ends cleanly
|
||||
if chapters:
|
||||
last_ch = chapters[-1]
|
||||
last_para = last_ch["paragraphs"][-1] if last_ch["paragraphs"] else ""
|
||||
last_preview = re.sub(r"<[^>]+>", "", last_para)[:80]
|
||||
print(f' ℹ Last paragraph of Ch {last_ch["chapter"]}: "{last_preview}..."')
|
||||
|
||||
print(f"\nDone! ✨")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,303 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract "An Analysis of Bhagwat Geeta" from others-before-1977.html.
|
||||
|
||||
This document is a structured analysis organized into three numbered sections
|
||||
(topics) with numbered points and verse references. It is plain English text
|
||||
(no ScaGoudy decoding needed).
|
||||
|
||||
Source: public/html/others-before-1977.html
|
||||
- Title: <p class="p116"><b>An Analysis of Bhagwat Geeta</b></p> (line ~15182)
|
||||
- Section 1: "Mans eternal relation with Godhead" — 45 <p class="p6"> items
|
||||
- Section 2: "Man's Duty Toward Godhead" — 34 <p class="p6"> items
|
||||
- Section 3: "Man's ultimate necessity and perfection of life" — 3 <p class="p6"> items
|
||||
- End boundary: <p class="p11">Solution of Present Crisises... (line ~15276)
|
||||
|
||||
CSS classes:
|
||||
p116 — Title (bold)
|
||||
p39 — Section headers ("1: Mans eternal relation with Godhead")
|
||||
p5 — Duplicate section title (bold, with <span class="s3">) — SKIP
|
||||
p6 — Body paragraphs (plain English)
|
||||
p34 — Math calculation at the end — SKIP
|
||||
p11 — Next document title (end boundary)
|
||||
|
||||
Output (prose book structure):
|
||||
lib/jsons/other-documents/analysis-of-bhagavad-gita/
|
||||
intro.json # metadata
|
||||
chapters/
|
||||
01.json # Chapter 1
|
||||
02.json # Chapter 2
|
||||
03.json # Chapter 3
|
||||
|
||||
Usage:
|
||||
python3 scripts/od_analysis_bg_extract.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
BOOK_CODE = "od-analysis-bg"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def get_css_class(tag: Tag) -> str:
|
||||
"""Get the first CSS class of a tag, or empty string."""
|
||||
classes = tag.get("class", [])
|
||||
return classes[0] if classes else ""
|
||||
|
||||
|
||||
def is_empty(tag: Tag) -> bool:
|
||||
"""Check if a tag is empty or whitespace-only."""
|
||||
text = tag.get_text().strip()
|
||||
return not text or text == "\xa0"
|
||||
|
||||
|
||||
def clean_text(tag: Tag) -> str:
|
||||
"""Extract inner HTML from a tag, strip span class artifacts, normalize whitespace."""
|
||||
inner = "".join(str(child) for child in tag.children)
|
||||
# Remove span tags with CSS classes (RTF conversion artifacts) but keep their content
|
||||
inner = re.sub(
|
||||
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
|
||||
)
|
||||
# Remove any remaining bare <span> or </span> tags
|
||||
inner = re.sub(r"</?span[^>]*>", "", inner)
|
||||
# Remove <b> and </b> tags (not needed for body paragraphs)
|
||||
inner = re.sub(r"</?b>", "", inner)
|
||||
# Clean up Apple-tab-span leftovers (tab characters)
|
||||
inner = inner.replace("\t", " ")
|
||||
# Strip <br> tags
|
||||
inner = re.sub(r"<br\s*/?>", " ", inner)
|
||||
# Normalize whitespace
|
||||
inner = re.sub(r"\s+", " ", inner).strip()
|
||||
# Fix HTML entities
|
||||
inner = inner.replace("&", "&")
|
||||
return inner
|
||||
|
||||
|
||||
def write_json(data, filepath):
|
||||
"""Write data as formatted JSON to filepath, creating directories as needed."""
|
||||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||||
with open(filepath, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
print(f" ✅ Wrote {filepath}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def extract_chapters(soup: BeautifulSoup) -> list[dict]:
|
||||
"""Extract the chapter data from 'An Analysis of Bhagwat Geeta'."""
|
||||
|
||||
# Find the title tag: <p class="p116"><b>An Analysis of Bhagwat Geeta</b></p>
|
||||
title_tag = None
|
||||
for p in soup.find_all("p", class_="p116"):
|
||||
text = p.get_text().strip()
|
||||
if "Analysis of Bhagwat Geeta" in text:
|
||||
title_tag = p
|
||||
break
|
||||
|
||||
if not title_tag:
|
||||
print("❌ Could not find title 'An Analysis of Bhagwat Geeta'")
|
||||
sys.exit(1)
|
||||
|
||||
print(f" 📍 Found title: {title_tag.get_text().strip()}")
|
||||
|
||||
# Collect all <p> tags after the title until we hit the end boundary
|
||||
chapters = []
|
||||
current_chapter = None
|
||||
|
||||
tag = title_tag.find_next_sibling()
|
||||
while tag:
|
||||
if not isinstance(tag, Tag) or tag.name != "p":
|
||||
tag = tag.find_next_sibling()
|
||||
continue
|
||||
|
||||
css = get_css_class(tag)
|
||||
text = tag.get_text().strip()
|
||||
|
||||
# End boundary: next document starts
|
||||
if css == "p11" and "Solution of Present Crisises" in text:
|
||||
break
|
||||
|
||||
# Skip empty tags
|
||||
if is_empty(tag):
|
||||
tag = tag.find_next_sibling()
|
||||
continue
|
||||
|
||||
# Section header: <p class="p39">1: Mans eternal relation with Godhead</p>
|
||||
if css == "p39":
|
||||
# Parse section number and title
|
||||
match = re.match(r"(\d+):\s*(.+)", text)
|
||||
if match:
|
||||
num = int(match.group(1))
|
||||
title = match.group(2).strip()
|
||||
current_chapter = {
|
||||
"number": num,
|
||||
"title": title,
|
||||
"paragraphs": [],
|
||||
}
|
||||
chapters.append(current_chapter)
|
||||
print(f" 📖 Section {num}: {title}")
|
||||
tag = tag.find_next_sibling()
|
||||
continue
|
||||
|
||||
# Skip p5 (duplicate section title in bold with span)
|
||||
if css == "p5":
|
||||
tag = tag.find_next_sibling()
|
||||
continue
|
||||
|
||||
# Skip p34 (math calculation at the end)
|
||||
if css == "p34":
|
||||
tag = tag.find_next_sibling()
|
||||
continue
|
||||
|
||||
# Body paragraph: <p class="p6">
|
||||
if css == "p6" and current_chapter is not None:
|
||||
cleaned = clean_text(tag)
|
||||
if cleaned:
|
||||
current_chapter["paragraphs"].append(cleaned)
|
||||
tag = tag.find_next_sibling()
|
||||
continue
|
||||
|
||||
# Any other class — skip
|
||||
tag = tag.find_next_sibling()
|
||||
|
||||
return chapters
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Quality check
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def quality_check(chapters: list[dict]):
|
||||
"""Validate the extracted chapters."""
|
||||
print("\n🔍 Quality check:\n")
|
||||
|
||||
print(f" Chapters: {len(chapters)}")
|
||||
assert len(chapters) == 3, f"Expected 3 chapters, got {len(chapters)}"
|
||||
|
||||
total_paragraphs = 0
|
||||
for ch in chapters:
|
||||
count = len(ch["paragraphs"])
|
||||
total_paragraphs += count
|
||||
print(f' Chapter {ch["number"]}: "{ch["title"]}" — {count} paragraphs')
|
||||
|
||||
# Spot-check first paragraph
|
||||
if ch["paragraphs"]:
|
||||
first = ch["paragraphs"][0]
|
||||
print(f" First: {first[:80]}...")
|
||||
|
||||
# Spot-check last paragraph
|
||||
if len(ch["paragraphs"]) > 1:
|
||||
last = ch["paragraphs"][-1]
|
||||
print(f" Last: {last[:80]}...")
|
||||
|
||||
print(f"\n Total paragraphs: {total_paragraphs}")
|
||||
|
||||
# Expected counts (from manual inspection):
|
||||
# Section 1: 45 items (numbered 1-45)
|
||||
# Section 2: 34 items (numbered 1-34)
|
||||
# Section 3: 3 items (numbered 1-3)
|
||||
expected_total = 82 # 45 + 34 + 3
|
||||
if total_paragraphs == expected_total:
|
||||
print(f" ✅ Paragraph count matches expected: {expected_total}")
|
||||
else:
|
||||
print(f" ⚠️ Expected {expected_total} paragraphs, got {total_paragraphs}")
|
||||
# This is a warning, not a hard failure — some items may have sub-items
|
||||
# or the HTML structure may slightly differ
|
||||
|
||||
# Check that paragraphs are not empty
|
||||
for ch in chapters:
|
||||
for i, p in enumerate(ch["paragraphs"]):
|
||||
assert p.strip(), f"Empty paragraph in chapter {ch['number']}, index {i}"
|
||||
|
||||
print("\n ✅ All checks passed!")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Output
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def write_output(chapters: list[dict], output_dir: str):
|
||||
"""Write intro.json and per-chapter JSON files."""
|
||||
|
||||
# Build intro.json
|
||||
intro = {
|
||||
"book": BOOK_CODE,
|
||||
"title": "An Analysis of Bhagwat Geeta",
|
||||
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
||||
"year": None,
|
||||
"description": "A structured analysis of Bhagavad-gītā organized into three topics with numbered points and verse references.",
|
||||
"totalChapters": len(chapters),
|
||||
"chapters": [{"number": ch["number"], "title": ch["title"]} for ch in chapters],
|
||||
}
|
||||
|
||||
write_json(intro, os.path.join(output_dir, "intro.json"))
|
||||
|
||||
# Write per-chapter files
|
||||
chapters_dir = os.path.join(output_dir, "chapters")
|
||||
for ch in chapters:
|
||||
chapter_data = {
|
||||
"book": BOOK_CODE,
|
||||
"chapter": ch["number"],
|
||||
"title": ch["title"],
|
||||
"type": "chapter",
|
||||
"paragraphs": ch["paragraphs"],
|
||||
}
|
||||
filename = f"{ch['number']:02d}.json"
|
||||
write_json(chapter_data, os.path.join(chapters_dir, filename))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html")
|
||||
output_dir = os.path.join(
|
||||
base_dir, "lib", "jsons", "other-documents", "analysis-of-bhagavad-gita"
|
||||
)
|
||||
|
||||
if not os.path.exists(html_path):
|
||||
print(f"❌ HTML file not found: {html_path}")
|
||||
sys.exit(1)
|
||||
|
||||
# Clean up old single-file JSON if it exists
|
||||
old_json = os.path.join(
|
||||
base_dir, "lib", "jsons", "other-documents", "analysis-of-bhagavad-gita.json"
|
||||
)
|
||||
if os.path.exists(old_json):
|
||||
os.remove(old_json)
|
||||
print(f" 🗑️ Removed old single-file JSON: {old_json}")
|
||||
|
||||
print(f"📂 Reading {html_path}...")
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
soup = BeautifulSoup(f, "html.parser")
|
||||
|
||||
print("\n📝 Extracting 'An Analysis of Bhagwat Geeta'...\n")
|
||||
chapters = extract_chapters(soup)
|
||||
|
||||
quality_check(chapters)
|
||||
|
||||
write_output(chapters, output_dir)
|
||||
print(f"\n🎉 Done! Output directory: {output_dir}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,533 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract Prayers of King Kulaśekhara (1958) from others-before-1977.html.
|
||||
|
||||
The book has 6 chapters (one per verse/prayer):
|
||||
1. Prayer 1 — "Śrī Vallabha" (prayer to Mukunda)
|
||||
2. Prayer 2 — "Jayatu Jayatu" (glories to Devakī-nandana)
|
||||
3. Prayer 3 — "Mukunda Mūrdhnā" (prayer for remembrance)
|
||||
4. Prayer 4 — "Nāhaṁ Vande" (neither heaven nor hell)
|
||||
5. Prayer 5 — "Nāsthā Dharme" (unflinching devotion)
|
||||
6. Prayer 6 — "Divi Vā Bhuvi" (remembrance at death)
|
||||
|
||||
Output:
|
||||
lib/jsons/books/pk/intro.json — Book metadata
|
||||
lib/jsons/books/pk/chapters/*.json — Chapters (01–06)
|
||||
|
||||
Usage:
|
||||
python3 scripts/pk_extract.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def decode_html_content(tag: Tag) -> str:
|
||||
"""Extract inner HTML from a tag, preserve formatting tags, strip spans."""
|
||||
inner = "".join(str(child) for child in tag.children)
|
||||
# Remove span tags with CSS classes (RTF conversion artifacts)
|
||||
inner = re.sub(
|
||||
r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner, flags=re.DOTALL
|
||||
)
|
||||
# Clean up Apple-tab-span leftovers (tab characters)
|
||||
inner = inner.replace("\t", " ")
|
||||
# Normalize whitespace
|
||||
inner = re.sub(r"\s+", " ", inner).strip()
|
||||
return inner
|
||||
|
||||
|
||||
def decode_text(tag: Tag) -> str:
|
||||
"""Get plain text from a tag."""
|
||||
return tag.get_text().strip()
|
||||
|
||||
|
||||
def get_css_class(tag: Tag) -> str:
|
||||
"""Get the first CSS class of a tag, or empty string."""
|
||||
classes = tag.get("class", [])
|
||||
return classes[0] if classes else ""
|
||||
|
||||
|
||||
def is_empty(tag: Tag) -> bool:
|
||||
"""Check if a tag is empty or whitespace-only."""
|
||||
text = tag.get_text().strip()
|
||||
return not text or text == "\xa0"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CSS class categorization for PK
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Document header
|
||||
DOC_HEADER_CLASS = "p4" # "Prayers of King Kulaçekhara 1958 [Handwritten]"
|
||||
TITLE_CLASS = "p115" # Bold title
|
||||
|
||||
# Chapter bookmark class
|
||||
CHAPTER_BOOKMARK_CLASS = "p17" # "Prayers of King Kulaçekhara 1950: N"
|
||||
|
||||
# Verse number markers — skip
|
||||
VERSE_MARKER_CLASSES = frozenset(["p108", "p9"])
|
||||
|
||||
# Devanagari marker — skip
|
||||
DEVANAGARI_MARKER_CLASS = "p36"
|
||||
|
||||
# Section header class (Wordings., Synonyms., Translation., Purport.)
|
||||
SECTION_HEADER_CLASSES = frozenset(["p20", "p121"])
|
||||
|
||||
# Body paragraph classes
|
||||
BODY_CLASSES = frozenset(["p6", "p34", "p38", "p71"])
|
||||
|
||||
# Page number class — skip
|
||||
PAGE_NUMBER_CLASSES = frozenset(["p15", "p41", "p124"])
|
||||
|
||||
# Empty/break classes — skip
|
||||
SKIP_CLASSES = frozenset(["p24", "p94", "p119", "p122", "p123", "p120"])
|
||||
|
||||
# Next work boundary
|
||||
NEXT_WORK_BOUNDARY_CLASS = "p125"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Boundary detection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def find_pk_boundaries(all_elements: list) -> tuple:
|
||||
"""Find the start and end element indices of the PK content.
|
||||
|
||||
Start: <p class="p4"> containing "Prayers of King Kula"
|
||||
End: <p class="p125"> that starts the next work ("Shaktipujah")
|
||||
"""
|
||||
start_idx = None
|
||||
end_idx = len(all_elements)
|
||||
|
||||
for i, el in enumerate(all_elements):
|
||||
if el.name != "p":
|
||||
continue
|
||||
cls = get_css_class(el)
|
||||
text = decode_text(el)
|
||||
|
||||
# Find start: p4 containing "Prayers of King"
|
||||
if start_idx is None:
|
||||
if cls == DOC_HEADER_CLASS and "Prayers of King" in text:
|
||||
start_idx = i
|
||||
continue
|
||||
continue
|
||||
|
||||
# Find end: p125 that starts a different work
|
||||
if cls == NEXT_WORK_BOUNDARY_CLASS:
|
||||
end_idx = i
|
||||
break
|
||||
|
||||
if start_idx is None:
|
||||
print(
|
||||
"ERROR: Could not find 'Prayers of King Kulaçekhara' in others-before-1977.html!"
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
return start_idx, end_idx
|
||||
|
||||
|
||||
def find_chapter_bookmarks(all_elements: list, start_idx: int, end_idx: int) -> list:
|
||||
"""Find all PK chapter bookmark tags (p17) in the PK range.
|
||||
|
||||
Returns list of (element_index, verse_number).
|
||||
"""
|
||||
bookmarks = []
|
||||
for i in range(start_idx, end_idx):
|
||||
el = all_elements[i]
|
||||
if el.name != "p":
|
||||
continue
|
||||
cls = get_css_class(el)
|
||||
if cls == CHAPTER_BOOKMARK_CLASS:
|
||||
text = decode_text(el).strip()
|
||||
if "Prayers of King" in text or "Kulaçekhara" in text.replace("ç", "ç"):
|
||||
# Extract the verse number from the end
|
||||
match = re.search(r":\s*(\d+)\s*$", text)
|
||||
if match:
|
||||
verse_num = int(match.group(1))
|
||||
bookmarks.append((i, verse_num))
|
||||
return bookmarks
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Table synonym extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def extract_synonyms_from_table(table: Tag) -> str:
|
||||
"""Extract synonyms from a <table class="t1"> and convert to text.
|
||||
|
||||
Each cell contains a <p> with "word=meaning" format.
|
||||
Convert to "word—meaning; word—meaning; ..." format.
|
||||
"""
|
||||
entries = []
|
||||
for row in table.find_all("tr"):
|
||||
for cell in row.find_all("td"):
|
||||
# Get all <p> tags inside the cell
|
||||
for p in cell.find_all("p"):
|
||||
text = decode_text(p).strip()
|
||||
if not text or text == "\xa0":
|
||||
continue
|
||||
# Must contain '=' to be a synonym entry
|
||||
if "=" not in text:
|
||||
continue
|
||||
# Replace = with em-dash
|
||||
text = text.replace("=", "—", 1)
|
||||
# Clean trailing whitespace/period if needed
|
||||
text = text.strip()
|
||||
entries.append(text)
|
||||
|
||||
if not entries:
|
||||
return ""
|
||||
return "; ".join(entries)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Paragraph extraction for a chapter
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def extract_chapter_paragraphs(
|
||||
all_elements: list,
|
||||
start_idx: int,
|
||||
end_idx: int,
|
||||
) -> list:
|
||||
"""Extract paragraphs for a chapter (verse).
|
||||
|
||||
Handles section headers (Wordings., Synonyms., Translation., Purport.),
|
||||
body paragraphs, and synonym tables.
|
||||
|
||||
Returns a list of HTML paragraph strings.
|
||||
"""
|
||||
result = []
|
||||
|
||||
# Build a set of <p> tags that are inside tables, so we don't double-process
|
||||
table_p_tags = set()
|
||||
for i in range(start_idx, end_idx):
|
||||
el = all_elements[i]
|
||||
if el.name == "table":
|
||||
for p in el.find_all("p"):
|
||||
table_p_tags.add(id(p))
|
||||
|
||||
for i in range(start_idx, end_idx):
|
||||
el = all_elements[i]
|
||||
|
||||
# Handle tables (synonyms)
|
||||
if el.name == "table":
|
||||
table_cls = get_css_class(el)
|
||||
if table_cls == "t1":
|
||||
synonyms_text = extract_synonyms_from_table(el)
|
||||
if synonyms_text:
|
||||
result.append(synonyms_text)
|
||||
continue
|
||||
|
||||
# Only process <p> tags from here
|
||||
if el.name != "p":
|
||||
continue
|
||||
|
||||
# Skip <p> tags that are inside tables (already processed)
|
||||
if id(el) in table_p_tags:
|
||||
continue
|
||||
|
||||
cls = get_css_class(el)
|
||||
|
||||
# Skip empty paragraphs
|
||||
if is_empty(el):
|
||||
continue
|
||||
|
||||
# Skip page numbers
|
||||
if cls in PAGE_NUMBER_CLASSES:
|
||||
continue
|
||||
|
||||
# Skip verse markers
|
||||
if cls in VERSE_MARKER_CLASSES:
|
||||
text = decode_text(el)
|
||||
# p9 can also be "(Devanägaré)" or prayer number markers — skip all
|
||||
continue
|
||||
|
||||
# Skip devanagari markers
|
||||
if cls == DEVANAGARI_MARKER_CLASS:
|
||||
continue
|
||||
|
||||
# Skip chapter bookmarks
|
||||
if cls == CHAPTER_BOOKMARK_CLASS:
|
||||
continue
|
||||
|
||||
# Skip document header and title (only at very start)
|
||||
if cls == DOC_HEADER_CLASS:
|
||||
continue
|
||||
if cls == TITLE_CLASS:
|
||||
continue
|
||||
|
||||
# Skip empty/break classes
|
||||
if cls in SKIP_CLASSES:
|
||||
continue
|
||||
|
||||
# Section headers: Wordings., Synonyms., Translation., Purport.
|
||||
if cls in SECTION_HEADER_CLASSES:
|
||||
text = decode_text(el).strip()
|
||||
# Normalize section header names
|
||||
text_lower = text.lower().rstrip(".")
|
||||
if "wording" in text_lower:
|
||||
result.append("<b>Wordings</b>")
|
||||
elif (
|
||||
"synonym" in text_lower
|
||||
or "synomym" in text_lower
|
||||
or "synonysm" in text_lower
|
||||
):
|
||||
result.append("<b>Synonyms</b>")
|
||||
elif "translation" in text_lower:
|
||||
result.append("<b>Translation</b>")
|
||||
elif "purport" in text_lower:
|
||||
result.append("<b>Purport</b>")
|
||||
else:
|
||||
# Unknown header — include as bold
|
||||
html = decode_html_content(el)
|
||||
if html:
|
||||
result.append(f"<b>{html}</b>")
|
||||
continue
|
||||
|
||||
# Body paragraphs
|
||||
if cls in BODY_CLASSES:
|
||||
html = decode_html_content(el)
|
||||
if html:
|
||||
result.append(html)
|
||||
continue
|
||||
|
||||
# Fallback: try to extract content from unknown classes
|
||||
html = decode_html_content(el)
|
||||
if html:
|
||||
result.append(html)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main extraction logic
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def extract_pk(soup: BeautifulSoup) -> dict:
|
||||
"""Extract the entire Prayers of King Kulaśekhara from the parsed HTML.
|
||||
|
||||
Returns a dict with:
|
||||
"chapters": list of chapter dicts
|
||||
"chapter_listing": list of {number, title} for intro.json
|
||||
"""
|
||||
# We need both <p> and <table> elements in document order
|
||||
all_elements = list(soup.find_all(["p", "table"]))
|
||||
|
||||
# Step 1: Find PK boundaries
|
||||
pk_start, pk_end = find_pk_boundaries(all_elements)
|
||||
print(f" PK content: elements {pk_start} to {pk_end}")
|
||||
print(f" Total elements in range: {pk_end - pk_start}")
|
||||
|
||||
# Step 2: Find all chapter bookmarks (p17)
|
||||
bookmarks = find_chapter_bookmarks(all_elements, pk_start, pk_end)
|
||||
print(f" Found {len(bookmarks)} PK chapter bookmarks:")
|
||||
for pos, verse_num in bookmarks:
|
||||
print(f" Element {pos}: Verse {verse_num}")
|
||||
|
||||
# If we didn't find 6 bookmarks, also look for the first verse that starts
|
||||
# before any p17 bookmark (verse 1 may start with p108 [1])
|
||||
# Actually, verse 1's bookmark IS in the HTML. Let's verify.
|
||||
if len(bookmarks) < 6:
|
||||
print(f" WARNING: Expected 6 bookmarks, found {len(bookmarks)}")
|
||||
|
||||
# Step 3: Build chapter ranges
|
||||
# Each bookmark marks the START of a verse section.
|
||||
# Content for verse N runs from bookmark N to bookmark N+1 (or pk_end).
|
||||
chapters = []
|
||||
for idx, (bk_pos, verse_num) in enumerate(bookmarks):
|
||||
# Content starts after the bookmark
|
||||
content_start = bk_pos + 1
|
||||
|
||||
# Content ends at next bookmark or pk_end
|
||||
if idx + 1 < len(bookmarks):
|
||||
content_end = bookmarks[idx + 1][0]
|
||||
else:
|
||||
content_end = pk_end
|
||||
|
||||
paragraphs = extract_chapter_paragraphs(
|
||||
all_elements, content_start, content_end
|
||||
)
|
||||
|
||||
ch = {
|
||||
"book": "pk",
|
||||
"chapter": verse_num,
|
||||
"title": f"Prayer {verse_num}",
|
||||
"type": "chapter",
|
||||
"paragraphs": paragraphs,
|
||||
}
|
||||
chapters.append(ch)
|
||||
|
||||
# Step 4: Build chapter listing for intro.json
|
||||
chapter_listing = [
|
||||
{"number": ch["chapter"], "title": ch["title"]} for ch in chapters
|
||||
]
|
||||
|
||||
return {
|
||||
"chapters": chapters,
|
||||
"chapter_listing": chapter_listing,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Output
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def write_json(data, filepath):
|
||||
"""Write data as formatted JSON to filepath, creating directories as needed."""
|
||||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||||
with open(filepath, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html")
|
||||
output_base = os.path.join(base_dir, "lib", "jsons", "books", "pk")
|
||||
|
||||
if not os.path.exists(html_path):
|
||||
print(f"ERROR: {html_path} not found!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Reading {html_path}...")
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
soup = BeautifulSoup(f, "html.parser")
|
||||
|
||||
print("Extracting Prayers of King Kulaśekhara (1958)...\n")
|
||||
result = extract_pk(soup)
|
||||
|
||||
chapters = result["chapters"]
|
||||
chapter_listing = result["chapter_listing"]
|
||||
|
||||
# --- Write intro.json ---
|
||||
intro_meta = {
|
||||
"book": "pk",
|
||||
"title": "Prayers of King Kulaśekhara (1958)",
|
||||
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
||||
"year": 1958,
|
||||
"description": "Early handwritten translations and purports of the Mukunda-mālā-stotra prayers (verses 1–6).",
|
||||
"totalChapters": 6,
|
||||
"chapters": chapter_listing,
|
||||
}
|
||||
intro_path = os.path.join(output_base, "intro.json")
|
||||
write_json(intro_meta, intro_path)
|
||||
print(f"\n Written: {intro_path}")
|
||||
|
||||
# --- Write chapters ---
|
||||
print(f"\n--- Chapters ---")
|
||||
for ch in chapters:
|
||||
ch_num_str = str(ch["chapter"]).zfill(2)
|
||||
filepath = os.path.join(output_base, "chapters", f"{ch_num_str}.json")
|
||||
write_json(ch, filepath)
|
||||
print(
|
||||
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — "
|
||||
f"{ch['title']}"
|
||||
)
|
||||
|
||||
# --- Summary ---
|
||||
total_chapter_paras = sum(len(ch["paragraphs"]) for ch in chapters)
|
||||
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f" EXTRACTION SUMMARY")
|
||||
print(f"{'=' * 60}")
|
||||
print(f" Chapters: {len(chapters)}")
|
||||
for ch in chapters:
|
||||
print(
|
||||
f" Ch {ch['chapter']}: {len(ch['paragraphs'])} paragraphs — {ch['title']}"
|
||||
)
|
||||
print(f" Total paragraphs: {total_chapter_paras}")
|
||||
if chapters:
|
||||
print(f" Avg paras/chapter: {total_chapter_paras / len(chapters):.1f}")
|
||||
print(f" Output directory: {output_base}/")
|
||||
|
||||
# --- Samples ---
|
||||
for ch_idx, ch in enumerate(chapters):
|
||||
print(f"\n--- Sample: Chapter {ch['chapter']} ({ch['title']}) ---")
|
||||
for i, p in enumerate(ch["paragraphs"][:5]):
|
||||
preview = re.sub(r"<[^>]+>", "", p)[:120]
|
||||
print(f" [{i}] {preview}")
|
||||
if len(ch["paragraphs"]) > 5:
|
||||
print(f" ... ({len(ch['paragraphs'])} total paragraphs)")
|
||||
|
||||
# --- Quality Checks ---
|
||||
print(f"\n--- Quality Checks ---")
|
||||
|
||||
# Check all 6 chapters present
|
||||
expected_chapters = set(range(1, 7))
|
||||
actual_chapters = set(ch["chapter"] for ch in chapters)
|
||||
missing = expected_chapters - actual_chapters
|
||||
extra = actual_chapters - expected_chapters
|
||||
if missing:
|
||||
print(f" ⚠ Missing chapters: {sorted(missing)}")
|
||||
if extra:
|
||||
print(f" ⚠ Extra chapters: {sorted(extra)}")
|
||||
if not missing and not extra:
|
||||
print(f" ✅ All 6 chapters present")
|
||||
|
||||
# Check empty chapters
|
||||
empty_chapters = [ch["chapter"] for ch in chapters if len(ch["paragraphs"]) == 0]
|
||||
if empty_chapters:
|
||||
print(f" ⚠ Empty chapters (0 paragraphs): {empty_chapters}")
|
||||
else:
|
||||
print(f" ✅ No empty chapters")
|
||||
|
||||
# Check each chapter has expected sections
|
||||
for ch in chapters:
|
||||
ch_text = " ".join(ch["paragraphs"])
|
||||
has_wordings = "<b>Wordings</b>" in ch_text
|
||||
has_synonyms = "<b>Synonyms</b>" in ch_text
|
||||
has_translation = "<b>Translation</b>" in ch_text
|
||||
has_purport = "<b>Purport</b>" in ch_text
|
||||
sections = []
|
||||
if has_wordings:
|
||||
sections.append("W")
|
||||
if has_synonyms:
|
||||
sections.append("S")
|
||||
if has_translation:
|
||||
sections.append("T")
|
||||
if has_purport:
|
||||
sections.append("P")
|
||||
status = "✅" if len(sections) == 4 else "⚠"
|
||||
print(f" {status} Ch {ch['chapter']}: sections [{', '.join(sections)}]")
|
||||
|
||||
# Check synonym entries use em-dash format
|
||||
synonym_count = 0
|
||||
for ch in chapters:
|
||||
for p in ch["paragraphs"]:
|
||||
if "—" in p and "<b>" not in p:
|
||||
synonym_count += 1
|
||||
break
|
||||
print(f" ℹ Chapters with em-dash synonyms: {synonym_count}/6")
|
||||
|
||||
# Verify no Shaktipujah content leaked in
|
||||
all_text = ""
|
||||
for ch in chapters:
|
||||
all_text += " ".join(ch["paragraphs"])
|
||||
if "Shaktipujah" in all_text or "Baidya Nath" in all_text:
|
||||
print(f" ⚠ Content from Shaktipujah may have leaked into PK!")
|
||||
else:
|
||||
print(f" ✅ No content leak from next work detected")
|
||||
|
||||
# Check last chapter ends cleanly
|
||||
if chapters:
|
||||
last_ch = chapters[-1]
|
||||
last_para = last_ch["paragraphs"][-1] if last_ch["paragraphs"] else ""
|
||||
last_preview = re.sub(r"<[^>]+>", "", last_para)[:80]
|
||||
print(f' ℹ Last paragraph of Ch {last_ch["chapter"]}: "{last_preview}..."')
|
||||
|
||||
print(f"\nDone! ✨")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,169 @@
|
||||
/**
|
||||
* seed-bk.ts — Push Bhagavāner Kathā JSON data into MongoDB.
|
||||
*
|
||||
* Uses the native MongoDB driver (same pattern as seed-cbhc.ts) to avoid
|
||||
* Prisma replica-set write requirements.
|
||||
*
|
||||
* BK is a simple prose book with no volume structure and no intro segments.
|
||||
* We use volume=0 as the sentinel for "no volumes".
|
||||
*
|
||||
* Collections written:
|
||||
* - book_meta → one doc { book: "bk", data: <intro.json> }
|
||||
* - prose_chapters → 2 chapters
|
||||
*
|
||||
* Usage:
|
||||
* npx tsx scripts/seed-bk.ts
|
||||
*/
|
||||
|
||||
import dotenv from "dotenv";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { MongoClient } from "mongodb";
|
||||
|
||||
// Load env
|
||||
dotenv.config({ path: ".env.local" });
|
||||
dotenv.config({ path: ".env" });
|
||||
|
||||
const DATABASE_URL = process.env.DATABASE_URL;
|
||||
if (!DATABASE_URL) {
|
||||
console.error("❌ DATABASE_URL is not set in .env");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const BASE_DIR = path.resolve(__dirname, "..");
|
||||
const BK_DIR = path.join(BASE_DIR, "lib", "jsons", "books", "bk");
|
||||
|
||||
/** Volume sentinel for books without volumes. */
|
||||
const NO_VOLUME = 0;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function readJson<T>(filePath: string): T {
|
||||
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert documents one at a time with a small delay every N docs.
|
||||
* Uses insertOne to avoid BSON wire-protocol corruption.
|
||||
*/
|
||||
async function insertBatched(
|
||||
col: ReturnType<ReturnType<MongoClient["db"]>["collection"]>,
|
||||
docs: Record<string, unknown>[],
|
||||
) {
|
||||
for (let i = 0; i < docs.length; i++) {
|
||||
await col.insertOne(docs[i]);
|
||||
if ((i + 1) % 100 === 0) {
|
||||
await new Promise((r) => setTimeout(r, 100));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types for the JSON files
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type BkIntroJson = {
|
||||
book: string;
|
||||
title: string;
|
||||
author: string;
|
||||
year: number | null;
|
||||
description: string;
|
||||
totalChapters: number;
|
||||
chapters: { number: number; title: string }[];
|
||||
};
|
||||
|
||||
type BkChapterJson = {
|
||||
book: string;
|
||||
chapter: number;
|
||||
title: string;
|
||||
type: string;
|
||||
paragraphs: string[];
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Seed function
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function seedBk(db: ReturnType<MongoClient["db"]>) {
|
||||
const introPath = path.join(BK_DIR, "intro.json");
|
||||
if (!fs.existsSync(introPath)) {
|
||||
console.error("❌ BK intro.json not found at", introPath);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const intro = readJson<BkIntroJson>(introPath);
|
||||
|
||||
// ---- 1. Book meta ----
|
||||
const metaCol = db.collection("book_meta");
|
||||
await metaCol.deleteMany({ book: "bk" });
|
||||
await metaCol.insertOne({ book: "bk", data: intro });
|
||||
console.log(" 📖 BK book meta inserted");
|
||||
|
||||
// ---- 2. Prose chapters collection ----
|
||||
const proseCol = db.collection("prose_chapters");
|
||||
await proseCol.deleteMany({ book: "bk" });
|
||||
|
||||
const docs: Record<string, unknown>[] = [];
|
||||
|
||||
// Chapters (01.json through 02.json)
|
||||
const chaptersDir = path.join(BK_DIR, "chapters");
|
||||
if (fs.existsSync(chaptersDir)) {
|
||||
const chapterFiles = fs
|
||||
.readdirSync(chaptersDir)
|
||||
.filter((f) => f.endsWith(".json"))
|
||||
.sort();
|
||||
|
||||
for (const file of chapterFiles) {
|
||||
const ch = readJson<BkChapterJson>(path.join(chaptersDir, file));
|
||||
const chNum = ch.chapter ?? parseInt(file.replace(".json", ""), 10);
|
||||
docs.push({
|
||||
book: "bk",
|
||||
volume: NO_VOLUME,
|
||||
chapter: chNum,
|
||||
title: ch.title,
|
||||
type: "chapter",
|
||||
slug: String(chNum),
|
||||
order: 100 + chNum,
|
||||
paragraphs: ch.paragraphs,
|
||||
});
|
||||
}
|
||||
console.log(` 📖 Chapters: ${chapterFiles.length}`);
|
||||
}
|
||||
|
||||
// Insert all docs
|
||||
console.log(`\n Inserting ${docs.length} documents...`);
|
||||
await insertBatched(proseCol, docs);
|
||||
|
||||
const totalDocs = await proseCol.countDocuments({ book: "bk" });
|
||||
console.log(` ✅ BK: ${totalDocs} prose entries in MongoDB`);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function main() {
|
||||
console.log("🌱 Seeding Bhagavāner Kathā...\n");
|
||||
|
||||
const client = new MongoClient(DATABASE_URL!);
|
||||
await client.connect();
|
||||
console.log("📡 Connected to MongoDB\n");
|
||||
|
||||
const db = client.db();
|
||||
|
||||
const start = Date.now();
|
||||
|
||||
await seedBk(db);
|
||||
|
||||
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
|
||||
console.log(`\n🎉 Done in ${elapsed}s`);
|
||||
|
||||
await client.close();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("❌ Seed failed:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,169 @@
|
||||
/**
|
||||
* seed-cbhc.ts — Push Śrī Caitanya-bhāgavata Commentary JSON data into MongoDB.
|
||||
*
|
||||
* Uses the native MongoDB driver (same pattern as seed-mog.ts) to avoid
|
||||
* Prisma replica-set write requirements.
|
||||
*
|
||||
* CBHC is a simple prose book with no volume structure and no intro segments.
|
||||
* We use volume=0 as the sentinel for "no volumes".
|
||||
*
|
||||
* Collections written:
|
||||
* - book_meta → one doc { book: "cbhc", data: <intro.json> }
|
||||
* - prose_chapters → 4 chapters
|
||||
*
|
||||
* Usage:
|
||||
* npx tsx scripts/seed-cbhc.ts
|
||||
*/
|
||||
|
||||
import dotenv from "dotenv";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { MongoClient } from "mongodb";
|
||||
|
||||
// Load env
|
||||
dotenv.config({ path: ".env.local" });
|
||||
dotenv.config({ path: ".env" });
|
||||
|
||||
const DATABASE_URL = process.env.DATABASE_URL;
|
||||
if (!DATABASE_URL) {
|
||||
console.error("❌ DATABASE_URL is not set in .env");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const BASE_DIR = path.resolve(__dirname, "..");
|
||||
const CBHC_DIR = path.join(BASE_DIR, "lib", "jsons", "books", "cbhc");
|
||||
|
||||
/** Volume sentinel for books without volumes. */
|
||||
const NO_VOLUME = 0;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function readJson<T>(filePath: string): T {
|
||||
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert documents one at a time with a small delay every N docs.
|
||||
* Uses insertOne to avoid BSON wire-protocol corruption.
|
||||
*/
|
||||
async function insertBatched(
|
||||
col: ReturnType<ReturnType<MongoClient["db"]>["collection"]>,
|
||||
docs: Record<string, unknown>[],
|
||||
) {
|
||||
for (let i = 0; i < docs.length; i++) {
|
||||
await col.insertOne(docs[i]);
|
||||
if ((i + 1) % 100 === 0) {
|
||||
await new Promise((r) => setTimeout(r, 100));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types for the JSON files
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type CbhcIntroJson = {
|
||||
book: string;
|
||||
title: string;
|
||||
author: string;
|
||||
year: number | null;
|
||||
description: string;
|
||||
totalChapters: number;
|
||||
chapters: { number: number; title: string }[];
|
||||
};
|
||||
|
||||
type CbhcChapterJson = {
|
||||
book: string;
|
||||
chapter: number;
|
||||
title: string;
|
||||
type: string;
|
||||
paragraphs: string[];
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Seed function
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function seedCbhc(db: ReturnType<MongoClient["db"]>) {
|
||||
const introPath = path.join(CBHC_DIR, "intro.json");
|
||||
if (!fs.existsSync(introPath)) {
|
||||
console.error("❌ CBHC intro.json not found at", introPath);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const intro = readJson<CbhcIntroJson>(introPath);
|
||||
|
||||
// ---- 1. Book meta ----
|
||||
const metaCol = db.collection("book_meta");
|
||||
await metaCol.deleteMany({ book: "cbhc" });
|
||||
await metaCol.insertOne({ book: "cbhc", data: intro });
|
||||
console.log(" 📖 CBHC book meta inserted");
|
||||
|
||||
// ---- 2. Prose chapters collection ----
|
||||
const proseCol = db.collection("prose_chapters");
|
||||
await proseCol.deleteMany({ book: "cbhc" });
|
||||
|
||||
const docs: Record<string, unknown>[] = [];
|
||||
|
||||
// Chapters (01.json through 04.json)
|
||||
const chaptersDir = path.join(CBHC_DIR, "chapters");
|
||||
if (fs.existsSync(chaptersDir)) {
|
||||
const chapterFiles = fs
|
||||
.readdirSync(chaptersDir)
|
||||
.filter((f) => f.endsWith(".json"))
|
||||
.sort();
|
||||
|
||||
for (const file of chapterFiles) {
|
||||
const ch = readJson<CbhcChapterJson>(path.join(chaptersDir, file));
|
||||
const chNum = ch.chapter ?? parseInt(file.replace(".json", ""), 10);
|
||||
docs.push({
|
||||
book: "cbhc",
|
||||
volume: NO_VOLUME,
|
||||
chapter: chNum,
|
||||
title: ch.title,
|
||||
type: "chapter",
|
||||
slug: String(chNum),
|
||||
order: 100 + chNum,
|
||||
paragraphs: ch.paragraphs,
|
||||
});
|
||||
}
|
||||
console.log(` 📖 Chapters: ${chapterFiles.length}`);
|
||||
}
|
||||
|
||||
// Insert all docs
|
||||
console.log(`\n Inserting ${docs.length} documents...`);
|
||||
await insertBatched(proseCol, docs);
|
||||
|
||||
const totalDocs = await proseCol.countDocuments({ book: "cbhc" });
|
||||
console.log(` ✅ CBHC: ${totalDocs} prose entries in MongoDB`);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function main() {
|
||||
console.log("🌱 Seeding Śrī Caitanya-bhāgavata Commentary...\n");
|
||||
|
||||
const client = new MongoClient(DATABASE_URL!);
|
||||
await client.connect();
|
||||
console.log("📡 Connected to MongoDB\n");
|
||||
|
||||
const db = client.db();
|
||||
|
||||
const start = Date.now();
|
||||
|
||||
await seedCbhc(db);
|
||||
|
||||
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
|
||||
console.log(`\n🎉 Done in ${elapsed}s`);
|
||||
|
||||
await client.close();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("❌ Seed failed:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,163 @@
|
||||
/**
|
||||
* seed-cc-essays.ts — Push all five "Essays on Śrī Caitanya-caritāmṛta" into MongoDB.
|
||||
*
|
||||
* Uses the native MongoDB driver (same pattern as seed-te.ts) to avoid
|
||||
* Prisma replica-set write requirements.
|
||||
*
|
||||
* Collections written:
|
||||
* - essays → 5 essay docs (upsert by slug)
|
||||
*
|
||||
* JSON sources:
|
||||
* - lib/jsons/essays/the-spiritual-masters.json
|
||||
* - lib/jsons/essays/the-absolute-truth.json
|
||||
* - lib/jsons/essays/causes-of-lord-caitanyas-incarnation.json
|
||||
* - lib/jsons/essays/confidential-reasons-for-lord-caitanyas-appearance.json
|
||||
* - lib/jsons/essays/glories-of-lord-nityananda.json
|
||||
*
|
||||
* Usage:
|
||||
* npx tsx scripts/seed-cc-essays.ts
|
||||
*/
|
||||
|
||||
import dotenv from "dotenv";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { MongoClient } from "mongodb";
|
||||
|
||||
// Load env
|
||||
dotenv.config({ path: ".env.local" });
|
||||
dotenv.config({ path: ".env" });
|
||||
|
||||
const DATABASE_URL = process.env.DATABASE_URL;
|
||||
if (!DATABASE_URL) {
|
||||
console.error("❌ DATABASE_URL is not set in .env");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const BASE_DIR = path.resolve(__dirname, "..");
|
||||
const ESSAYS_DIR = path.join(BASE_DIR, "lib", "jsons", "essays");
|
||||
|
||||
const ESSAY_FILES = [
|
||||
"the-spiritual-masters.json",
|
||||
"the-absolute-truth.json",
|
||||
"causes-of-lord-caitanyas-incarnation.json",
|
||||
"confidential-reasons-for-lord-caitanyas-appearance.json",
|
||||
"glories-of-lord-nityananda.json",
|
||||
];
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type EssayJson = {
|
||||
slug: string;
|
||||
title: string;
|
||||
author: string;
|
||||
year: number;
|
||||
group: string;
|
||||
order: number;
|
||||
paragraphs: string[];
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function readJson<T>(filePath: string): T {
|
||||
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Seed function
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function seedCcEssays(db: ReturnType<MongoClient["db"]>) {
|
||||
const essaysCol = db.collection("essays");
|
||||
|
||||
let inserted = 0;
|
||||
let updated = 0;
|
||||
let unchanged = 0;
|
||||
|
||||
for (const filename of ESSAY_FILES) {
|
||||
const jsonPath = path.join(ESSAYS_DIR, filename);
|
||||
|
||||
if (!fs.existsSync(jsonPath)) {
|
||||
console.error(` ❌ Missing: ${jsonPath}`);
|
||||
console.error(" Run `python3 scripts/cc_essays_extract.py` first.");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const essay = readJson<EssayJson>(jsonPath);
|
||||
|
||||
const result = await essaysCol.updateOne(
|
||||
{ slug: essay.slug },
|
||||
{
|
||||
$set: {
|
||||
slug: essay.slug,
|
||||
title: essay.title,
|
||||
author: essay.author,
|
||||
year: essay.year,
|
||||
group: essay.group,
|
||||
order: essay.order,
|
||||
paragraphs: essay.paragraphs,
|
||||
},
|
||||
},
|
||||
{ upsert: true },
|
||||
);
|
||||
|
||||
if (result.upsertedCount > 0) {
|
||||
console.log(
|
||||
` 📝 Inserted "${essay.title}" (${essay.paragraphs.length} paragraphs)`,
|
||||
);
|
||||
inserted++;
|
||||
} else if (result.modifiedCount > 0) {
|
||||
console.log(
|
||||
` 📝 Updated "${essay.title}" (${essay.paragraphs.length} paragraphs)`,
|
||||
);
|
||||
updated++;
|
||||
} else {
|
||||
console.log(` 📝 "${essay.title}" already up to date`);
|
||||
unchanged++;
|
||||
}
|
||||
}
|
||||
|
||||
// Verify
|
||||
const totalDocs = await essaysCol.countDocuments({
|
||||
group: "cc-essays",
|
||||
});
|
||||
console.log(
|
||||
`\n ✅ Verified: ${totalDocs} document(s) in essays collection with group "cc-essays"`,
|
||||
);
|
||||
|
||||
// Summary
|
||||
console.log(
|
||||
`\n Summary: ${inserted} inserted, ${updated} updated, ${unchanged} unchanged`,
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function main() {
|
||||
console.log("🌱 Seeding CC Essays (5 essays)...\n");
|
||||
|
||||
const client = new MongoClient(DATABASE_URL!);
|
||||
await client.connect();
|
||||
console.log("📡 Connected to MongoDB\n");
|
||||
|
||||
const db = client.db();
|
||||
|
||||
const start = Date.now();
|
||||
|
||||
await seedCcEssays(db);
|
||||
|
||||
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
|
||||
console.log(`\n🎉 Done in ${elapsed}s`);
|
||||
|
||||
await client.close();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("❌ Seed failed:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,171 @@
|
||||
/**
|
||||
* seed-journals.ts — Push Journals into MongoDB.
|
||||
*
|
||||
* Collections written:
|
||||
* - journal_meta → one doc per journal
|
||||
* - journal_chapters → N chapter docs per journal
|
||||
*
|
||||
* Usage:
|
||||
* npx tsx scripts/seed-journals.ts
|
||||
*/
|
||||
|
||||
import dotenv from "dotenv";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { MongoClient } from "mongodb";
|
||||
|
||||
dotenv.config({ path: ".env.local" });
|
||||
dotenv.config({ path: ".env" });
|
||||
|
||||
const DATABASE_URL = process.env.DATABASE_URL;
|
||||
if (!DATABASE_URL) {
|
||||
console.error("❌ DATABASE_URL is not set");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const BASE_DIR = path.resolve(__dirname, "..");
|
||||
const JOURNALS_DIR = path.join(BASE_DIR, "lib", "jsons", "journals");
|
||||
|
||||
type IntroJson = {
|
||||
book: string;
|
||||
title: string;
|
||||
author: string;
|
||||
year: number | null;
|
||||
description: string | null;
|
||||
totalChapters: number;
|
||||
chapters: unknown[];
|
||||
};
|
||||
type ChapterJson = {
|
||||
book: string;
|
||||
chapter: number;
|
||||
title: string;
|
||||
type: string;
|
||||
slug?: string;
|
||||
paragraphs: string[];
|
||||
};
|
||||
|
||||
function readJson<T>(filePath: string): T {
|
||||
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
|
||||
}
|
||||
|
||||
function sanitizeDoc(obj: unknown): unknown {
|
||||
if (typeof obj === "string") {
|
||||
let s = obj.normalize("NFC");
|
||||
s = s.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, "");
|
||||
s = s.replace(/[\uD800-\uDFFF]/g, "");
|
||||
s = s.replace(/[\u200B-\u200F\u2028-\u202F\uFEFF]/g, "");
|
||||
s = Buffer.from(s, "utf-8").toString("utf-8");
|
||||
return s;
|
||||
}
|
||||
if (Array.isArray(obj)) return obj.map(sanitizeDoc);
|
||||
if (obj !== null && typeof obj === "object") {
|
||||
const result: Record<string, unknown> = {};
|
||||
for (const [key, value] of Object.entries(obj as Record<string, unknown>))
|
||||
result[key] = sanitizeDoc(value);
|
||||
return result;
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
async function seedOneJournal(
|
||||
db: ReturnType<MongoClient["db"]>,
|
||||
docDir: string,
|
||||
): Promise<{ bookCode: string; chapters: number }> {
|
||||
const introPath = path.join(docDir, "intro.json");
|
||||
if (!fs.existsSync(introPath)) throw new Error(`No intro.json in ${docDir}`);
|
||||
const intro = sanitizeDoc(readJson<IntroJson>(introPath)) as IntroJson;
|
||||
const bookCode = intro.book;
|
||||
|
||||
const metaCol = db.collection("journal_meta");
|
||||
await metaCol.deleteMany({ book: bookCode });
|
||||
await metaCol.insertOne({ book: bookCode, data: intro });
|
||||
console.log(` 📖 Inserted journal_meta for "${intro.title}" (${bookCode})`);
|
||||
|
||||
const chaptersCol = db.collection("journal_chapters");
|
||||
await chaptersCol.deleteMany({ book: bookCode });
|
||||
|
||||
const chaptersDir = path.join(docDir, "chapters");
|
||||
let inserted = 0;
|
||||
if (fs.existsSync(chaptersDir)) {
|
||||
const files = fs
|
||||
.readdirSync(chaptersDir)
|
||||
.filter((f) => f.endsWith(".json"))
|
||||
.sort();
|
||||
for (const file of files) {
|
||||
const ch = sanitizeDoc(
|
||||
readJson<ChapterJson>(path.join(chaptersDir, file)),
|
||||
) as ChapterJson;
|
||||
const chNum = ch.chapter ?? parseInt(file.replace(".json", ""), 10);
|
||||
const slug = ch.slug ?? String(chNum);
|
||||
await chaptersCol.insertOne({
|
||||
book: bookCode,
|
||||
chapter: chNum,
|
||||
title: ch.title,
|
||||
type: ch.type || "chapter",
|
||||
slug,
|
||||
order: 100 + chNum,
|
||||
paragraphs: ch.paragraphs,
|
||||
});
|
||||
inserted++;
|
||||
}
|
||||
console.log(` 📄 Chapters: ${inserted}`);
|
||||
}
|
||||
|
||||
const total = await chaptersCol.countDocuments({ book: bookCode });
|
||||
console.log(` ✅ ${bookCode}: ${total} chapter entries`);
|
||||
return { bookCode, chapters: inserted };
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log("🌱 Seeding Journals...\n");
|
||||
const client = new MongoClient(DATABASE_URL!);
|
||||
await client.connect();
|
||||
console.log("📡 Connected to MongoDB\n");
|
||||
const db = client.db();
|
||||
const start = Date.now();
|
||||
|
||||
if (!fs.existsSync(JOURNALS_DIR)) {
|
||||
console.error(`❌ ${JOURNALS_DIR} not found`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const subdirs = fs
|
||||
.readdirSync(JOURNALS_DIR)
|
||||
.filter((e) => {
|
||||
const p = path.join(JOURNALS_DIR, e);
|
||||
return (
|
||||
fs.statSync(p).isDirectory() &&
|
||||
fs.existsSync(path.join(p, "intro.json"))
|
||||
);
|
||||
})
|
||||
.sort();
|
||||
|
||||
if (subdirs.length === 0) {
|
||||
console.error("❌ No journal directories found");
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(`📂 Found ${subdirs.length} journal(s)\n`);
|
||||
|
||||
let totalChapters = 0;
|
||||
const bookCodes: string[] = [];
|
||||
for (const subdir of subdirs) {
|
||||
console.log(`\n📝 Processing: ${subdir}`);
|
||||
const result = await seedOneJournal(db, path.join(JOURNALS_DIR, subdir));
|
||||
bookCodes.push(result.bookCode);
|
||||
totalChapters += result.chapters;
|
||||
}
|
||||
|
||||
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
|
||||
console.log("\n" + "─".repeat(50));
|
||||
console.log(
|
||||
`📊 Summary: ${subdirs.length} journals, ${totalChapters} chapters`,
|
||||
);
|
||||
console.log(` Book codes: ${bookCodes.join(", ")}`);
|
||||
console.log(`\n🎉 Done in ${elapsed}s`);
|
||||
await client.close();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("❌ Seed failed:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,204 @@
|
||||
/**
|
||||
* seed-mog.ts — Push Message of Godhead JSON data into MongoDB.
|
||||
*
|
||||
* Uses the native MongoDB driver (same pattern as seed-mg.ts) to avoid
|
||||
* Prisma replica-set write requirements.
|
||||
*
|
||||
* MOG is a prose book with no volume structure.
|
||||
* We use volume=0 as the sentinel for "no volumes".
|
||||
*
|
||||
* Collections written:
|
||||
* - book_meta → one doc { book: "mog", data: <intro.json> }
|
||||
* - prose_chapters → 1 intro segment + 3 chapters = 4 docs
|
||||
*
|
||||
* Usage:
|
||||
* npx tsx scripts/seed-mog.ts
|
||||
*/
|
||||
|
||||
import dotenv from "dotenv";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { MongoClient } from "mongodb";
|
||||
|
||||
// Load env
|
||||
dotenv.config({ path: ".env.local" });
|
||||
dotenv.config({ path: ".env" });
|
||||
|
||||
const DATABASE_URL = process.env.DATABASE_URL;
|
||||
if (!DATABASE_URL) {
|
||||
console.error("❌ DATABASE_URL is not set in .env");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const BASE_DIR = path.resolve(__dirname, "..");
|
||||
const MOG_DIR = path.join(BASE_DIR, "lib", "jsons", "books", "mog");
|
||||
|
||||
/** Volume sentinel for books without volumes. */
|
||||
const NO_VOLUME = 0;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function readJson<T>(filePath: string): T {
|
||||
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert documents one at a time with a small delay every N docs.
|
||||
* Uses insertOne to avoid BSON wire-protocol corruption.
|
||||
*/
|
||||
async function insertBatched(
|
||||
col: ReturnType<ReturnType<MongoClient["db"]>["collection"]>,
|
||||
docs: Record<string, unknown>[],
|
||||
) {
|
||||
for (let i = 0; i < docs.length; i++) {
|
||||
await col.insertOne(docs[i]);
|
||||
if ((i + 1) % 100 === 0) {
|
||||
await new Promise((r) => setTimeout(r, 100));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types for the JSON files
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type MogIntroJson = {
|
||||
book: string;
|
||||
title: string;
|
||||
author: string;
|
||||
year: number;
|
||||
description: string;
|
||||
totalChapters: number;
|
||||
chapters: { number: number; title: string }[];
|
||||
};
|
||||
|
||||
type MogIntroSegmentJson = {
|
||||
slug: string;
|
||||
title: string;
|
||||
type: string;
|
||||
book: string;
|
||||
order: number;
|
||||
paragraphs: string[];
|
||||
};
|
||||
|
||||
type MogChapterJson = {
|
||||
book: string;
|
||||
chapter: number;
|
||||
title: string;
|
||||
type: string;
|
||||
paragraphs: string[];
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Seed function
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function seedMog(db: ReturnType<MongoClient["db"]>) {
|
||||
const introPath = path.join(MOG_DIR, "intro.json");
|
||||
if (!fs.existsSync(introPath)) {
|
||||
console.error("❌ MOG intro.json not found at", introPath);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const intro = readJson<MogIntroJson>(introPath);
|
||||
|
||||
// ---- 1. Book meta ----
|
||||
const metaCol = db.collection("book_meta");
|
||||
await metaCol.deleteMany({ book: "mog" });
|
||||
await metaCol.insertOne({ book: "mog", data: intro });
|
||||
console.log(" 📖 MOG book meta inserted");
|
||||
|
||||
// ---- 2. Prose chapters collection ----
|
||||
const proseCol = db.collection("prose_chapters");
|
||||
await proseCol.deleteMany({ book: "mog" });
|
||||
|
||||
const docs: Record<string, unknown>[] = [];
|
||||
|
||||
// Intro segments (front-pages)
|
||||
const introSegmentFiles = ["front-pages.json"];
|
||||
|
||||
const introDir = path.join(MOG_DIR, "intro");
|
||||
for (const file of introSegmentFiles) {
|
||||
const filePath = path.join(introDir, file);
|
||||
if (!fs.existsSync(filePath)) {
|
||||
console.warn(` ⚠️ Intro segment not found, skipping: ${file}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const seg = readJson<MogIntroSegmentJson>(filePath);
|
||||
|
||||
docs.push({
|
||||
book: "mog",
|
||||
volume: NO_VOLUME,
|
||||
chapter: null,
|
||||
title: seg.title,
|
||||
type: "intro-segment",
|
||||
slug: `intro-${seg.slug}`,
|
||||
order: seg.order,
|
||||
paragraphs: seg.paragraphs,
|
||||
});
|
||||
}
|
||||
console.log(` 📄 Intro segments: ${docs.length}`);
|
||||
|
||||
// Chapters (01.json through 03.json)
|
||||
const chaptersDir = path.join(MOG_DIR, "chapters");
|
||||
if (fs.existsSync(chaptersDir)) {
|
||||
const chapterFiles = fs
|
||||
.readdirSync(chaptersDir)
|
||||
.filter((f) => f.endsWith(".json"))
|
||||
.sort();
|
||||
|
||||
for (const file of chapterFiles) {
|
||||
const ch = readJson<MogChapterJson>(path.join(chaptersDir, file));
|
||||
const chNum = ch.chapter ?? parseInt(file.replace(".json", ""), 10);
|
||||
docs.push({
|
||||
book: "mog",
|
||||
volume: NO_VOLUME,
|
||||
chapter: chNum,
|
||||
title: ch.title,
|
||||
type: "chapter",
|
||||
slug: String(chNum),
|
||||
order: 100 + chNum,
|
||||
paragraphs: ch.paragraphs,
|
||||
});
|
||||
}
|
||||
console.log(` 📖 Chapters: ${chapterFiles.length}`);
|
||||
}
|
||||
|
||||
// Insert all docs
|
||||
console.log(`\n Inserting ${docs.length} documents...`);
|
||||
await insertBatched(proseCol, docs);
|
||||
|
||||
const totalDocs = await proseCol.countDocuments({ book: "mog" });
|
||||
console.log(` ✅ MOG: ${totalDocs} prose entries in MongoDB`);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function main() {
|
||||
console.log("🌱 Seeding Message of Godhead...\n");
|
||||
|
||||
const client = new MongoClient(DATABASE_URL!);
|
||||
await client.connect();
|
||||
console.log("📡 Connected to MongoDB\n");
|
||||
|
||||
const db = client.db();
|
||||
|
||||
const start = Date.now();
|
||||
|
||||
await seedMog(db);
|
||||
|
||||
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
|
||||
console.log(`\n🎉 Done in ${elapsed}s`);
|
||||
|
||||
await client.close();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("❌ Seed failed:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,274 @@
|
||||
/**
|
||||
* seed-other-documents.ts — Push "Other Documents" into MongoDB.
|
||||
*
|
||||
* Uses dedicated collections for other documents (not shared with books):
|
||||
* - other_document_meta → one doc per document { book: "<slug>", data: <intro.json> }
|
||||
* - other_document_chapters → N chapter docs per document
|
||||
*
|
||||
* Each "other document" is a subdirectory under lib/jsons/other-documents/
|
||||
* with an intro.json and chapters/*.json.
|
||||
*
|
||||
* Book codes have NO prefix (e.g. "analysis-bg", "real-peace-formula").
|
||||
*
|
||||
* Usage:
|
||||
* npx tsx scripts/seed-other-documents.ts
|
||||
*/
|
||||
|
||||
import dotenv from "dotenv";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { MongoClient } from "mongodb";
|
||||
|
||||
// Load env
|
||||
dotenv.config({ path: ".env.local" });
|
||||
dotenv.config({ path: ".env" });
|
||||
|
||||
const DATABASE_URL = process.env.DATABASE_URL;
|
||||
if (!DATABASE_URL) {
|
||||
console.error("❌ DATABASE_URL is not set in .env");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const BASE_DIR = path.resolve(__dirname, "..");
|
||||
const OTHER_DOCS_DIR = path.join(BASE_DIR, "lib", "jsons", "other-documents");
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type IntroJson = {
|
||||
book: string;
|
||||
title: string;
|
||||
author: string;
|
||||
year: number | null;
|
||||
description: string | null;
|
||||
totalChapters: number;
|
||||
chapters: { number: number; title: string }[];
|
||||
};
|
||||
|
||||
type ChapterJson = {
|
||||
book: string;
|
||||
chapter: number;
|
||||
title: string;
|
||||
type: string;
|
||||
paragraphs: string[];
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function readJson<T>(filePath: string): T {
|
||||
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively sanitize all strings in a document to prevent BSON encoding errors.
|
||||
*/
|
||||
function sanitizeDoc(obj: unknown): unknown {
|
||||
if (typeof obj === "string") {
|
||||
let s = obj.normalize("NFC");
|
||||
s = s.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, "");
|
||||
s = s.replace(/[\uD800-\uDFFF]/g, "");
|
||||
s = s.replace(/[\u200B-\u200F\u2028-\u202F\uFEFF]/g, "");
|
||||
s = Buffer.from(s, "utf-8").toString("utf-8");
|
||||
return s;
|
||||
}
|
||||
if (Array.isArray(obj)) {
|
||||
return obj.map(sanitizeDoc);
|
||||
}
|
||||
if (obj !== null && typeof obj === "object") {
|
||||
const result: Record<string, unknown> = {};
|
||||
for (const [key, value] of Object.entries(obj as Record<string, unknown>)) {
|
||||
result[key] = sanitizeDoc(value);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Seed one document
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function seedOneDocument(
|
||||
db: ReturnType<MongoClient["db"]>,
|
||||
docDir: string,
|
||||
): Promise<{ bookCode: string; chapters: number }> {
|
||||
const introPath = path.join(docDir, "intro.json");
|
||||
if (!fs.existsSync(introPath)) {
|
||||
throw new Error(`No intro.json found in ${docDir}`);
|
||||
}
|
||||
|
||||
const intro = sanitizeDoc(readJson<IntroJson>(introPath)) as IntroJson;
|
||||
const bookCode = intro.book;
|
||||
|
||||
// ---- Document meta ----
|
||||
const metaCol = db.collection("other_document_meta");
|
||||
await metaCol.deleteMany({ book: bookCode });
|
||||
await metaCol.insertOne({ book: bookCode, data: intro });
|
||||
console.log(
|
||||
` 📖 Inserted other_document_meta for "${intro.title}" (${bookCode})`,
|
||||
);
|
||||
|
||||
// ---- Document chapters ----
|
||||
const chaptersCol = db.collection("other_document_chapters");
|
||||
await chaptersCol.deleteMany({ book: bookCode });
|
||||
|
||||
const docs: Record<string, unknown>[] = [];
|
||||
|
||||
const chaptersDir = path.join(docDir, "chapters");
|
||||
if (fs.existsSync(chaptersDir)) {
|
||||
const chapterFiles = fs
|
||||
.readdirSync(chaptersDir)
|
||||
.filter((f) => f.endsWith(".json"))
|
||||
.sort();
|
||||
|
||||
for (const file of chapterFiles) {
|
||||
const ch = sanitizeDoc(
|
||||
readJson<ChapterJson>(path.join(chaptersDir, file)),
|
||||
) as ChapterJson;
|
||||
|
||||
const chNum = ch.chapter ?? parseInt(file.replace(".json", ""), 10);
|
||||
docs.push({
|
||||
book: bookCode,
|
||||
chapter: chNum,
|
||||
title: ch.title,
|
||||
type: ch.type || "chapter",
|
||||
slug: String(chNum),
|
||||
order: 100 + chNum,
|
||||
paragraphs: ch.paragraphs,
|
||||
});
|
||||
}
|
||||
console.log(` 📄 Chapters: ${chapterFiles.length}`);
|
||||
}
|
||||
|
||||
// Insert all chapter docs
|
||||
if (docs.length > 0) {
|
||||
console.log(
|
||||
` 💾 Inserting ${docs.length} other_document_chapters docs...`,
|
||||
);
|
||||
for (const doc of docs) {
|
||||
await chaptersCol.insertOne(doc);
|
||||
}
|
||||
}
|
||||
|
||||
const totalDocs = await chaptersCol.countDocuments({ book: bookCode });
|
||||
console.log(` ✅ ${bookCode}: ${totalDocs} chapter entries in MongoDB`);
|
||||
|
||||
return { bookCode, chapters: docs.length };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Clean up old data from shared collections
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function cleanupOldData(db: ReturnType<MongoClient["db"]>) {
|
||||
// Remove od- prefixed entries from book_meta (old location)
|
||||
const bookMetaCol = db.collection("book_meta");
|
||||
const deletedMeta = await bookMetaCol.deleteMany({
|
||||
book: { $regex: /^od-/ },
|
||||
});
|
||||
if (deletedMeta.deletedCount > 0) {
|
||||
console.log(
|
||||
` 🗑️ Deleted ${deletedMeta.deletedCount} od-* doc(s) from book_meta`,
|
||||
);
|
||||
}
|
||||
|
||||
// Remove od- prefixed entries from prose_chapters (old location)
|
||||
const proseCol = db.collection("prose_chapters");
|
||||
const deletedProse = await proseCol.deleteMany({
|
||||
book: { $regex: /^od-/ },
|
||||
});
|
||||
if (deletedProse.deletedCount > 0) {
|
||||
console.log(
|
||||
` 🗑️ Deleted ${deletedProse.deletedCount} od-* doc(s) from prose_chapters`,
|
||||
);
|
||||
}
|
||||
|
||||
// Clean up legacy other_documents collection
|
||||
const collections = await db
|
||||
.listCollections({ name: "other_documents" })
|
||||
.toArray();
|
||||
if (collections.length > 0) {
|
||||
const col = db.collection("other_documents");
|
||||
const count = await col.countDocuments();
|
||||
if (count > 0) {
|
||||
await col.deleteMany({});
|
||||
console.log(
|
||||
` 🗑️ Deleted ${count} doc(s) from legacy other_documents collection`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function main() {
|
||||
console.log("🌱 Seeding Other Documents...\n");
|
||||
|
||||
const client = new MongoClient(DATABASE_URL!);
|
||||
await client.connect();
|
||||
console.log("📡 Connected to MongoDB\n");
|
||||
|
||||
const db = client.db();
|
||||
const start = Date.now();
|
||||
|
||||
// Discover all subdirectories
|
||||
if (!fs.existsSync(OTHER_DOCS_DIR)) {
|
||||
console.error(`❌ Directory not found: ${OTHER_DOCS_DIR}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const subdirs = fs
|
||||
.readdirSync(OTHER_DOCS_DIR)
|
||||
.filter((entry) => {
|
||||
const fullPath = path.join(OTHER_DOCS_DIR, entry);
|
||||
return (
|
||||
fs.statSync(fullPath).isDirectory() &&
|
||||
fs.existsSync(path.join(fullPath, "intro.json"))
|
||||
);
|
||||
})
|
||||
.sort();
|
||||
|
||||
if (subdirs.length === 0) {
|
||||
console.error(`❌ No document directories found in ${OTHER_DOCS_DIR}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`📂 Found ${subdirs.length} document(s)\n`);
|
||||
|
||||
// Seed each document
|
||||
let totalChapters = 0;
|
||||
const bookCodes: string[] = [];
|
||||
|
||||
for (const subdir of subdirs) {
|
||||
console.log(`\n📝 Processing: ${subdir}`);
|
||||
const docDir = path.join(OTHER_DOCS_DIR, subdir);
|
||||
const result = await seedOneDocument(db, docDir);
|
||||
bookCodes.push(result.bookCode);
|
||||
totalChapters += result.chapters;
|
||||
}
|
||||
|
||||
// Clean up old data from shared collections
|
||||
console.log("\n🧹 Cleaning up old data from shared collections...");
|
||||
await cleanupOldData(db);
|
||||
|
||||
// Summary
|
||||
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
|
||||
console.log("\n" + "─".repeat(50));
|
||||
console.log(`📊 Summary:`);
|
||||
console.log(` Documents: ${subdirs.length}`);
|
||||
console.log(` Book codes: ${bookCodes.join(", ")}`);
|
||||
console.log(` Total chapters: ${totalChapters}`);
|
||||
console.log(`\n🎉 Done in ${elapsed}s`);
|
||||
|
||||
await client.close();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("❌ Seed failed:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,169 @@
|
||||
/**
|
||||
* seed-pk.ts — Push Prayers of King Kulaśekhara JSON data into MongoDB.
|
||||
*
|
||||
* Uses the native MongoDB driver (same pattern as seed-mog.ts) to avoid
|
||||
* Prisma replica-set write requirements.
|
||||
*
|
||||
* PK is a prose book with no volume structure and no intro segments.
|
||||
* We use volume=0 as the sentinel for "no volumes".
|
||||
*
|
||||
* Collections written:
|
||||
* - book_meta → one doc { book: "pk", data: <intro.json> }
|
||||
* - prose_chapters → 6 chapters (one per verse/prayer)
|
||||
*
|
||||
* Usage:
|
||||
* npx tsx scripts/seed-pk.ts
|
||||
*/
|
||||
|
||||
import dotenv from "dotenv";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { MongoClient } from "mongodb";
|
||||
|
||||
// Load env
|
||||
dotenv.config({ path: ".env.local" });
|
||||
dotenv.config({ path: ".env" });
|
||||
|
||||
const DATABASE_URL = process.env.DATABASE_URL;
|
||||
if (!DATABASE_URL) {
|
||||
console.error("❌ DATABASE_URL is not set in .env");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const BASE_DIR = path.resolve(__dirname, "..");
|
||||
const PK_DIR = path.join(BASE_DIR, "lib", "jsons", "books", "pk");
|
||||
|
||||
/** Volume sentinel for books without volumes. */
|
||||
const NO_VOLUME = 0;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function readJson<T>(filePath: string): T {
|
||||
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert documents one at a time with a small delay every N docs.
|
||||
* Uses insertOne to avoid BSON wire-protocol corruption.
|
||||
*/
|
||||
async function insertBatched(
|
||||
col: ReturnType<ReturnType<MongoClient["db"]>["collection"]>,
|
||||
docs: Record<string, unknown>[],
|
||||
) {
|
||||
for (let i = 0; i < docs.length; i++) {
|
||||
await col.insertOne(docs[i]);
|
||||
if ((i + 1) % 100 === 0) {
|
||||
await new Promise((r) => setTimeout(r, 100));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types for the JSON files
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type PkIntroJson = {
|
||||
book: string;
|
||||
title: string;
|
||||
author: string;
|
||||
year: number;
|
||||
description: string;
|
||||
totalChapters: number;
|
||||
chapters: { number: number; title: string }[];
|
||||
};
|
||||
|
||||
type PkChapterJson = {
|
||||
book: string;
|
||||
chapter: number;
|
||||
title: string;
|
||||
type: string;
|
||||
paragraphs: string[];
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Seed function
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function seedPk(db: ReturnType<MongoClient["db"]>) {
|
||||
const introPath = path.join(PK_DIR, "intro.json");
|
||||
if (!fs.existsSync(introPath)) {
|
||||
console.error("❌ PK intro.json not found at", introPath);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const intro = readJson<PkIntroJson>(introPath);
|
||||
|
||||
// ---- 1. Book meta ----
|
||||
const metaCol = db.collection("book_meta");
|
||||
await metaCol.deleteMany({ book: "pk" });
|
||||
await metaCol.insertOne({ book: "pk", data: intro });
|
||||
console.log(" 📖 PK book meta inserted");
|
||||
|
||||
// ---- 2. Prose chapters collection ----
|
||||
const proseCol = db.collection("prose_chapters");
|
||||
await proseCol.deleteMany({ book: "pk" });
|
||||
|
||||
const docs: Record<string, unknown>[] = [];
|
||||
|
||||
// Chapters (01.json through 06.json)
|
||||
const chaptersDir = path.join(PK_DIR, "chapters");
|
||||
if (fs.existsSync(chaptersDir)) {
|
||||
const chapterFiles = fs
|
||||
.readdirSync(chaptersDir)
|
||||
.filter((f) => f.endsWith(".json"))
|
||||
.sort();
|
||||
|
||||
for (const file of chapterFiles) {
|
||||
const ch = readJson<PkChapterJson>(path.join(chaptersDir, file));
|
||||
const chNum = ch.chapter ?? parseInt(file.replace(".json", ""), 10);
|
||||
docs.push({
|
||||
book: "pk",
|
||||
volume: NO_VOLUME,
|
||||
chapter: chNum,
|
||||
title: ch.title,
|
||||
type: "chapter",
|
||||
slug: String(chNum),
|
||||
order: 100 + chNum,
|
||||
paragraphs: ch.paragraphs,
|
||||
});
|
||||
}
|
||||
console.log(` 📖 Chapters: ${chapterFiles.length}`);
|
||||
}
|
||||
|
||||
// Insert all docs
|
||||
console.log(`\n Inserting ${docs.length} documents...`);
|
||||
await insertBatched(proseCol, docs);
|
||||
|
||||
const totalDocs = await proseCol.countDocuments({ book: "pk" });
|
||||
console.log(` ✅ PK: ${totalDocs} prose entries in MongoDB`);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function main() {
|
||||
console.log("🌱 Seeding Prayers of King Kulaśekhara (1958)...\n");
|
||||
|
||||
const client = new MongoClient(DATABASE_URL!);
|
||||
await client.connect();
|
||||
console.log("📡 Connected to MongoDB\n");
|
||||
|
||||
const db = client.db();
|
||||
|
||||
const start = Date.now();
|
||||
|
||||
await seedPk(db);
|
||||
|
||||
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
|
||||
console.log(`\n🎉 Done in ${elapsed}s`);
|
||||
|
||||
await client.close();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("❌ Seed failed:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,179 @@
|
||||
/**
|
||||
* seed-pre-1977-essays.ts — Push thirteen pre-1977 essays into MongoDB.
|
||||
*
|
||||
* Uses the native MongoDB driver (same pattern as seed-cc-essays.ts) to avoid
|
||||
* Prisma replica-set write requirements.
|
||||
*
|
||||
* Collections written:
|
||||
* - essays → 9 essay docs (upsert by slug)
|
||||
*
|
||||
* JSON sources:
|
||||
* - lib/jsons/essays/man-is-the-architect-of-his-own-fortune.json
|
||||
* - lib/jsons/essays/what-is-the-matter-with-the-world.json
|
||||
* - lib/jsons/essays/religion-is-godly-life.json
|
||||
* - lib/jsons/essays/truth-and-beauty.json
|
||||
* - lib/jsons/essays/misuse-of-public-fund-in-the-name-of-religion.json
|
||||
* - lib/jsons/essays/perfection-at-home.json
|
||||
* - lib/jsons/essays/how-should-one-cultivate-human-spirit.json
|
||||
* - lib/jsons/essays/viswa-dharma.json
|
||||
* - lib/jsons/essays/glorification-of-srila-bhaktisiddhanta-sarasvati-thakura.json
|
||||
* - lib/jsons/essays/who-is-crazy-btg66.json
|
||||
* - lib/jsons/essays/manifestation-of-dharma-in-human-being.json
|
||||
* - lib/jsons/essays/message-of-krishna-consciousness.json
|
||||
* - lib/jsons/essays/a-wrong-civilization.json
|
||||
*
|
||||
* Usage:
|
||||
* npx tsx scripts/seed-pre-1977-essays.ts
|
||||
*/
|
||||
|
||||
import dotenv from "dotenv";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { MongoClient } from "mongodb";
|
||||
|
||||
// Load env
|
||||
dotenv.config({ path: ".env.local" });
|
||||
dotenv.config({ path: ".env" });
|
||||
|
||||
const DATABASE_URL = process.env.DATABASE_URL;
|
||||
if (!DATABASE_URL) {
|
||||
console.error("❌ DATABASE_URL is not set in .env");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const BASE_DIR = path.resolve(__dirname, "..");
|
||||
const ESSAYS_DIR = path.join(BASE_DIR, "lib", "jsons", "essays");
|
||||
|
||||
const ESSAY_FILES = [
|
||||
"man-is-the-architect-of-his-own-fortune.json",
|
||||
"what-is-the-matter-with-the-world.json",
|
||||
"religion-is-godly-life.json",
|
||||
"truth-and-beauty.json",
|
||||
"misuse-of-public-fund-in-the-name-of-religion.json",
|
||||
"perfection-at-home.json",
|
||||
"how-should-one-cultivate-human-spirit.json",
|
||||
"viswa-dharma.json",
|
||||
"glorification-of-srila-bhaktisiddhanta-sarasvati-thakura.json",
|
||||
"who-is-crazy-btg66.json",
|
||||
"manifestation-of-dharma-in-human-being.json",
|
||||
"message-of-krishna-consciousness.json",
|
||||
"a-wrong-civilization.json",
|
||||
];
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type EssayJson = {
|
||||
slug: string;
|
||||
title: string;
|
||||
author: string;
|
||||
year: number | null;
|
||||
group: string;
|
||||
order: number;
|
||||
paragraphs: string[];
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function readJson<T>(filePath: string): T {
|
||||
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Seed function
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function seedPre1977Essays(db: ReturnType<MongoClient["db"]>) {
|
||||
const essaysCol = db.collection("essays");
|
||||
|
||||
let inserted = 0;
|
||||
let updated = 0;
|
||||
let unchanged = 0;
|
||||
|
||||
for (const filename of ESSAY_FILES) {
|
||||
const jsonPath = path.join(ESSAYS_DIR, filename);
|
||||
|
||||
if (!fs.existsSync(jsonPath)) {
|
||||
console.error(` ❌ Missing: ${jsonPath}`);
|
||||
console.error(" Ensure the JSON file exists in lib/jsons/essays/.");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const essay = readJson<EssayJson>(jsonPath);
|
||||
|
||||
const result = await essaysCol.updateOne(
|
||||
{ slug: essay.slug },
|
||||
{
|
||||
$set: {
|
||||
slug: essay.slug,
|
||||
title: essay.title,
|
||||
author: essay.author,
|
||||
year: essay.year,
|
||||
group: essay.group,
|
||||
order: essay.order,
|
||||
paragraphs: essay.paragraphs,
|
||||
},
|
||||
},
|
||||
{ upsert: true },
|
||||
);
|
||||
|
||||
if (result.upsertedCount > 0) {
|
||||
console.log(
|
||||
` 📝 Inserted "${essay.title}" (${essay.paragraphs.length} paragraphs)`,
|
||||
);
|
||||
inserted++;
|
||||
} else if (result.modifiedCount > 0) {
|
||||
console.log(
|
||||
` 📝 Updated "${essay.title}" (${essay.paragraphs.length} paragraphs)`,
|
||||
);
|
||||
updated++;
|
||||
} else {
|
||||
console.log(` 📝 "${essay.title}" already up to date`);
|
||||
unchanged++;
|
||||
}
|
||||
}
|
||||
|
||||
// Verify
|
||||
const totalDocs = await essaysCol.countDocuments({
|
||||
group: "pre-1977-essays",
|
||||
});
|
||||
console.log(
|
||||
`\n ✅ Verified: ${totalDocs} document(s) in essays collection with group "pre-1977-essays"`,
|
||||
);
|
||||
|
||||
// Summary
|
||||
console.log(
|
||||
`\n Summary: ${inserted} inserted, ${updated} updated, ${unchanged} unchanged`,
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function main() {
|
||||
console.log("🌱 Seeding Pre-1977 Essays (13 essays)...\n");
|
||||
|
||||
const client = new MongoClient(DATABASE_URL!);
|
||||
await client.connect();
|
||||
console.log("📡 Connected to MongoDB\n");
|
||||
|
||||
const db = client.db();
|
||||
|
||||
const start = Date.now();
|
||||
|
||||
await seedPre1977Essays(db);
|
||||
|
||||
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
|
||||
console.log(`\n🎉 Done in ${elapsed}s`);
|
||||
|
||||
await client.close();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("❌ Seed failed:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,139 @@
|
||||
/**
|
||||
* seed-va.ts — Push Vaiśiṣṭyāṣṭakam (VA) into MongoDB.
|
||||
*
|
||||
* Collections written:
|
||||
* - book_meta → 1 doc { book: "va", data: <intro.json> }
|
||||
* - prose_chapters → 8 chapter docs
|
||||
*
|
||||
* JSON sources:
|
||||
* - lib/jsons/books/va/intro.json
|
||||
* - lib/jsons/books/va/chapters/01.json through 08.json
|
||||
*
|
||||
* Usage:
|
||||
* npx tsx scripts/seed-va.ts
|
||||
*/
|
||||
|
||||
import dotenv from "dotenv";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { MongoClient } from "mongodb";
|
||||
|
||||
// Load env
|
||||
dotenv.config({ path: ".env.local" });
|
||||
dotenv.config({ path: ".env" });
|
||||
|
||||
const DATABASE_URL = process.env.DATABASE_URL;
|
||||
if (!DATABASE_URL) {
|
||||
console.error("❌ DATABASE_URL is not set in .env");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const BASE_DIR = path.resolve(__dirname, "..");
|
||||
const BOOK_DIR = path.join(BASE_DIR, "lib", "jsons", "books", "va");
|
||||
const BOOK = "va";
|
||||
const NO_VOLUME = 0;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type IntroJson = {
|
||||
book: string;
|
||||
title: string;
|
||||
author: string;
|
||||
year: number | null;
|
||||
description: string;
|
||||
totalChapters: number;
|
||||
chapters: { number: number; title: string }[];
|
||||
};
|
||||
|
||||
type ChapterJson = {
|
||||
book: string;
|
||||
chapter: number;
|
||||
title: string;
|
||||
type: string;
|
||||
paragraphs: string[];
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function readJson<T>(filePath: string): T {
|
||||
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function main() {
|
||||
console.log("🌱 Seeding Vaiśiṣṭyāṣṭakam (VA)...\n");
|
||||
|
||||
const client = new MongoClient(DATABASE_URL!);
|
||||
await client.connect();
|
||||
console.log("📡 Connected to MongoDB\n");
|
||||
|
||||
const db = client.db();
|
||||
const start = Date.now();
|
||||
|
||||
// ---- Read intro ----
|
||||
const introPath = path.join(BOOK_DIR, "intro.json");
|
||||
if (!fs.existsSync(introPath)) {
|
||||
console.error(`❌ Missing: ${introPath}`);
|
||||
process.exit(1);
|
||||
}
|
||||
const intro = readJson<IntroJson>(introPath);
|
||||
|
||||
// ---- Book meta ----
|
||||
const metaCol = db.collection("book_meta");
|
||||
await metaCol.deleteMany({ book: BOOK });
|
||||
await metaCol.insertOne({ book: BOOK, data: intro });
|
||||
console.log(`📖 Inserted book_meta for "${intro.title}" (${BOOK})`);
|
||||
|
||||
// ---- Chapters ----
|
||||
const proseCol = db.collection("prose_chapters");
|
||||
await proseCol.deleteMany({ book: BOOK });
|
||||
|
||||
const chaptersDir = path.join(BOOK_DIR, "chapters");
|
||||
const chapterFiles = fs
|
||||
.readdirSync(chaptersDir)
|
||||
.filter((f) => f.endsWith(".json"))
|
||||
.sort();
|
||||
|
||||
let inserted = 0;
|
||||
for (const file of chapterFiles) {
|
||||
const ch = readJson<ChapterJson>(path.join(chaptersDir, file));
|
||||
const chNum = ch.chapter;
|
||||
|
||||
await proseCol.insertOne({
|
||||
book: BOOK,
|
||||
volume: NO_VOLUME,
|
||||
chapter: chNum,
|
||||
title: ch.title,
|
||||
type: ch.type || "chapter",
|
||||
slug: String(chNum),
|
||||
order: 100 + chNum,
|
||||
paragraphs: ch.paragraphs,
|
||||
});
|
||||
|
||||
console.log(
|
||||
` 📄 Chapter ${chNum}: "${ch.title}" (${ch.paragraphs.length} paragraphs)`,
|
||||
);
|
||||
inserted++;
|
||||
}
|
||||
|
||||
// Verify
|
||||
const totalDocs = await proseCol.countDocuments({ book: BOOK });
|
||||
console.log(`\n✅ Verified: ${totalDocs} prose_chapters docs for ${BOOK}`);
|
||||
|
||||
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
|
||||
console.log(`\n🎉 Done in ${elapsed}s (${inserted} chapters inserted)`);
|
||||
|
||||
await client.close();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("❌ Seed failed:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user