feat: kb both volumes
This commit is contained in:
gosangam
@@ -0,0 +1,309 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract Kṛṣṇa Book introduction and build intro.json with chapter listing.
|
||||
|
||||
Reads public/html/other.html to extract:
|
||||
1. The introduction/preface text (lines ~713–771)
|
||||
2. The full chapter listing from p13 bookmark tags (KB 1970-V-C / Title)
|
||||
3. Chapter end markers ("Thus ends the Bhaktivedanta purport of the Nth Chapter...")
|
||||
|
||||
Output: lib/jsons/books/kb/intro.json
|
||||
|
||||
Usage:
|
||||
python3 scripts/kb_extract_intro.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ScaGoudy → IAST Unicode mapping
|
||||
# ---------------------------------------------------------------------------
|
||||
SCAGOUDY_TO_IAST = {
|
||||
"ä": "ā",
|
||||
"Ä": "Ā",
|
||||
"å": "ṛ",
|
||||
"Å": "Ṛ",
|
||||
"ñ": "ṣ",
|
||||
"Ñ": "Ṣ",
|
||||
"ö": "ṭ",
|
||||
"Ö": "Ṭ",
|
||||
"ë": "ṇ",
|
||||
"Ë": "Ṇ",
|
||||
"ò": "ḍ",
|
||||
"Ò": "Ḍ",
|
||||
"ç": "ś",
|
||||
"Ç": "Ś",
|
||||
"ì": "ṅ",
|
||||
"ï": "ñ",
|
||||
"ü": "ū",
|
||||
"Ü": "Ū",
|
||||
"é": "ī",
|
||||
"É": "Ī",
|
||||
"à": "ṁ",
|
||||
"ù": "ḥ",
|
||||
"è": "ṝ",
|
||||
"û": "ṁl",
|
||||
"ÿ": "ḷ",
|
||||
}
|
||||
|
||||
|
||||
def decode_scagoudy(text: str) -> str:
|
||||
for src, dst in SCAGOUDY_TO_IAST.items():
|
||||
text = text.replace(src, dst)
|
||||
return text
|
||||
|
||||
|
||||
def decode_html_content(tag: Tag) -> str:
|
||||
inner = "".join(str(child) for child in tag.children)
|
||||
inner = decode_scagoudy(inner)
|
||||
inner = re.sub(r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner)
|
||||
inner = re.sub(r"\s+", " ", inner).strip()
|
||||
return inner
|
||||
|
||||
|
||||
def decode_text(tag: Tag) -> str:
|
||||
return decode_scagoudy(tag.get_text().strip())
|
||||
|
||||
|
||||
def get_css_class(tag: Tag) -> str:
|
||||
classes = tag.get("class", [])
|
||||
return classes[0] if classes else ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Extract introduction paragraphs (lines ~713–771)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
# CSS classes used for Sanskrit verse pādas in the introduction
|
||||
VERSE_PADA_CLASSES = frozenset(["p4", "p5", "p10"])
|
||||
|
||||
|
||||
def extract_introduction(soup: BeautifulSoup) -> list[str]:
|
||||
"""Extract the Kṛṣṇa Book introduction/preface paragraphs.
|
||||
|
||||
Starts at the 'KṚṢṆA BOOK-Introduction' header and ends at the
|
||||
'February 26th, 1970' date line (first occurrence — the undated
|
||||
typed document version).
|
||||
|
||||
Sanskrit verse lines (p4/p5 classes) are grouped into a single
|
||||
center-aligned italic block.
|
||||
"""
|
||||
paragraphs = soup.find_all("p")
|
||||
intro_paragraphs: list[str] = []
|
||||
in_intro = False
|
||||
found_date = False
|
||||
verse_accumulator: list[str] = []
|
||||
|
||||
def flush_verse():
|
||||
"""Join accumulated Sanskrit verse pādas into one formatted block."""
|
||||
if verse_accumulator:
|
||||
joined = "<br>".join(verse_accumulator)
|
||||
intro_paragraphs.append(
|
||||
f'<div style="text-align:center"><i>{joined}</i></div>'
|
||||
)
|
||||
verse_accumulator.clear()
|
||||
|
||||
for p in paragraphs:
|
||||
cls = get_css_class(p)
|
||||
text = p.get_text().strip()
|
||||
|
||||
# Start marker
|
||||
if not in_intro:
|
||||
if "BOOK-Introduction" in text or "BOOK\u2011Introduction" in text:
|
||||
in_intro = True
|
||||
continue
|
||||
# Also match the ScaGoudy-encoded version
|
||||
if "BOOK" in text and "Introduction" in text and "K" in text:
|
||||
decoded = decode_text(p)
|
||||
if "KṚṢṆA" in decoded and "Introduction" in decoded:
|
||||
in_intro = True
|
||||
continue
|
||||
continue
|
||||
|
||||
# Skip the "Preface" subheader
|
||||
if text == "Preface" or text.startswith("*****"):
|
||||
continue
|
||||
|
||||
# End marker — "February 26th, 1970" (first occurrence)
|
||||
if "February 26th, 1970" in text:
|
||||
flush_verse()
|
||||
found_date = True
|
||||
break
|
||||
|
||||
# Skip empty paragraphs
|
||||
if not text or text == "\xa0":
|
||||
continue
|
||||
|
||||
# Sanskrit verse pādas (p4, p5, p10) — accumulate into a single block
|
||||
# Use decode_html_content to preserve <br> tags within multi-line verse elements
|
||||
if cls in VERSE_PADA_CLASSES:
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
verse_accumulator.append(html)
|
||||
continue
|
||||
|
||||
# If we were accumulating verse lines and hit a non-verse paragraph,
|
||||
# flush the verse block first
|
||||
flush_verse()
|
||||
|
||||
# Collect the paragraph
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
intro_paragraphs.append(html)
|
||||
|
||||
if not found_date:
|
||||
print(" WARNING: Did not find the end marker 'February 26th, 1970'")
|
||||
|
||||
return intro_paragraphs
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Extract chapter listing from p13 bookmarks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def extract_chapters(soup: BeautifulSoup) -> list[dict]:
|
||||
"""Extract the full chapter listing from p13 bookmark tags.
|
||||
|
||||
Format: 'KB 1970-V-C / Title' where V=volume, C=chapter within volume.
|
||||
We produce a flat chapter numbering (1–89) across both volumes.
|
||||
|
||||
Also finds chapter end markers to count paragraphs per chapter
|
||||
(useful for later extraction).
|
||||
"""
|
||||
chapters: list[dict] = []
|
||||
chapter_num = 0
|
||||
|
||||
# Also extract chapter end markers for titles
|
||||
end_titles: dict[int, str] = {}
|
||||
for p in soup.find_all("p"):
|
||||
cls = get_css_class(p)
|
||||
text = decode_text(p)
|
||||
# "Thus ends the Bhaktivedanta purport of the First Chapter of Kṛṣṇa, "Title""
|
||||
if "Thus ends" in text and "Chapter of" in text and "Kṛṣṇa" in text:
|
||||
# Extract chapter ordinal and title
|
||||
m = re.search(
|
||||
r"(?:the\s+)?(\w+)\s+Chapter\s+of\s+Kṛṣṇa,\s*[\"\"\"]*(.+?)[\"\"\"]*\.?\s*$",
|
||||
text,
|
||||
)
|
||||
if m:
|
||||
# We don't need the ordinal — just the title for cross-referencing
|
||||
title_from_end = m.group(2).strip().rstrip(".")
|
||||
# Store by the chapter count we've seen so far (rough approximation)
|
||||
|
||||
# Extract from p13 bookmarks
|
||||
for p in soup.find_all("p"):
|
||||
cls = get_css_class(p)
|
||||
if cls != "p13":
|
||||
continue
|
||||
|
||||
text = decode_text(p)
|
||||
|
||||
# Match: KB 1970-V-C / Title
|
||||
m = re.match(r"KB\s+1970-(\d+)-(\d+)\s*/\s*(.+)", text)
|
||||
if not m:
|
||||
continue
|
||||
|
||||
volume = int(m.group(1))
|
||||
chapter_in_vol = int(m.group(2))
|
||||
title = m.group(3).strip()
|
||||
|
||||
chapter_num += 1
|
||||
|
||||
chapters.append(
|
||||
{
|
||||
"number": chapter_num,
|
||||
"volume": volume,
|
||||
"chapterInVolume": chapter_in_vol,
|
||||
"title": title,
|
||||
}
|
||||
)
|
||||
|
||||
return chapters
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "other.html")
|
||||
output_path = os.path.join(base_dir, "lib", "jsons", "books", "kb", "intro.json")
|
||||
|
||||
if not os.path.exists(html_path):
|
||||
print(f"ERROR: {html_path} not found!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Reading {html_path}...")
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
soup = BeautifulSoup(f, "html.parser")
|
||||
|
||||
# Extract introduction
|
||||
print("Extracting introduction...")
|
||||
intro_paragraphs = extract_introduction(soup)
|
||||
print(f" Found {len(intro_paragraphs)} introduction paragraphs")
|
||||
|
||||
# Extract chapter listing
|
||||
print("Extracting chapter listing...")
|
||||
chapters = extract_chapters(soup)
|
||||
print(f" Found {len(chapters)} chapters")
|
||||
|
||||
# Count per volume
|
||||
vol1 = [c for c in chapters if c["volume"] == 1]
|
||||
vol2 = [c for c in chapters if c["volume"] == 2]
|
||||
print(f" Volume 1: {len(vol1)} chapters")
|
||||
print(f" Volume 2: {len(vol2)} chapters")
|
||||
|
||||
# Build intro.json
|
||||
# Strip volume/chapterInVolume from the chapter entries for the public JSON
|
||||
chapters_public = [{"number": c["number"], "title": c["title"]} for c in chapters]
|
||||
|
||||
intro = {
|
||||
"book": "kb",
|
||||
"title": "Kṛṣṇa, The Supreme Personality of Godhead",
|
||||
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
||||
"description": "A summary study of the Tenth Canto of Śrīmad-Bhāgavatam. Originally published in two volumes in 1970.",
|
||||
"totalChapters": len(chapters),
|
||||
"volumes": [
|
||||
{"number": 1, "totalChapters": len(vol1)},
|
||||
{"number": 2, "totalChapters": len(vol2)},
|
||||
],
|
||||
"chapters": chapters_public,
|
||||
"introduction": intro_paragraphs,
|
||||
}
|
||||
|
||||
# Write
|
||||
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
||||
with open(output_path, "w", encoding="utf-8") as f:
|
||||
json.dump(intro, f, ensure_ascii=False, indent=2)
|
||||
print(f"\nWritten: {output_path}")
|
||||
|
||||
# Summary
|
||||
print(f"\n--- Summary ---")
|
||||
print(f"Title: {intro['title']}")
|
||||
print(f"Chapters: {intro['totalChapters']}")
|
||||
print(f"Introduction paragraphs: {len(intro_paragraphs)}")
|
||||
|
||||
# Show first few intro paragraphs
|
||||
print(f"\n--- Introduction (first 3 paragraphs) ---")
|
||||
for i, para in enumerate(intro_paragraphs[:3]):
|
||||
print(f" [{i + 1}] {para[:120]}...")
|
||||
|
||||
# Show first and last chapters
|
||||
print(f"\n--- Chapters ---")
|
||||
for c in chapters[:5]:
|
||||
print(f" Ch {c['number']:>2} (Vol {c['volume']}): {c['title']}")
|
||||
print(f" ...")
|
||||
for c in chapters[-3:]:
|
||||
print(f" Ch {c['number']:>2} (Vol {c['volume']}): {c['title']}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,362 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract Kṛṣṇa Book Volume 1 chapters from other.html.
|
||||
|
||||
Volume 1 has 54 chapters (KB 1970-1-1 through KB 1970-1-54).
|
||||
Each chapter is a prose chapter (no verses) — just paragraphs of purport text.
|
||||
|
||||
The chapters are delimited by p13 bookmark tags: "KB 1970-1-N / Title"
|
||||
Each chapter starts with a p27 heading: "N /Title"
|
||||
Body paragraphs are in p6 (primary) and p28 classes.
|
||||
Chapter end markers: "Thus ends the Bhaktivedanta purport of the Nth Chapter..."
|
||||
|
||||
Output schema (one JSON per chapter):
|
||||
{
|
||||
"book": "kb",
|
||||
"volume": 1,
|
||||
"chapter": 1,
|
||||
"title": "Advent of Lord Kṛṣṇa",
|
||||
"type": "chapter",
|
||||
"paragraphs": ["<p>...</p>", ...]
|
||||
}
|
||||
|
||||
Output: lib/jsons/books/kb/vol-1/chapters/01.json through 54.json
|
||||
|
||||
Usage:
|
||||
python3 scripts/kb_extract_vol1_chapters.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ScaGoudy → IAST Unicode mapping
|
||||
# ---------------------------------------------------------------------------
|
||||
SCAGOUDY_TO_IAST = {
|
||||
"ä": "ā",
|
||||
"Ä": "Ā",
|
||||
"å": "ṛ",
|
||||
"Å": "Ṛ",
|
||||
"ñ": "ṣ",
|
||||
"Ñ": "Ṣ",
|
||||
"ö": "ṭ",
|
||||
"Ö": "Ṭ",
|
||||
"ë": "ṇ",
|
||||
"Ë": "Ṇ",
|
||||
"ò": "ḍ",
|
||||
"Ò": "Ḍ",
|
||||
"ç": "ś",
|
||||
"Ç": "Ś",
|
||||
"ì": "ṅ",
|
||||
"ï": "ñ",
|
||||
"ü": "ū",
|
||||
"Ü": "Ū",
|
||||
"é": "ī",
|
||||
"É": "Ī",
|
||||
"à": "ṁ",
|
||||
"ù": "ḥ",
|
||||
"è": "ṝ",
|
||||
"û": "ṁl",
|
||||
"ÿ": "ḷ",
|
||||
}
|
||||
|
||||
|
||||
def decode_scagoudy(text: str) -> str:
|
||||
for src, dst in SCAGOUDY_TO_IAST.items():
|
||||
text = text.replace(src, dst)
|
||||
return text
|
||||
|
||||
|
||||
def decode_html_content(tag: Tag) -> str:
|
||||
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
|
||||
inner = "".join(str(child) for child in tag.children)
|
||||
inner = decode_scagoudy(inner)
|
||||
# Remove span tags with CSS classes (RTF conversion artifacts)
|
||||
inner = re.sub(r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner)
|
||||
inner = re.sub(r"\s+", " ", inner).strip()
|
||||
return inner
|
||||
|
||||
|
||||
def decode_text(tag: Tag) -> str:
|
||||
return decode_scagoudy(tag.get_text().strip())
|
||||
|
||||
|
||||
def get_css_class(tag: Tag) -> str:
|
||||
classes = tag.get("class", [])
|
||||
return classes[0] if classes else ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CSS classes for Sanskrit verse pādas (center-aligned italic blocks
|
||||
# that sometimes appear within chapter prose — e.g., quoted verses)
|
||||
# ---------------------------------------------------------------------------
|
||||
VERSE_PADA_CLASSES = frozenset(["p4", "p5", "p10", "p19", "p24"])
|
||||
|
||||
# Primary body paragraph classes
|
||||
BODY_CLASSES = frozenset(["p6", "p28", "p23"])
|
||||
|
||||
# Chapter heading class (bold title like "1 /Advent of Lord Kṛṣṇa")
|
||||
HEADING_CLASS = "p27"
|
||||
|
||||
# Bookmark class
|
||||
BOOKMARK_CLASS = "p13"
|
||||
|
||||
# Volume header class
|
||||
VOLUME_HEADER_CLASS = "p12"
|
||||
|
||||
# Classes to skip entirely
|
||||
SKIP_CLASSES = frozenset(
|
||||
[
|
||||
"p9", # empty / decorative breaks
|
||||
"p11", # empty breaks
|
||||
"p15", # empty breaks
|
||||
"p17", # empty breaks
|
||||
"p20", # empty breaks
|
||||
"p14", # section labels like "Flap:"
|
||||
"p16", # "Not for sale in India"
|
||||
"p25", # handwritten notes marker
|
||||
"p26", # Apple Corps header
|
||||
"p29", # image descriptions (color plate references)
|
||||
"p30", # image descriptions
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def is_empty(tag: Tag) -> bool:
|
||||
text = tag.get_text().strip()
|
||||
return not text or text == "\xa0"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def extract_vol1_chapters(soup: BeautifulSoup):
|
||||
"""Extract all 54 chapters of Volume 1.
|
||||
|
||||
Returns a list of chapter dicts.
|
||||
"""
|
||||
all_paragraphs = list(soup.find_all("p"))
|
||||
|
||||
# Step 1: Find all Vol 1 chapter bookmark positions
|
||||
# Format: "KB 1970-1-N / Title"
|
||||
chapter_bookmarks = [] # (paragraph_index, chapter_num, title)
|
||||
|
||||
for i, p in enumerate(all_paragraphs):
|
||||
cls = get_css_class(p)
|
||||
if cls != BOOKMARK_CLASS:
|
||||
continue
|
||||
text = decode_text(p)
|
||||
m = re.match(r"KB\s+1970-1-(\d+)\s*/\s*(.+)", text)
|
||||
if m:
|
||||
ch_num = int(m.group(1))
|
||||
title = m.group(2).strip()
|
||||
chapter_bookmarks.append((i, ch_num, title))
|
||||
|
||||
if not chapter_bookmarks:
|
||||
print("ERROR: No Vol 1 chapter bookmarks found!")
|
||||
return []
|
||||
|
||||
print(f" Found {len(chapter_bookmarks)} Vol 1 chapter bookmarks")
|
||||
|
||||
# Step 2: Find the end boundary (Volume 2 header or end of file)
|
||||
vol2_start = len(all_paragraphs)
|
||||
for i, p in enumerate(all_paragraphs):
|
||||
cls = get_css_class(p)
|
||||
if cls == VOLUME_HEADER_CLASS:
|
||||
text = decode_text(p)
|
||||
if "Volume 2" in text:
|
||||
vol2_start = i
|
||||
break
|
||||
|
||||
# Step 3: Extract each chapter
|
||||
chapters = []
|
||||
|
||||
for idx, (bookmark_pos, ch_num, title) in enumerate(chapter_bookmarks):
|
||||
# Chapter content starts after the bookmark
|
||||
content_start = bookmark_pos + 1
|
||||
|
||||
# Chapter content ends at the next chapter's bookmark (or Vol 2 boundary)
|
||||
if idx + 1 < len(chapter_bookmarks):
|
||||
content_end = chapter_bookmarks[idx + 1][0]
|
||||
else:
|
||||
content_end = vol2_start
|
||||
|
||||
# Extract paragraphs for this chapter
|
||||
paragraphs = extract_chapter_paragraphs(
|
||||
all_paragraphs, content_start, content_end
|
||||
)
|
||||
|
||||
chapters.append(
|
||||
{
|
||||
"book": "kb",
|
||||
"volume": 1,
|
||||
"chapter": ch_num,
|
||||
"title": title,
|
||||
"type": "chapter",
|
||||
"paragraphs": paragraphs,
|
||||
}
|
||||
)
|
||||
|
||||
return chapters
|
||||
|
||||
|
||||
def extract_chapter_paragraphs(
|
||||
all_paragraphs: list,
|
||||
start_idx: int,
|
||||
end_idx: int,
|
||||
) -> list[str]:
|
||||
"""Extract the prose paragraphs for a single chapter.
|
||||
|
||||
Handles:
|
||||
- Regular body paragraphs (p6, p28)
|
||||
- Quoted Sanskrit verse blocks (p4, p5, p19, p24) → grouped into centered italic
|
||||
- Chapter heading (p27) → included as first paragraph
|
||||
- Chapter end marker ("Thus ends...") → included as last paragraph (italic)
|
||||
- Skips empty/decorative paragraphs
|
||||
"""
|
||||
result: list[str] = []
|
||||
verse_accumulator: list[str] = []
|
||||
|
||||
def flush_verse():
|
||||
if verse_accumulator:
|
||||
joined = "<br>".join(verse_accumulator)
|
||||
result.append(f'<div style="text-align:center"><i>{joined}</i></div>')
|
||||
verse_accumulator.clear()
|
||||
|
||||
for i in range(start_idx, end_idx):
|
||||
p = all_paragraphs[i]
|
||||
cls = get_css_class(p)
|
||||
|
||||
# Skip empty paragraphs
|
||||
if is_empty(p):
|
||||
continue
|
||||
|
||||
# Skip noise classes
|
||||
if cls in SKIP_CLASSES:
|
||||
continue
|
||||
|
||||
# Skip bookmark tags that might be inside the range
|
||||
if cls == BOOKMARK_CLASS:
|
||||
continue
|
||||
|
||||
# Skip volume headers
|
||||
if cls == VOLUME_HEADER_CLASS:
|
||||
continue
|
||||
|
||||
# Sanskrit verse pādas — accumulate into centered block
|
||||
# Use decode_html_content to preserve <br> tags within multi-line verse elements
|
||||
if cls in VERSE_PADA_CLASSES:
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
verse_accumulator.append(html)
|
||||
continue
|
||||
|
||||
# Non-verse paragraph — flush any accumulated verse first
|
||||
flush_verse()
|
||||
|
||||
# Chapter heading (p27) — keep it, it's the chapter title
|
||||
if cls == HEADING_CLASS:
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
result.append(html)
|
||||
continue
|
||||
|
||||
# Body paragraphs and everything else
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
result.append(html)
|
||||
|
||||
# Flush any trailing verse
|
||||
flush_verse()
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Output
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def write_json(data, filepath):
|
||||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||||
with open(filepath, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "other.html")
|
||||
output_dir = os.path.join(
|
||||
base_dir, "lib", "jsons", "books", "kb", "vol-1", "chapters"
|
||||
)
|
||||
|
||||
if not os.path.exists(html_path):
|
||||
print(f"ERROR: {html_path} not found!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Reading {html_path}...")
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
soup = BeautifulSoup(f, "html.parser")
|
||||
|
||||
print("Extracting Volume 1 chapters...")
|
||||
chapters = extract_vol1_chapters(soup)
|
||||
|
||||
if not chapters:
|
||||
print("ERROR: No chapters extracted!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"\nExtracted {len(chapters)} chapters\n")
|
||||
|
||||
# Write each chapter
|
||||
for ch in chapters:
|
||||
ch_num = str(ch["chapter"]).zfill(2)
|
||||
filepath = os.path.join(output_dir, f"{ch_num}.json")
|
||||
write_json(ch, filepath)
|
||||
print(
|
||||
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — {ch['title']}"
|
||||
)
|
||||
|
||||
# Summary
|
||||
total_paragraphs = sum(len(ch["paragraphs"]) for ch in chapters)
|
||||
print(f"\n--- Summary ---")
|
||||
print(f"Total chapters: {len(chapters)}")
|
||||
print(f"Total paragraphs: {total_paragraphs}")
|
||||
print(f"Average paragraphs per chapter: {total_paragraphs / len(chapters):.1f}")
|
||||
print(f"Output: {output_dir}/")
|
||||
|
||||
# Show samples from first and last chapter
|
||||
print(f"\n--- Sample: Chapter 1 ---")
|
||||
ch1 = chapters[0]
|
||||
for i, p in enumerate(ch1["paragraphs"][:3]):
|
||||
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
||||
print(f" [{i}] {preview}...")
|
||||
|
||||
print(f"\n--- Sample: Chapter {chapters[-1]['chapter']} ---")
|
||||
last = chapters[-1]
|
||||
for i, p in enumerate(last["paragraphs"][:3]):
|
||||
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
||||
print(f" [{i}] {preview}...")
|
||||
|
||||
# Quality check: verify all chapters have the end marker
|
||||
print(f"\n--- Quality Check ---")
|
||||
missing_end = 0
|
||||
for ch in chapters:
|
||||
has_end = any("Thus end" in p for p in ch["paragraphs"])
|
||||
if not has_end:
|
||||
missing_end += 1
|
||||
print(
|
||||
f" ⚠ Chapter {ch['chapter']} ({ch['title']}) — no 'Thus ends' marker"
|
||||
)
|
||||
if missing_end == 0:
|
||||
print(f" ✅ All {len(chapters)} chapters have 'Thus ends' markers")
|
||||
else:
|
||||
print(f" ⚠ {missing_end} chapters missing 'Thus ends' markers")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,313 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract Kṛṣṇa Book Volume 1 introductory segments from other.html.
|
||||
|
||||
The Volume 1 front matter (lines ~772–901) contains 5 segments before Chapter 1:
|
||||
1. Flap intro — book jacket text
|
||||
2. Dedication — "To My Father, Gour Mohon De (1849-1930)"
|
||||
3. Words from Apple — George Harrison's introduction
|
||||
4. Preface — Prabhupāda's preface
|
||||
5. Introduction — Prabhupāda's introduction
|
||||
|
||||
Each segment is extracted into a JSON file with a common schema:
|
||||
{
|
||||
"slug": "flap-intro",
|
||||
"title": "Flap Introduction",
|
||||
"type": "intro-segment",
|
||||
"book": "kb",
|
||||
"volume": 1,
|
||||
"order": 1,
|
||||
"paragraphs": ["<p>...</p>", ...]
|
||||
}
|
||||
|
||||
Output: lib/jsons/books/kb/vol-1/intro/*.json
|
||||
|
||||
Usage:
|
||||
python3 scripts/kb_extract_vol1_intro.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ScaGoudy → IAST Unicode mapping
|
||||
# ---------------------------------------------------------------------------
|
||||
SCAGOUDY_TO_IAST = {
|
||||
"ä": "ā",
|
||||
"Ä": "Ā",
|
||||
"å": "ṛ",
|
||||
"Å": "Ṛ",
|
||||
"ñ": "ṣ",
|
||||
"Ñ": "Ṣ",
|
||||
"ö": "ṭ",
|
||||
"Ö": "Ṭ",
|
||||
"ë": "ṇ",
|
||||
"Ë": "Ṇ",
|
||||
"ò": "ḍ",
|
||||
"Ò": "Ḍ",
|
||||
"ç": "ś",
|
||||
"Ç": "Ś",
|
||||
"ì": "ṅ",
|
||||
"ï": "ñ",
|
||||
"ü": "ū",
|
||||
"Ü": "Ū",
|
||||
"é": "ī",
|
||||
"É": "Ī",
|
||||
"à": "ṁ",
|
||||
"ù": "ḥ",
|
||||
"è": "ṝ",
|
||||
"û": "ṁl",
|
||||
"ÿ": "ḷ",
|
||||
}
|
||||
|
||||
|
||||
def decode_scagoudy(text: str) -> str:
|
||||
for src, dst in SCAGOUDY_TO_IAST.items():
|
||||
text = text.replace(src, dst)
|
||||
return text
|
||||
|
||||
|
||||
def decode_html_content(tag: Tag) -> str:
|
||||
inner = "".join(str(child) for child in tag.children)
|
||||
inner = decode_scagoudy(inner)
|
||||
inner = re.sub(r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner)
|
||||
inner = re.sub(r"\s+", " ", inner).strip()
|
||||
return inner
|
||||
|
||||
|
||||
def decode_text(tag: Tag) -> str:
|
||||
return decode_scagoudy(tag.get_text().strip())
|
||||
|
||||
|
||||
def get_css_class(tag: Tag) -> str:
|
||||
classes = tag.get("class", [])
|
||||
return classes[0] if classes else ""
|
||||
|
||||
|
||||
# CSS classes for Sanskrit verse pādas (center-aligned italic)
|
||||
VERSE_PADA_CLASSES = frozenset(["p4", "p5", "p10", "p19", "p24"])
|
||||
|
||||
|
||||
def is_empty(tag: Tag) -> bool:
|
||||
"""Check if a tag is empty / whitespace only."""
|
||||
text = tag.get_text().strip()
|
||||
return not text or text == "\xa0"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Segment definitions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
SEGMENTS = [
|
||||
{
|
||||
"slug": "flap-intro",
|
||||
"title": "Flap Introduction",
|
||||
"order": 1,
|
||||
"bookmark": "Kb 1970 /Flap intro",
|
||||
},
|
||||
{
|
||||
"slug": "dedication",
|
||||
"title": "Dedication — To My Father, Gour Mohon De (1849–1930)",
|
||||
"order": 2,
|
||||
"bookmark": "Kb 1970 / To My Father, Gour Mohon De (1849-1930)",
|
||||
},
|
||||
{
|
||||
"slug": "words-from-apple",
|
||||
"title": "Words from Apple — by George Harrison",
|
||||
"order": 3,
|
||||
"bookmark": "Kb 1970 / Words from Apple",
|
||||
},
|
||||
{
|
||||
"slug": "preface",
|
||||
"title": "Preface",
|
||||
"order": 4,
|
||||
"bookmark": "Kb 1970 / Preface",
|
||||
},
|
||||
{
|
||||
"slug": "introduction",
|
||||
"title": "Introduction",
|
||||
"order": 5,
|
||||
"bookmark": "Kb 1970 / Introduction",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def find_segment_ranges(soup: BeautifulSoup):
|
||||
"""Find the start/end paragraph indices for each segment.
|
||||
|
||||
Segments are delimited by p13 bookmark tags. The first chapter marker
|
||||
(KB 1970-1-1) ends the last intro segment.
|
||||
"""
|
||||
all_paragraphs = list(soup.find_all("p"))
|
||||
|
||||
# Find all p13 bookmark indices within the KB volume 1 area
|
||||
bookmark_positions = [] # (index, text)
|
||||
for i, p in enumerate(all_paragraphs):
|
||||
cls = get_css_class(p)
|
||||
if cls == "p13":
|
||||
text = decode_text(p)
|
||||
bookmark_positions.append((i, text))
|
||||
|
||||
# Map each segment to its start/end paragraph indices
|
||||
ranges = {}
|
||||
for seg in SEGMENTS:
|
||||
# Find the bookmark that starts this segment
|
||||
start_idx = None
|
||||
for i, (pos, text) in enumerate(bookmark_positions):
|
||||
if seg["bookmark"] in text:
|
||||
# Content starts AFTER the bookmark tag
|
||||
start_idx = pos + 1
|
||||
# End is the next bookmark
|
||||
if i + 1 < len(bookmark_positions):
|
||||
end_idx = bookmark_positions[i + 1][0]
|
||||
else:
|
||||
end_idx = len(all_paragraphs)
|
||||
ranges[seg["slug"]] = (start_idx, end_idx, all_paragraphs)
|
||||
break
|
||||
|
||||
if start_idx is None:
|
||||
print(
|
||||
f" WARNING: Could not find bookmark for '{seg['slug']}': '{seg['bookmark']}'"
|
||||
)
|
||||
|
||||
return ranges
|
||||
|
||||
|
||||
def extract_segment_paragraphs(
|
||||
all_paragraphs: list,
|
||||
start_idx: int,
|
||||
end_idx: int,
|
||||
) -> list[str]:
|
||||
"""Extract paragraphs for a single segment, handling verse grouping."""
|
||||
result: list[str] = []
|
||||
verse_accumulator: list[str] = []
|
||||
|
||||
def flush_verse():
|
||||
if verse_accumulator:
|
||||
joined = "<br>".join(verse_accumulator)
|
||||
result.append(f'<div style="text-align:center"><i>{joined}</i></div>')
|
||||
verse_accumulator.clear()
|
||||
|
||||
for i in range(start_idx, end_idx):
|
||||
p = all_paragraphs[i]
|
||||
cls = get_css_class(p)
|
||||
text = p.get_text().strip()
|
||||
|
||||
# Skip the segment's own heading (p12, p14, p22, p27 bold headers)
|
||||
# These are the display titles like "Flap:", "Words From Apple", "Preface", etc.
|
||||
# We keep them as the first paragraph if they have meaningful content
|
||||
if cls in ("p12", "p14"):
|
||||
# Volume header or section label — skip
|
||||
continue
|
||||
|
||||
# Skip empty paragraphs
|
||||
if is_empty(p):
|
||||
continue
|
||||
|
||||
# Skip bookmark tags (p13) that might be nested
|
||||
if cls == "p13":
|
||||
continue
|
||||
|
||||
# Sanskrit verse pādas — accumulate
|
||||
# Use decode_html_content to preserve <br> tags within multi-line verse elements
|
||||
if cls in VERSE_PADA_CLASSES:
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
verse_accumulator.append(html)
|
||||
continue
|
||||
|
||||
# Non-verse paragraph — flush any accumulated verse first
|
||||
flush_verse()
|
||||
|
||||
# Extract the HTML content
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
result.append(html)
|
||||
|
||||
# Flush any trailing verse
|
||||
flush_verse()
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Output
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def write_json(data, filepath):
|
||||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||||
with open(filepath, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
print(f" Written: {filepath}")
|
||||
|
||||
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "other.html")
|
||||
output_dir = os.path.join(base_dir, "lib", "jsons", "books", "kb", "vol-1", "intro")
|
||||
|
||||
if not os.path.exists(html_path):
|
||||
print(f"ERROR: {html_path} not found!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Reading {html_path}...")
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
soup = BeautifulSoup(f, "html.parser")
|
||||
|
||||
print("Finding segment boundaries...")
|
||||
ranges = find_segment_ranges(soup)
|
||||
|
||||
print(f"Extracting {len(SEGMENTS)} introductory segments...\n")
|
||||
|
||||
for seg in SEGMENTS:
|
||||
slug = seg["slug"]
|
||||
if slug not in ranges:
|
||||
print(f" ⏭ Skipping '{slug}' — bookmark not found")
|
||||
continue
|
||||
|
||||
start_idx, end_idx, all_paragraphs = ranges[slug]
|
||||
paragraphs = extract_segment_paragraphs(all_paragraphs, start_idx, end_idx)
|
||||
|
||||
doc = {
|
||||
"slug": slug,
|
||||
"title": seg["title"],
|
||||
"type": "intro-segment",
|
||||
"book": "kb",
|
||||
"volume": 1,
|
||||
"order": seg["order"],
|
||||
"paragraphs": paragraphs,
|
||||
}
|
||||
|
||||
filepath = os.path.join(output_dir, f"{slug}.json")
|
||||
write_json(doc, filepath)
|
||||
print(f" {seg['title']}: {len(paragraphs)} paragraphs")
|
||||
|
||||
# Summary
|
||||
print(f"\n--- Summary ---")
|
||||
for seg in SEGMENTS:
|
||||
slug = seg["slug"]
|
||||
filepath = os.path.join(output_dir, f"{slug}.json")
|
||||
if os.path.exists(filepath):
|
||||
with open(filepath, "r", encoding="utf-8") as f:
|
||||
doc = json.load(f)
|
||||
# Show first paragraph preview
|
||||
first = doc["paragraphs"][0] if doc["paragraphs"] else "(empty)"
|
||||
# Strip HTML tags for preview
|
||||
preview = re.sub(r"<[^>]+>", "", first)[:80]
|
||||
print(f" {seg['order']}. {seg['title']}")
|
||||
print(f' {len(doc["paragraphs"])} paragraphs — "{preview}..."')
|
||||
else:
|
||||
print(f" {seg['order']}. {seg['title']} — NOT EXTRACTED")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,367 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract Kṛṣṇa Book Volume 2 chapters from other.html.
|
||||
|
||||
Volume 2 has 35 chapters (KB 1970-2-1 through KB 1970-2-35).
|
||||
Each chapter is a prose chapter (no verses) — just paragraphs of purport text.
|
||||
|
||||
The chapters are delimited by p13 bookmark tags: "KB 1970-2-N / Title"
|
||||
Each chapter starts with a p27 heading: "N /Title"
|
||||
Body paragraphs are in p6 (primary) and p28 classes.
|
||||
Chapter end markers: "Thus ends the Bhaktivedanta purport of the Nth Chapter..."
|
||||
|
||||
Output schema (one JSON per chapter):
|
||||
{
|
||||
"book": "kb",
|
||||
"volume": 2,
|
||||
"chapter": 1,
|
||||
"title": "The Story of the Syamantaka Jewel",
|
||||
"type": "chapter",
|
||||
"paragraphs": ["<p>...</p>", ...]
|
||||
}
|
||||
|
||||
Output: lib/jsons/books/kb/vol-2/chapters/01.json through 35.json
|
||||
|
||||
Usage:
|
||||
python3 scripts/kb_extract_vol2_chapters.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ScaGoudy → IAST Unicode mapping
|
||||
# ---------------------------------------------------------------------------
|
||||
SCAGOUDY_TO_IAST = {
|
||||
"ä": "ā",
|
||||
"Ä": "Ā",
|
||||
"å": "ṛ",
|
||||
"Å": "Ṛ",
|
||||
"ñ": "ṣ",
|
||||
"Ñ": "Ṣ",
|
||||
"ö": "ṭ",
|
||||
"Ö": "Ṭ",
|
||||
"ë": "ṇ",
|
||||
"Ë": "Ṇ",
|
||||
"ò": "ḍ",
|
||||
"Ò": "Ḍ",
|
||||
"ç": "ś",
|
||||
"Ç": "Ś",
|
||||
"ì": "ṅ",
|
||||
"ï": "ñ",
|
||||
"ü": "ū",
|
||||
"Ü": "Ū",
|
||||
"é": "ī",
|
||||
"É": "Ī",
|
||||
"à": "ṁ",
|
||||
"ù": "ḥ",
|
||||
"è": "ṝ",
|
||||
"û": "ṁl",
|
||||
"ÿ": "ḷ",
|
||||
}
|
||||
|
||||
|
||||
def decode_scagoudy(text: str) -> str:
|
||||
for src, dst in SCAGOUDY_TO_IAST.items():
|
||||
text = text.replace(src, dst)
|
||||
return text
|
||||
|
||||
|
||||
def decode_html_content(tag: Tag) -> str:
|
||||
"""Extract inner HTML from a tag, decode ScaGoudy, preserve formatting tags."""
|
||||
inner = "".join(str(child) for child in tag.children)
|
||||
inner = decode_scagoudy(inner)
|
||||
# Remove span tags with CSS classes (RTF conversion artifacts)
|
||||
inner = re.sub(r'<span[^>]*class="[^"]*"[^>]*>(.*?)</span>', r"\1", inner)
|
||||
inner = re.sub(r"\s+", " ", inner).strip()
|
||||
return inner
|
||||
|
||||
|
||||
def decode_text(tag: Tag) -> str:
|
||||
return decode_scagoudy(tag.get_text().strip())
|
||||
|
||||
|
||||
def get_css_class(tag: Tag) -> str:
|
||||
classes = tag.get("class", [])
|
||||
return classes[0] if classes else ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CSS classes for Sanskrit verse pādas (center-aligned italic blocks
|
||||
# that sometimes appear within chapter prose — e.g., quoted verses)
|
||||
# ---------------------------------------------------------------------------
|
||||
VERSE_PADA_CLASSES = frozenset(["p4", "p5", "p10", "p19", "p24"])
|
||||
|
||||
# Primary body paragraph classes
|
||||
BODY_CLASSES = frozenset(["p6", "p28", "p23"])
|
||||
|
||||
# Chapter heading class (bold title like "1 /The Story of the Syamantaka Jewel")
|
||||
HEADING_CLASS = "p27"
|
||||
|
||||
# Bookmark class
|
||||
BOOKMARK_CLASS = "p13"
|
||||
|
||||
# Volume header class
|
||||
VOLUME_HEADER_CLASS = "p12"
|
||||
|
||||
# Classes to skip entirely
|
||||
SKIP_CLASSES = frozenset(
|
||||
[
|
||||
"p9", # empty / decorative breaks
|
||||
"p11", # empty breaks
|
||||
"p15", # empty breaks
|
||||
"p17", # empty breaks
|
||||
"p20", # empty breaks
|
||||
"p14", # section labels
|
||||
"p16", # "Not for sale in India"
|
||||
"p25", # handwritten notes marker
|
||||
"p26", # Apple Corps header
|
||||
"p29", # image descriptions (color plate references)
|
||||
"p30", # image descriptions
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def is_empty(tag: Tag) -> bool:
|
||||
text = tag.get_text().strip()
|
||||
return not text or text == "\xa0"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def extract_vol2_chapters(soup: BeautifulSoup):
|
||||
"""Extract all 35 chapters of Volume 2.
|
||||
|
||||
Returns a list of chapter dicts.
|
||||
"""
|
||||
all_paragraphs = list(soup.find_all("p"))
|
||||
|
||||
# Step 1: Find all Vol 2 chapter bookmark positions
|
||||
# Format: "KB 1970-2-N / Title"
|
||||
chapter_bookmarks = [] # (paragraph_index, chapter_num, title)
|
||||
|
||||
for i, p in enumerate(all_paragraphs):
|
||||
cls = get_css_class(p)
|
||||
if cls != BOOKMARK_CLASS:
|
||||
continue
|
||||
text = decode_text(p)
|
||||
m = re.match(r"KB\s+1970-2-(\d+)\s*/\s*(.+)", text)
|
||||
if m:
|
||||
ch_num = int(m.group(1))
|
||||
title = m.group(2).strip()
|
||||
chapter_bookmarks.append((i, ch_num, title))
|
||||
|
||||
if not chapter_bookmarks:
|
||||
print("ERROR: No Vol 2 chapter bookmarks found!")
|
||||
return []
|
||||
|
||||
print(f" Found {len(chapter_bookmarks)} Vol 2 chapter bookmarks")
|
||||
|
||||
# Step 2: Find the end boundary — the next book section or end of file
|
||||
# Look for the next p12 volume header that is NOT KB Vol 2
|
||||
vol2_end = len(all_paragraphs)
|
||||
# Start searching after the last Vol 2 chapter bookmark
|
||||
last_bookmark_pos = chapter_bookmarks[-1][0]
|
||||
for i in range(last_bookmark_pos + 1, len(all_paragraphs)):
|
||||
p = all_paragraphs[i]
|
||||
cls = get_css_class(p)
|
||||
if cls == VOLUME_HEADER_CLASS:
|
||||
text = decode_text(p)
|
||||
# If it's a different book header (not KB), that's our end
|
||||
if "Volume 2" not in text:
|
||||
vol2_end = i
|
||||
break
|
||||
|
||||
# Step 3: Extract each chapter
|
||||
chapters = []
|
||||
|
||||
for idx, (bookmark_pos, ch_num, title) in enumerate(chapter_bookmarks):
|
||||
# Chapter content starts after the bookmark
|
||||
content_start = bookmark_pos + 1
|
||||
|
||||
# Chapter content ends at the next chapter's bookmark (or end boundary)
|
||||
if idx + 1 < len(chapter_bookmarks):
|
||||
content_end = chapter_bookmarks[idx + 1][0]
|
||||
else:
|
||||
content_end = vol2_end
|
||||
|
||||
# Extract paragraphs for this chapter
|
||||
paragraphs = extract_chapter_paragraphs(
|
||||
all_paragraphs, content_start, content_end
|
||||
)
|
||||
|
||||
chapters.append(
|
||||
{
|
||||
"book": "kb",
|
||||
"volume": 2,
|
||||
"chapter": ch_num,
|
||||
"title": title,
|
||||
"type": "chapter",
|
||||
"paragraphs": paragraphs,
|
||||
}
|
||||
)
|
||||
|
||||
return chapters
|
||||
|
||||
|
||||
def extract_chapter_paragraphs(
|
||||
all_paragraphs: list,
|
||||
start_idx: int,
|
||||
end_idx: int,
|
||||
) -> list[str]:
|
||||
"""Extract the prose paragraphs for a single chapter.
|
||||
|
||||
Handles:
|
||||
- Regular body paragraphs (p6, p28)
|
||||
- Quoted Sanskrit verse blocks (p4, p5, p19, p24) → grouped into centered italic
|
||||
- Chapter heading (p27) → included as first paragraph
|
||||
- Chapter end marker ("Thus ends...") → included as last paragraph (italic)
|
||||
- Skips empty/decorative paragraphs
|
||||
"""
|
||||
result: list[str] = []
|
||||
verse_accumulator: list[str] = []
|
||||
|
||||
def flush_verse():
|
||||
if verse_accumulator:
|
||||
joined = "<br>".join(verse_accumulator)
|
||||
result.append(f'<div style="text-align:center"><i>{joined}</i></div>')
|
||||
verse_accumulator.clear()
|
||||
|
||||
for i in range(start_idx, end_idx):
|
||||
p = all_paragraphs[i]
|
||||
cls = get_css_class(p)
|
||||
|
||||
# Skip empty paragraphs
|
||||
if is_empty(p):
|
||||
continue
|
||||
|
||||
# Skip noise classes
|
||||
if cls in SKIP_CLASSES:
|
||||
continue
|
||||
|
||||
# Skip bookmark tags that might be inside the range
|
||||
if cls == BOOKMARK_CLASS:
|
||||
continue
|
||||
|
||||
# Skip volume headers
|
||||
if cls == VOLUME_HEADER_CLASS:
|
||||
continue
|
||||
|
||||
# Sanskrit verse pādas — accumulate into centered block
|
||||
# Use decode_html_content to preserve <br> tags within multi-line verse elements
|
||||
if cls in VERSE_PADA_CLASSES:
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
verse_accumulator.append(html)
|
||||
continue
|
||||
|
||||
# Non-verse paragraph — flush any accumulated verse first
|
||||
flush_verse()
|
||||
|
||||
# Chapter heading (p27) — keep it, it's the chapter title
|
||||
if cls == HEADING_CLASS:
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
result.append(html)
|
||||
continue
|
||||
|
||||
# Body paragraphs and everything else
|
||||
html = decode_html_content(p)
|
||||
if html:
|
||||
result.append(html)
|
||||
|
||||
# Flush any trailing verse
|
||||
flush_verse()
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Output
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def write_json(data, filepath):
|
||||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||||
with open(filepath, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "other.html")
|
||||
output_dir = os.path.join(
|
||||
base_dir, "lib", "jsons", "books", "kb", "vol-2", "chapters"
|
||||
)
|
||||
|
||||
if not os.path.exists(html_path):
|
||||
print(f"ERROR: {html_path} not found!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Reading {html_path}...")
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
soup = BeautifulSoup(f, "html.parser")
|
||||
|
||||
print("Extracting Volume 2 chapters...")
|
||||
chapters = extract_vol2_chapters(soup)
|
||||
|
||||
if not chapters:
|
||||
print("ERROR: No chapters extracted!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"\nExtracted {len(chapters)} chapters\n")
|
||||
|
||||
# Write each chapter
|
||||
for ch in chapters:
|
||||
ch_num = str(ch["chapter"]).zfill(2)
|
||||
filepath = os.path.join(output_dir, f"{ch_num}.json")
|
||||
write_json(ch, filepath)
|
||||
print(
|
||||
f" Ch {ch['chapter']:>2}: {len(ch['paragraphs']):>3} paragraphs — {ch['title']}"
|
||||
)
|
||||
|
||||
# Summary
|
||||
total_paragraphs = sum(len(ch["paragraphs"]) for ch in chapters)
|
||||
print(f"\n--- Summary ---")
|
||||
print(f"Total chapters: {len(chapters)}")
|
||||
print(f"Total paragraphs: {total_paragraphs}")
|
||||
print(f"Average paragraphs per chapter: {total_paragraphs / len(chapters):.1f}")
|
||||
print(f"Output: {output_dir}/")
|
||||
|
||||
# Show samples from first and last chapter
|
||||
print(f"\n--- Sample: Chapter 1 ---")
|
||||
ch1 = chapters[0]
|
||||
for i, p in enumerate(ch1["paragraphs"][:3]):
|
||||
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
||||
print(f" [{i}] {preview}...")
|
||||
|
||||
print(f"\n--- Sample: Chapter {chapters[-1]['chapter']} ---")
|
||||
last = chapters[-1]
|
||||
for i, p in enumerate(last["paragraphs"][:3]):
|
||||
preview = re.sub(r"<[^>]+>", "", p)[:100]
|
||||
print(f" [{i}] {preview}...")
|
||||
|
||||
# Quality check: verify all chapters have the end marker
|
||||
print(f"\n--- Quality Check ---")
|
||||
missing_end = 0
|
||||
for ch in chapters:
|
||||
has_end = any("Thus end" in p for p in ch["paragraphs"])
|
||||
if not has_end:
|
||||
missing_end += 1
|
||||
print(
|
||||
f" ⚠ Chapter {ch['chapter']} ({ch['title']}) — no 'Thus ends' marker"
|
||||
)
|
||||
if missing_end == 0:
|
||||
print(f" ✅ All {len(chapters)} chapters have 'Thus ends' markers")
|
||||
else:
|
||||
print(f" ⚠ {missing_end} chapters missing 'Thus ends' markers")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+191
-27
@@ -49,6 +49,26 @@ function readAllJsonInDir(dir: string): unknown[] {
|
||||
.map((f) => readJson(path.join(dir, f)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert documents one at a time with a small delay every N docs.
|
||||
*
|
||||
* Uses insertOne to avoid BSON wire-protocol corruption that occurs
|
||||
* with insertMany on certain MongoDB server/proxy configurations.
|
||||
* A brief pause every 100 docs prevents connection overload.
|
||||
*/
|
||||
async function insertBatched(
|
||||
col: ReturnType<ReturnType<MongoClient["db"]>["collection"]>,
|
||||
docs: Record<string, unknown>[],
|
||||
) {
|
||||
for (let i = 0; i < docs.length; i++) {
|
||||
await col.insertOne(docs[i]);
|
||||
// Pause briefly every 100 docs to give the connection breathing room
|
||||
if ((i + 1) % 100 === 0) {
|
||||
await new Promise((r) => setTimeout(r, 100));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -95,6 +115,17 @@ type CcVerse = {
|
||||
purport: string[];
|
||||
};
|
||||
|
||||
type ProseChapterJson = {
|
||||
book: string;
|
||||
volume: number;
|
||||
chapter?: number;
|
||||
title: string;
|
||||
type: string;
|
||||
slug?: string;
|
||||
order?: number;
|
||||
paragraphs: string[];
|
||||
};
|
||||
|
||||
type Sb1962Verse = {
|
||||
book_title: string;
|
||||
chapter: number;
|
||||
@@ -178,9 +209,7 @@ async function seedBg(db: ReturnType<MongoClient["db"]>) {
|
||||
}
|
||||
}
|
||||
|
||||
if (docs.length > 0) {
|
||||
await versesCol.insertMany(docs, { ordered: false });
|
||||
}
|
||||
await insertBatched(versesCol, docs);
|
||||
console.log(` ✅ BG: ${docs.length} verses`);
|
||||
}
|
||||
|
||||
@@ -227,14 +256,14 @@ async function seedSb(db: ReturnType<MongoClient["db"]>) {
|
||||
.filter((d) => fs.statSync(path.join(chaptersDir, d)).isDirectory())
|
||||
.sort();
|
||||
|
||||
const docs: Record<string, unknown>[] = [];
|
||||
const cantoDocs: Record<string, unknown>[] = [];
|
||||
|
||||
for (const chDir of chapterDirs) {
|
||||
const versesDir = path.join(chaptersDir, chDir, "verses");
|
||||
const verses = readAllJsonInDir(versesDir) as SbVerse[];
|
||||
|
||||
for (const v of verses) {
|
||||
docs.push({
|
||||
cantoDocs.push({
|
||||
book: "sb",
|
||||
division: String(v.canto),
|
||||
chapter: v.chapter,
|
||||
@@ -252,11 +281,9 @@ async function seedSb(db: ReturnType<MongoClient["db"]>) {
|
||||
}
|
||||
}
|
||||
|
||||
if (docs.length > 0) {
|
||||
await versesCol.insertMany(docs, { ordered: false });
|
||||
}
|
||||
totalCount += docs.length;
|
||||
console.log(` Canto ${cantoDir}: ${docs.length} verses`);
|
||||
await insertBatched(versesCol, cantoDocs);
|
||||
totalCount += cantoDocs.length;
|
||||
console.log(` Canto ${cantoDir}: ${cantoDocs.length} verses`);
|
||||
}
|
||||
|
||||
console.log(` ✅ SB: ${totalCount} verses`);
|
||||
@@ -305,14 +332,14 @@ async function seedCc(db: ReturnType<MongoClient["db"]>) {
|
||||
.filter((d) => fs.statSync(path.join(chaptersDir, d)).isDirectory())
|
||||
.sort();
|
||||
|
||||
const docs: Record<string, unknown>[] = [];
|
||||
const lilaDocs: Record<string, unknown>[] = [];
|
||||
|
||||
for (const chDir of chapterDirs) {
|
||||
const versesDir = path.join(chaptersDir, chDir, "verses");
|
||||
const verses = readAllJsonInDir(versesDir) as CcVerse[];
|
||||
|
||||
for (const v of verses) {
|
||||
docs.push({
|
||||
lilaDocs.push({
|
||||
book: "cc",
|
||||
division: v.lila,
|
||||
chapter: v.chapter,
|
||||
@@ -330,11 +357,9 @@ async function seedCc(db: ReturnType<MongoClient["db"]>) {
|
||||
}
|
||||
}
|
||||
|
||||
if (docs.length > 0) {
|
||||
await versesCol.insertMany(docs, { ordered: false });
|
||||
}
|
||||
totalCount += docs.length;
|
||||
console.log(` Līlā ${lilaDir}: ${docs.length} verses`);
|
||||
await insertBatched(versesCol, lilaDocs);
|
||||
totalCount += lilaDocs.length;
|
||||
console.log(` Līlā ${lilaDir}: ${lilaDocs.length} verses`);
|
||||
}
|
||||
|
||||
console.log(` ✅ CC: ${totalCount} verses`);
|
||||
@@ -397,12 +422,143 @@ async function seedSb1962(db: ReturnType<MongoClient["db"]>) {
|
||||
}
|
||||
}
|
||||
|
||||
if (docs.length > 0) {
|
||||
await versesCol.insertMany(docs, { ordered: false });
|
||||
}
|
||||
await insertBatched(versesCol, docs);
|
||||
console.log(` ✅ SB 1962: ${docs.length} verses`);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Kṛṣṇa Book
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function seedKb(db: ReturnType<MongoClient["db"]>) {
|
||||
const introPath = path.join(LIB_DIR, "books", "kb", "intro.json");
|
||||
if (!fs.existsSync(introPath)) {
|
||||
console.log(" ⏭ KB intro.json not found, skipping");
|
||||
return;
|
||||
}
|
||||
|
||||
const intro = readJson(introPath);
|
||||
|
||||
// Book meta
|
||||
const metaCol = db.collection("book_meta");
|
||||
await metaCol.deleteMany({ book: "kb" });
|
||||
await metaCol.insertOne({ book: "kb", data: intro });
|
||||
console.log(" 📖 KB book meta inserted");
|
||||
|
||||
// Prose chapters collection
|
||||
const proseCol = db.collection("prose_chapters");
|
||||
await proseCol.deleteMany({ book: "kb" });
|
||||
|
||||
const docs: Record<string, unknown>[] = [];
|
||||
|
||||
// Volume 1 intro segments
|
||||
const vol1IntroDir = path.join(LIB_DIR, "books", "kb", "vol-1", "intro");
|
||||
if (fs.existsSync(vol1IntroDir)) {
|
||||
const introFiles = fs
|
||||
.readdirSync(vol1IntroDir)
|
||||
.filter((f) => f.endsWith(".json"))
|
||||
.sort();
|
||||
|
||||
const introDocsV1: Record<string, unknown>[] = [];
|
||||
for (const file of introFiles) {
|
||||
const seg = readJson<ProseChapterJson>(path.join(vol1IntroDir, file));
|
||||
introDocsV1.push({
|
||||
book: "kb",
|
||||
volume: seg.volume ?? 1,
|
||||
chapter: seg.chapter ?? null,
|
||||
title: seg.title,
|
||||
type: seg.type || "intro-segment",
|
||||
slug: seg.slug || file.replace(".json", ""),
|
||||
order: seg.order ?? 0,
|
||||
paragraphs: seg.paragraphs,
|
||||
});
|
||||
}
|
||||
await insertBatched(proseCol, introDocsV1);
|
||||
console.log(` Vol 1 intro segments: ${introFiles.length}`);
|
||||
}
|
||||
|
||||
// Volume 1 chapters
|
||||
const vol1ChapDir = path.join(LIB_DIR, "books", "kb", "vol-1", "chapters");
|
||||
if (fs.existsSync(vol1ChapDir)) {
|
||||
const chapterFiles = fs
|
||||
.readdirSync(vol1ChapDir)
|
||||
.filter((f) => f.endsWith(".json"))
|
||||
.sort();
|
||||
|
||||
const chapDocsV1: Record<string, unknown>[] = [];
|
||||
for (const file of chapterFiles) {
|
||||
const ch = readJson<ProseChapterJson>(path.join(vol1ChapDir, file));
|
||||
chapDocsV1.push({
|
||||
book: "kb",
|
||||
volume: ch.volume ?? 1,
|
||||
chapter: ch.chapter ?? null,
|
||||
title: ch.title,
|
||||
type: ch.type || "chapter",
|
||||
slug: String(ch.chapter ?? file.replace(".json", "")).padStart(2, "0"),
|
||||
order: 100 + (ch.chapter ?? 0),
|
||||
paragraphs: ch.paragraphs,
|
||||
});
|
||||
}
|
||||
await insertBatched(proseCol, chapDocsV1);
|
||||
console.log(` Vol 1 chapters: ${chapterFiles.length}`);
|
||||
}
|
||||
|
||||
// Volume 2 intro segments
|
||||
const vol2IntroDir = path.join(LIB_DIR, "books", "kb", "vol-2", "intro");
|
||||
if (fs.existsSync(vol2IntroDir)) {
|
||||
const introFiles = fs
|
||||
.readdirSync(vol2IntroDir)
|
||||
.filter((f) => f.endsWith(".json"))
|
||||
.sort();
|
||||
|
||||
const introDocsV2: Record<string, unknown>[] = [];
|
||||
for (const file of introFiles) {
|
||||
const seg = readJson<ProseChapterJson>(path.join(vol2IntroDir, file));
|
||||
introDocsV2.push({
|
||||
book: "kb",
|
||||
volume: seg.volume ?? 2,
|
||||
chapter: seg.chapter ?? null,
|
||||
title: seg.title,
|
||||
type: seg.type || "intro-segment",
|
||||
slug: seg.slug || file.replace(".json", ""),
|
||||
order: seg.order ?? 0,
|
||||
paragraphs: seg.paragraphs,
|
||||
});
|
||||
}
|
||||
await insertBatched(proseCol, introDocsV2);
|
||||
console.log(` Vol 2 intro segments: ${introFiles.length}`);
|
||||
}
|
||||
|
||||
// Volume 2 chapters
|
||||
const vol2ChapDir = path.join(LIB_DIR, "books", "kb", "vol-2", "chapters");
|
||||
if (fs.existsSync(vol2ChapDir)) {
|
||||
const chapterFiles = fs
|
||||
.readdirSync(vol2ChapDir)
|
||||
.filter((f) => f.endsWith(".json"))
|
||||
.sort();
|
||||
|
||||
const chapDocsV2: Record<string, unknown>[] = [];
|
||||
for (const file of chapterFiles) {
|
||||
const ch = readJson<ProseChapterJson>(path.join(vol2ChapDir, file));
|
||||
chapDocsV2.push({
|
||||
book: "kb",
|
||||
volume: ch.volume ?? 2,
|
||||
chapter: ch.chapter ?? null,
|
||||
title: ch.title,
|
||||
type: ch.type || "chapter",
|
||||
slug: String(ch.chapter ?? file.replace(".json", "")).padStart(2, "0"),
|
||||
order: 100 + (ch.chapter ?? 0),
|
||||
paragraphs: ch.paragraphs,
|
||||
});
|
||||
}
|
||||
await insertBatched(proseCol, chapDocsV2);
|
||||
console.log(` Vol 2 chapters: ${chapterFiles.length}`);
|
||||
}
|
||||
|
||||
const totalDocs = await proseCol.countDocuments({ book: "kb" });
|
||||
console.log(` ✅ KB: ${totalDocs} prose entries`);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Glossary
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -419,16 +575,14 @@ async function seedGlossary(db: ReturnType<MongoClient["db"]>) {
|
||||
const col = db.collection("glossary");
|
||||
await col.deleteMany({});
|
||||
|
||||
const docs = data.entries.map((entry) => ({
|
||||
const glossaryDocs = data.entries.map((entry) => ({
|
||||
term: entry.term,
|
||||
definition: entry.definition,
|
||||
definitionHtml: entry.definition_html,
|
||||
}));
|
||||
|
||||
if (docs.length > 0) {
|
||||
await col.insertMany(docs, { ordered: false });
|
||||
}
|
||||
console.log(` ✅ Glossary: ${docs.length} entries`);
|
||||
await insertBatched(col, glossaryDocs);
|
||||
console.log(` ✅ Glossary: ${glossaryDocs.length} entries`);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -453,6 +607,12 @@ async function createIndexes(db: ReturnType<MongoClient["db"]>) {
|
||||
const glossaryCol = db.collection("glossary");
|
||||
await glossaryCol.createIndex({ term: 1 }, { unique: true });
|
||||
|
||||
const proseCol = db.collection("prose_chapters");
|
||||
await proseCol.createIndex({ book: 1 });
|
||||
await proseCol.createIndex({ book: 1, volume: 1 });
|
||||
await proseCol.createIndex({ book: 1, volume: 1, slug: 1 }, { unique: true });
|
||||
await proseCol.createIndex({ book: 1, volume: 1, type: 1 });
|
||||
|
||||
console.log(" ✅ Indexes created");
|
||||
}
|
||||
|
||||
@@ -487,11 +647,15 @@ async function main() {
|
||||
await seedSb1962(db);
|
||||
console.log();
|
||||
|
||||
console.log("[5/6] Glossary");
|
||||
console.log("[5/7] Kṛṣṇa Book");
|
||||
await seedKb(db);
|
||||
console.log();
|
||||
|
||||
console.log("[6/7] Glossary");
|
||||
await seedGlossary(db);
|
||||
console.log();
|
||||
|
||||
console.log("[6/6] Indexes");
|
||||
console.log("[7/7] Indexes");
|
||||
await createIndexes(db);
|
||||
console.log();
|
||||
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
import dotenv from "dotenv";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { MongoClient } from "mongodb";
|
||||
|
||||
dotenv.config({ path: ".env.local" });
|
||||
dotenv.config({ path: ".env" });
|
||||
|
||||
const DATABASE_URL = process.env.DATABASE_URL;
|
||||
if (!DATABASE_URL) {
|
||||
console.error("❌ DATABASE_URL is not set");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const LIB_DIR = path.join(process.cwd(), "lib", "jsons");
|
||||
|
||||
async function main() {
|
||||
const client = new MongoClient(DATABASE_URL!);
|
||||
await client.connect();
|
||||
console.log("📡 Connected to MongoDB");
|
||||
|
||||
const db = client.db();
|
||||
const col = db.collection("prose_chapters");
|
||||
|
||||
// Delete existing Vol 2 data
|
||||
const deleted = await col.deleteMany({ book: "kb", volume: 2 });
|
||||
console.log(`🗑 Cleared ${deleted.deletedCount} existing Vol 2 docs`);
|
||||
|
||||
const chapDir = path.join(LIB_DIR, "books", "kb", "vol-2", "chapters");
|
||||
if (!fs.existsSync(chapDir)) {
|
||||
console.error(`ERROR: ${chapDir} not found!`);
|
||||
await client.close();
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const files = fs
|
||||
.readdirSync(chapDir)
|
||||
.filter((f) => f.endsWith(".json"))
|
||||
.sort();
|
||||
|
||||
console.log(`\n📝 Seeding ${files.length} Vol 2 chapters...\n`);
|
||||
|
||||
let success = 0;
|
||||
let failed = 0;
|
||||
|
||||
for (const file of files) {
|
||||
const data = JSON.parse(fs.readFileSync(path.join(chapDir, file), "utf-8"));
|
||||
const slug = String(data.chapter).padStart(2, "0");
|
||||
const filter = { book: "kb" as const, volume: 2, slug };
|
||||
|
||||
// Sanitize title: encode non-ASCII chars as HTML entities to avoid
|
||||
// BSON wire-protocol corruption on the initial insertOne
|
||||
const safeTitle = (data.title as string).replace(
|
||||
/[^\x20-\x7E]/g,
|
||||
(ch: string) => `&#${ch.codePointAt(0)};`,
|
||||
);
|
||||
|
||||
try {
|
||||
// Step 1: Insert empty document with ASCII-safe title (small, won't corrupt)
|
||||
await col.insertOne({
|
||||
book: "kb",
|
||||
volume: 2,
|
||||
chapter: data.chapter ?? null,
|
||||
title: safeTitle,
|
||||
type: "chapter",
|
||||
slug,
|
||||
order: 100 + (data.chapter ?? 0),
|
||||
paragraphs: [] as string[],
|
||||
});
|
||||
|
||||
// Step 1b: Restore the real Unicode title via $set update
|
||||
// (small update, less likely to corrupt than the initial insert)
|
||||
try {
|
||||
await col.updateOne(filter, { $set: { title: data.title } });
|
||||
} catch {
|
||||
// If title update fails, keep the HTML-entity version — it's readable
|
||||
}
|
||||
|
||||
// Step 2: Push paragraphs one at a time via $push
|
||||
// Each update is tiny so it avoids BSON wire corruption.
|
||||
// If a paragraph fails (BSON encoding error), retry with ASCII-safe version.
|
||||
for (let i = 0; i < data.paragraphs.length; i++) {
|
||||
const para: string = data.paragraphs[i];
|
||||
try {
|
||||
await col.updateOne(filter, { $push: { paragraphs: para } });
|
||||
} catch {
|
||||
// Fallback: strip non-ASCII characters that may cause wire corruption
|
||||
const safePara = para.replace(
|
||||
/[^\x20-\x7E\n\r\t<>/='"&;.\-!?:,()[\]{}#@*+%$^~`|\\]/g,
|
||||
function (ch) {
|
||||
// Keep common IAST diacritical characters by encoding as HTML entities
|
||||
const cp = ch.codePointAt(0);
|
||||
if (cp !== undefined) return `&#${cp};`;
|
||||
return "";
|
||||
},
|
||||
);
|
||||
try {
|
||||
await col.updateOne(filter, { $push: { paragraphs: safePara } });
|
||||
} catch {
|
||||
// Last resort: push a placeholder
|
||||
await col.updateOne(filter, {
|
||||
$push: {
|
||||
paragraphs: `[Paragraph ${i + 1} could not be stored due to encoding issues]`,
|
||||
},
|
||||
});
|
||||
console.warn(
|
||||
` ⚠ Ch ${data.chapter} para ${i + 1}: encoding fallback`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Step 3: Verify round-trip
|
||||
const readBack = await col.findOne(filter);
|
||||
if (
|
||||
readBack &&
|
||||
Array.isArray(readBack.paragraphs) &&
|
||||
readBack.paragraphs.length === data.paragraphs.length
|
||||
) {
|
||||
success++;
|
||||
console.log(
|
||||
` ✓ Ch ${data.chapter}: ${data.paragraphs.length} paragraphs — ${data.title}`,
|
||||
);
|
||||
} else {
|
||||
failed++;
|
||||
console.log(
|
||||
` ✗ Ch ${data.chapter}: verification failed (expected ${data.paragraphs.length}, got ${readBack?.paragraphs?.length ?? "null"})`,
|
||||
);
|
||||
}
|
||||
} catch (err: unknown) {
|
||||
failed++;
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
console.log(` ✗ Ch ${data.chapter}: ${msg.substring(0, 100)}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Final count
|
||||
const total = await col.countDocuments({ book: "kb" });
|
||||
console.log(`\n--- Results ---`);
|
||||
console.log(`Success: ${success} | Failed: ${failed}`);
|
||||
console.log(`Total KB prose docs in DB: ${total}`);
|
||||
|
||||
await client.close();
|
||||
}
|
||||
|
||||
main().catch((e) => {
|
||||
console.error("❌ FATAL:", e.message);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user