feat: add journals
This commit is contained in:
gosangam
@@ -0,0 +1,688 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract The Jaladuta Diary (1965) from others-before-1977.html.
|
||||
|
||||
Each diary day entry becomes its own chapter file.
|
||||
Chapters are grouped by month in the intro.json metadata.
|
||||
|
||||
Structure:
|
||||
- Preface and Introduction are standalone chapters (slugs "preface", "introduction")
|
||||
- Day entries use slugs like "apr-3", "may-2", "aug-13", "sep-1", etc.
|
||||
- The "Memoranda" entry at the end uses slug "memoranda"
|
||||
- Months are parent groups in intro.json (non-clickable headings)
|
||||
|
||||
Output:
|
||||
lib/jsons/journals/jaladuta-diary/intro.json
|
||||
lib/jsons/journals/jaladuta-diary/chapters/preface.json
|
||||
lib/jsons/journals/jaladuta-diary/chapters/introduction.json
|
||||
lib/jsons/journals/jaladuta-diary/chapters/apr-3.json
|
||||
lib/jsons/journals/jaladuta-diary/chapters/apr-5.json
|
||||
...
|
||||
lib/jsons/journals/jaladuta-diary/chapters/nov-28.json
|
||||
lib/jsons/journals/jaladuta-diary/chapters/memoranda.json
|
||||
|
||||
Usage:
|
||||
python3 scripts/jaladuta_diary_extract.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ScaGoudy → IAST Unicode mapping
|
||||
# ---------------------------------------------------------------------------
|
||||
SCAGOUDY_TO_IAST = {
|
||||
"ä": "ā",
|
||||
"Ä": "Ā",
|
||||
"å": "ṛ",
|
||||
"Å": "Ṛ",
|
||||
"ñ": "ṣ",
|
||||
"Ñ": "Ṣ",
|
||||
"ö": "ṭ",
|
||||
"Ö": "Ṭ",
|
||||
"ë": "ṇ",
|
||||
"Ë": "Ṇ",
|
||||
"ò": "ḍ",
|
||||
"Ò": "Ḍ",
|
||||
"ç": "ś",
|
||||
"Ç": "Ś",
|
||||
"ì": "ṅ",
|
||||
"Ì": "Ṅ",
|
||||
"ï": "ñ",
|
||||
"Ï": "Ñ",
|
||||
"ü": "ū",
|
||||
"Ü": "Ū",
|
||||
"é": "ī",
|
||||
"É": "Ī",
|
||||
"à": "ṁ",
|
||||
"ù": "ḥ",
|
||||
"è": "ṝ",
|
||||
"ÿ": "ḷ",
|
||||
}
|
||||
|
||||
|
||||
def decode_scagoudy(text: str) -> str:
|
||||
"""Replace ScaGoudy-encoded characters with proper IAST Unicode."""
|
||||
for src, dst in SCAGOUDY_TO_IAST.items():
|
||||
text = text.replace(src, dst)
|
||||
return text
|
||||
|
||||
|
||||
def get_css_class(tag: Tag) -> str:
|
||||
classes = tag.get("class", [])
|
||||
return classes[0] if classes else ""
|
||||
|
||||
|
||||
def inner_html(tag: Tag) -> str:
|
||||
"""Get inner HTML of a tag, decode ScaGoudy, clean up spans."""
|
||||
html = "".join(str(child) for child in tag.children)
|
||||
html = decode_scagoudy(html)
|
||||
# <span class="s3">text</span> → <i>text</i>
|
||||
html = re.sub(
|
||||
r'<span[^>]*class="s3"[^>]*>(.*?)</span>', r"<i>\1</i>", html, flags=re.DOTALL
|
||||
)
|
||||
# <span class="s5">...</span> → keep inner text
|
||||
html = re.sub(
|
||||
r'<span[^>]*class="s5"[^>]*>(.*?)</span>', r"\1", html, flags=re.DOTALL
|
||||
)
|
||||
# <span class="s13">...</span> → keep inner text
|
||||
html = re.sub(
|
||||
r'<span[^>]*class="s1[0-9]+"[^>]*>(.*?)</span>', r"\1", html, flags=re.DOTALL
|
||||
)
|
||||
# <span class="s23">...</span> → keep inner text
|
||||
html = re.sub(
|
||||
r'<span[^>]*class="s2[0-9]+"[^>]*>(.*?)</span>', r"\1", html, flags=re.DOTALL
|
||||
)
|
||||
# Remove any remaining spans but keep content
|
||||
html = re.sub(r"<span[^>]*>(.*?)</span>", r"\1", html, flags=re.DOTALL)
|
||||
# Clean up Apple-tab-span leftovers
|
||||
html = html.replace('<span class="Apple-tab-span">\t</span>', "\t")
|
||||
# Collapse multiple spaces (but not around <br>)
|
||||
html = re.sub(r"[ \t]+", " ", html)
|
||||
html = html.strip()
|
||||
# Remove leading <br> tags
|
||||
html = re.sub(r"^(<br\s*/?>[\s]*)+", "", html)
|
||||
return html
|
||||
|
||||
|
||||
def write_json(data: dict, path: str):
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
f.write("\n")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Month abbreviation helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
MONTH_ABBR = {
|
||||
"april": "apr",
|
||||
"may": "may",
|
||||
"june": "jun",
|
||||
"july": "jul",
|
||||
"august": "aug",
|
||||
"september": "sep",
|
||||
"october": "oct",
|
||||
"november": "nov",
|
||||
}
|
||||
|
||||
MONTH_ORDER = ["apr", "may", "jun", "jul", "aug", "sep", "oct", "nov"]
|
||||
|
||||
DAY_NAMES = {
|
||||
"monday",
|
||||
"tuesday",
|
||||
"wednesday",
|
||||
"thursday",
|
||||
"friday",
|
||||
"saturday",
|
||||
"sunday",
|
||||
}
|
||||
|
||||
|
||||
def parse_day_header(bold_text: str):
|
||||
"""Parse the bold header of a diary entry to extract day name and date number.
|
||||
|
||||
Handles formats like:
|
||||
"SATURDAY 3"
|
||||
"13 FRIDAY"
|
||||
"SATURDAY" (date-less, like Aug 14)
|
||||
"MEMORANDA"
|
||||
Returns (day_name, date_number) or (None, None) if not a day entry.
|
||||
"""
|
||||
text = bold_text.strip().split("\n")[0].strip()
|
||||
text = re.sub(r"<br\s*/?>", " ", text).strip()
|
||||
|
||||
if text.upper() == "MEMORANDA":
|
||||
return ("MEMORANDA", None)
|
||||
|
||||
# Try "DAY NUMBER" pattern
|
||||
m = re.match(r"([A-Za-z]+)\s+(\d+)", text)
|
||||
if m:
|
||||
day = m.group(1).upper()
|
||||
num = int(m.group(2))
|
||||
if day in {d.upper() for d in DAY_NAMES}:
|
||||
return (day, num)
|
||||
|
||||
# Try "NUMBER DAY" pattern
|
||||
m = re.match(r"(\d+)\s+([A-Za-z]+)", text)
|
||||
if m:
|
||||
num = int(m.group(1))
|
||||
day = m.group(2).upper()
|
||||
if day in {d.upper() for d in DAY_NAMES}:
|
||||
return (day, num)
|
||||
|
||||
# Try just "DAY" (no number — like SATURDAY 14 where 14 is on next line)
|
||||
word = text.split()[0].upper() if text.split() else ""
|
||||
if word in {d.upper() for d in DAY_NAMES}:
|
||||
# Try to find number after
|
||||
rest = text[len(word) :].strip()
|
||||
m2 = re.match(r"(\d+)", rest)
|
||||
if m2:
|
||||
return (word, int(m2.group(1)))
|
||||
return (word, None)
|
||||
|
||||
return (None, None)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
def main():
|
||||
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
html_path = os.path.join(base_dir, "public", "html", "others-before-1977.html")
|
||||
output_dir = os.path.join(base_dir, "lib", "jsons", "journals", "jaladuta-diary")
|
||||
chapters_dir = os.path.join(output_dir, "chapters")
|
||||
|
||||
if not os.path.exists(html_path):
|
||||
print(f"ERROR: {html_path} not found!")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Reading {html_path}...")
|
||||
with open(html_path, "r", encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
|
||||
soup = BeautifulSoup(content, "html.parser")
|
||||
all_tags = soup.find_all("p")
|
||||
print(f" Parsed {len(all_tags)} <p> tags total")
|
||||
|
||||
# Find the Jaladuta Diary section boundaries
|
||||
start_idx = None
|
||||
end_idx = None
|
||||
for i, tag in enumerate(all_tags):
|
||||
text = tag.get_text().strip()
|
||||
cls = get_css_class(tag)
|
||||
if cls == "p35" and "Jaladuta Diary" in text:
|
||||
start_idx = i
|
||||
if start_idx is not None and i > start_idx:
|
||||
# End at "Prayers Written by Śrīla Prabhupāda aboard the Jaladuta" or the next p4 bookmark after JD
|
||||
if cls == "p4" and "JD 8" in text:
|
||||
end_idx = i
|
||||
break
|
||||
|
||||
if start_idx is None:
|
||||
print("ERROR: Could not find Jaladuta Diary start")
|
||||
sys.exit(1)
|
||||
if end_idx is None:
|
||||
# Fallback: use the memoranda end
|
||||
for i, tag in enumerate(all_tags):
|
||||
if (
|
||||
i > start_idx
|
||||
and "MEMORANDA" in tag.get_text()
|
||||
and get_css_class(tag) == "p110"
|
||||
):
|
||||
end_idx = i + 1
|
||||
break
|
||||
|
||||
print(f" Section: tags[{start_idx}] → tags[{end_idx}]")
|
||||
|
||||
section_tags = all_tags[start_idx:end_idx]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 1: Extract Preface and Introduction
|
||||
# ---------------------------------------------------------------------------
|
||||
preface_paragraphs = []
|
||||
intro_paragraphs = []
|
||||
current_section = None # "preface" | "introduction" | None
|
||||
|
||||
# Also track where months/diary entries begin
|
||||
diary_start_idx = None
|
||||
|
||||
for i, tag in enumerate(section_tags):
|
||||
cls = get_css_class(tag)
|
||||
text = tag.get_text().strip()
|
||||
|
||||
# Detect Preface header
|
||||
if cls == "p3" and text == "Preface":
|
||||
current_section = "preface"
|
||||
continue
|
||||
if cls == "p153": # bookmark
|
||||
continue
|
||||
|
||||
# Detect Introduction header
|
||||
if cls in ("p11", "p157") and "Introduction" in text:
|
||||
if cls == "p157":
|
||||
current_section = "introduction"
|
||||
continue
|
||||
elif cls == "p11":
|
||||
current_section = "introduction"
|
||||
continue
|
||||
|
||||
# Detect month start (diary entries begin)
|
||||
if cls == "p11" and re.match(r"JD \d+:", text):
|
||||
diary_start_idx = i
|
||||
current_section = None
|
||||
break
|
||||
|
||||
if cls == "p29":
|
||||
# Bold month header like "APRIL 1965" — this is diary territory
|
||||
diary_start_idx = i
|
||||
current_section = None
|
||||
break
|
||||
|
||||
# Collect preface/intro paragraphs
|
||||
if current_section == "preface" and cls in ("p154", "p155", "p156"):
|
||||
html = inner_html(tag)
|
||||
if html:
|
||||
preface_paragraphs.append(html)
|
||||
elif current_section == "introduction" and cls in ("p154", "p155", "p156"):
|
||||
html = inner_html(tag)
|
||||
if html:
|
||||
intro_paragraphs.append(html)
|
||||
|
||||
print(f"\n Preface: {len(preface_paragraphs)} paragraphs")
|
||||
print(f" Introduction: {len(intro_paragraphs)} paragraphs")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 2: Extract diary entries grouped by month
|
||||
# ---------------------------------------------------------------------------
|
||||
# Walk through remaining tags, tracking current month
|
||||
|
||||
current_month = None # "apr", "may", etc.
|
||||
current_month_full = None # "April 1965", etc.
|
||||
|
||||
# Section headers that appear mid-month
|
||||
section_headers = {} # Maps tag index to header text
|
||||
|
||||
# All day entries: list of { month, day, date, slug, title, paragraphs }
|
||||
day_entries = []
|
||||
current_entry = None
|
||||
|
||||
MONTH_MAP = {
|
||||
"APRIL": ("apr", "April 1965"),
|
||||
"MAY": ("may", "May 1965"),
|
||||
"JUNE": ("jun", "June 1965"),
|
||||
"JULY": ("jul", "July 1965"),
|
||||
"AUGUST": ("aug", "August 1965"),
|
||||
"SEPTEMBER": ("sep", "September 1965"),
|
||||
"OCTOBER": ("oct", "October 1965"),
|
||||
"NOVEMBER": ("nov", "November 1965"),
|
||||
}
|
||||
|
||||
def finalize_entry():
|
||||
nonlocal current_entry
|
||||
if current_entry and current_entry["paragraphs"]:
|
||||
day_entries.append(current_entry)
|
||||
current_entry = None
|
||||
|
||||
diary_tags = section_tags[diary_start_idx:] if diary_start_idx else []
|
||||
|
||||
for i, tag in enumerate(diary_tags):
|
||||
cls = get_css_class(tag)
|
||||
text = tag.get_text().strip()
|
||||
|
||||
# Skip bookmarks
|
||||
if cls in ("p11", "p4", "p39", "p153"):
|
||||
continue
|
||||
|
||||
# Detect month headers (p29 bold)
|
||||
if cls == "p29":
|
||||
html = inner_html(tag)
|
||||
# Check if this is a month header like "APRIL 1965" or "MAY 1965"
|
||||
for month_key, (abbr, full) in MONTH_MAP.items():
|
||||
if month_key in text.upper():
|
||||
finalize_entry()
|
||||
current_month = abbr
|
||||
current_month_full = full
|
||||
break
|
||||
else:
|
||||
# Could be a section header like "Verse 1." — skip if not month
|
||||
# Or it could be a section header within a month
|
||||
pass
|
||||
continue
|
||||
|
||||
# Detect section headers (p68 class — "The Journey", "America")
|
||||
if cls == "p68":
|
||||
finalize_entry()
|
||||
html = inner_html(tag)
|
||||
# Create a section-header entry
|
||||
if html and current_month:
|
||||
section_entry = {
|
||||
"month": current_month,
|
||||
"month_full": current_month_full,
|
||||
"day": None,
|
||||
"date": None,
|
||||
"slug": None,
|
||||
"title": None,
|
||||
"paragraphs": [f"<b>{re.sub('<[^>]+>', ' ', html).strip()}</b>"],
|
||||
"is_section_header": True,
|
||||
}
|
||||
# Extract title text
|
||||
plain = re.sub("<[^>]+>", " ", html)
|
||||
plain = re.sub(r"\s+", " ", plain).strip()
|
||||
section_entry["title"] = plain
|
||||
day_entries.append(section_entry)
|
||||
continue
|
||||
|
||||
# Detect diary entries (p110 class)
|
||||
if cls == "p110":
|
||||
finalize_entry()
|
||||
|
||||
html = inner_html(tag)
|
||||
if not html:
|
||||
continue
|
||||
|
||||
# Parse the bold header
|
||||
b_tag = tag.find("b")
|
||||
if b_tag:
|
||||
bold_text = b_tag.get_text().strip()
|
||||
day_name, date_num = parse_day_header(bold_text)
|
||||
else:
|
||||
day_name, date_num = (None, None)
|
||||
|
||||
if day_name == "MEMORANDA":
|
||||
current_entry = {
|
||||
"month": current_month,
|
||||
"month_full": current_month_full,
|
||||
"day": "MEMORANDA",
|
||||
"date": None,
|
||||
"slug": "memoranda",
|
||||
"title": "Memoranda",
|
||||
"paragraphs": [html],
|
||||
"is_section_header": False,
|
||||
}
|
||||
elif day_name and date_num and current_month:
|
||||
slug = f"{current_month}-{date_num}"
|
||||
title = f"{day_name.capitalize()} {date_num}"
|
||||
current_entry = {
|
||||
"month": current_month,
|
||||
"month_full": current_month_full,
|
||||
"day": day_name,
|
||||
"date": date_num,
|
||||
"slug": slug,
|
||||
"title": title,
|
||||
"paragraphs": [html],
|
||||
"is_section_header": False,
|
||||
}
|
||||
elif day_name and current_month:
|
||||
# Day without date number (rare — like "SATURDAY" alone on Aug 14)
|
||||
# Try to infer from context or just skip
|
||||
slug = f"{current_month}-{day_name.lower()}"
|
||||
title = day_name.capitalize()
|
||||
current_entry = {
|
||||
"month": current_month,
|
||||
"month_full": current_month_full,
|
||||
"day": day_name,
|
||||
"date": None,
|
||||
"slug": slug,
|
||||
"title": title,
|
||||
"paragraphs": [html],
|
||||
"is_section_header": False,
|
||||
}
|
||||
else:
|
||||
# Can't parse — treat as continuation or standalone
|
||||
current_entry = {
|
||||
"month": current_month,
|
||||
"month_full": current_month_full,
|
||||
"day": None,
|
||||
"date": None,
|
||||
"slug": None,
|
||||
"title": text[:50],
|
||||
"paragraphs": [html],
|
||||
"is_section_header": False,
|
||||
}
|
||||
continue
|
||||
|
||||
# Detect p12 paragraphs (continuation text in some entries like Sep 16)
|
||||
if cls == "p12" and current_entry:
|
||||
html = inner_html(tag)
|
||||
if html:
|
||||
current_entry["paragraphs"].append(html)
|
||||
continue
|
||||
|
||||
# p155 (source citations within diary section)
|
||||
if cls == "p155" and current_entry:
|
||||
html = inner_html(tag)
|
||||
if html:
|
||||
current_entry["paragraphs"].append(html)
|
||||
continue
|
||||
|
||||
finalize_entry()
|
||||
|
||||
print(f"\n Total day entries extracted: {len(day_entries)}")
|
||||
|
||||
# Count by month
|
||||
month_counts = {}
|
||||
for entry in day_entries:
|
||||
m = entry["month"]
|
||||
month_counts[m] = month_counts.get(m, 0) + 1
|
||||
for m in MONTH_ORDER:
|
||||
if m in month_counts:
|
||||
print(f" {m}: {month_counts[m]} entries")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 3: Fix up entries — handle dateless entries, dedup slugs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Some entries have no date — try to fix known cases
|
||||
# Aug "SATURDAY" without number is Aug 14 (follows Aug 13 FRIDAY)
|
||||
for i, entry in enumerate(day_entries):
|
||||
if entry.get("is_section_header"):
|
||||
continue
|
||||
if entry["date"] is None and entry["day"] and entry["day"] != "MEMORANDA":
|
||||
# Try to infer from previous entry
|
||||
if i > 0:
|
||||
prev = day_entries[i - 1]
|
||||
if prev["date"] and prev["month"] == entry["month"]:
|
||||
inferred = prev["date"] + 1
|
||||
entry["date"] = inferred
|
||||
entry["slug"] = f"{entry['month']}-{inferred}"
|
||||
entry["title"] = f"{entry['day'].capitalize()} {inferred}"
|
||||
print(
|
||||
f" Inferred date: {entry['slug']} (from previous {prev['slug']})"
|
||||
)
|
||||
|
||||
# Ensure unique slugs
|
||||
seen_slugs = set()
|
||||
for entry in day_entries:
|
||||
if entry.get("is_section_header"):
|
||||
continue
|
||||
slug = entry["slug"]
|
||||
if slug and slug in seen_slugs:
|
||||
# Append suffix
|
||||
suffix = 2
|
||||
while f"{slug}-{suffix}" in seen_slugs:
|
||||
suffix += 1
|
||||
entry["slug"] = f"{slug}-{suffix}"
|
||||
print(f" Deduped slug: {slug} → {entry['slug']}")
|
||||
if slug:
|
||||
seen_slugs.add(entry["slug"])
|
||||
|
||||
# Merge section headers into the next entry's paragraphs (prepend)
|
||||
merged_entries = []
|
||||
pending_header = None
|
||||
for entry in day_entries:
|
||||
if entry.get("is_section_header"):
|
||||
pending_header = entry["paragraphs"][0]
|
||||
continue
|
||||
if pending_header:
|
||||
entry["paragraphs"] = [pending_header] + entry["paragraphs"]
|
||||
pending_header = None
|
||||
merged_entries.append(entry)
|
||||
day_entries = merged_entries
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 4: Build intro.json with hierarchical month groups
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Assign chapter numbers: preface=1, introduction=2, then day entries sequentially
|
||||
chapter_num = 1
|
||||
|
||||
all_chapters = []
|
||||
|
||||
# Preface
|
||||
preface_chapter = {
|
||||
"slug": "preface",
|
||||
"title": "Preface",
|
||||
"number": chapter_num,
|
||||
}
|
||||
all_chapters.append(preface_chapter)
|
||||
chapter_num += 1
|
||||
|
||||
# Introduction
|
||||
intro_chapter = {
|
||||
"slug": "introduction",
|
||||
"title": "Introduction",
|
||||
"number": chapter_num,
|
||||
}
|
||||
all_chapters.append(intro_chapter)
|
||||
chapter_num += 1
|
||||
|
||||
# Group day entries by month
|
||||
months_data = [] # List of { month, title, subChapters: [...] }
|
||||
current_month_group = None
|
||||
|
||||
for entry in day_entries:
|
||||
month = entry["month"]
|
||||
month_full = entry["month_full"]
|
||||
|
||||
if current_month_group is None or current_month_group["month"] != month:
|
||||
current_month_group = {
|
||||
"month": month,
|
||||
"title": month_full,
|
||||
"subChapters": [],
|
||||
}
|
||||
months_data.append(current_month_group)
|
||||
|
||||
entry["number"] = chapter_num
|
||||
current_month_group["subChapters"].append(
|
||||
{
|
||||
"slug": entry["slug"],
|
||||
"title": entry["title"],
|
||||
}
|
||||
)
|
||||
chapter_num += 1
|
||||
|
||||
# Build the intro.json chapters array
|
||||
# Front matter chapters (preface, introduction) are top-level
|
||||
# Month groups use the hierarchical sub-chapter pattern
|
||||
intro_chapters = []
|
||||
|
||||
# Add preface and introduction as simple chapters
|
||||
intro_chapters.append({"slug": "preface", "title": "Preface"})
|
||||
intro_chapters.append({"slug": "introduction", "title": "Introduction"})
|
||||
|
||||
# Add month groups
|
||||
for mg in months_data:
|
||||
intro_chapters.append(
|
||||
{
|
||||
"title": mg["title"],
|
||||
"subChapters": mg["subChapters"],
|
||||
}
|
||||
)
|
||||
|
||||
intro_json = {
|
||||
"book": "jaladuta-diary",
|
||||
"title": "The Jaladuta Diary, 1965 — Journey to the U.S.A.",
|
||||
"author": "His Divine Grace A.C. Bhaktivedanta Swami Prabhupāda",
|
||||
"year": 1965,
|
||||
"description": "Śrīla Prabhupāda's personal diary kept during his historic voyage from Calcutta to New York aboard the cargo ship Jaladuta in 1965.",
|
||||
"totalChapters": chapter_num - 1,
|
||||
"chapters": intro_chapters,
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 5: Write JSON files
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Clean output directory
|
||||
if os.path.exists(chapters_dir):
|
||||
for f in os.listdir(chapters_dir):
|
||||
os.remove(os.path.join(chapters_dir, f))
|
||||
|
||||
# Write intro.json
|
||||
write_json(intro_json, os.path.join(output_dir, "intro.json"))
|
||||
print(f"\n Written: intro.json ({len(intro_chapters)} top-level entries)")
|
||||
|
||||
# Write preface
|
||||
write_json(
|
||||
{
|
||||
"book": "jaladuta-diary",
|
||||
"chapter": 1,
|
||||
"title": "Preface",
|
||||
"type": "preface",
|
||||
"slug": "preface",
|
||||
"paragraphs": preface_paragraphs,
|
||||
},
|
||||
os.path.join(chapters_dir, "preface.json"),
|
||||
)
|
||||
print(f" Written: preface.json ({len(preface_paragraphs)} paragraphs)")
|
||||
|
||||
# Write introduction
|
||||
write_json(
|
||||
{
|
||||
"book": "jaladuta-diary",
|
||||
"chapter": 2,
|
||||
"title": "Introduction",
|
||||
"type": "introduction",
|
||||
"slug": "introduction",
|
||||
"paragraphs": intro_paragraphs,
|
||||
},
|
||||
os.path.join(chapters_dir, "introduction.json"),
|
||||
)
|
||||
print(f" Written: introduction.json ({len(intro_paragraphs)} paragraphs)")
|
||||
|
||||
# Write day entries
|
||||
ch_num = 3
|
||||
for entry in day_entries:
|
||||
slug = entry["slug"]
|
||||
if not slug:
|
||||
print(f" ⚠ Skipping entry with no slug: {entry.get('title', '?')}")
|
||||
continue
|
||||
|
||||
chapter_data = {
|
||||
"book": "jaladuta-diary",
|
||||
"chapter": ch_num,
|
||||
"title": entry["title"],
|
||||
"type": "chapter",
|
||||
"slug": slug,
|
||||
"paragraphs": entry["paragraphs"],
|
||||
}
|
||||
write_json(chapter_data, os.path.join(chapters_dir, f"{slug}.json"))
|
||||
ch_num += 1
|
||||
|
||||
total_files = 2 + len([e for e in day_entries if e.get("slug")])
|
||||
print(f"\n Total chapter files written: {total_files}")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Summary
|
||||
# ---------------------------------------------------------------------------
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f" EXTRACTION SUMMARY")
|
||||
print(f"{'=' * 60}")
|
||||
print(f" Preface: {len(preface_paragraphs)} paragraphs")
|
||||
print(f" Introduction: {len(intro_paragraphs)} paragraphs")
|
||||
print(f" Day entries: {len(day_entries)}")
|
||||
for mg in months_data:
|
||||
print(f" {mg['title']}: {len(mg['subChapters'])} entries")
|
||||
for sc in mg["subChapters"]:
|
||||
print(f" - {sc['slug']}: {sc['title']}")
|
||||
print(f" Total chapters: {ch_num - 1}")
|
||||
print(f"{'=' * 60}")
|
||||
print(f"\nDone! ✨")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user