feat: add journals
This commit is contained in:
gosangam
@@ -0,0 +1,274 @@
|
||||
/**
|
||||
* seed-other-documents.ts — Push "Other Documents" into MongoDB.
|
||||
*
|
||||
* Uses dedicated collections for other documents (not shared with books):
|
||||
* - other_document_meta → one doc per document { book: "<slug>", data: <intro.json> }
|
||||
* - other_document_chapters → N chapter docs per document
|
||||
*
|
||||
* Each "other document" is a subdirectory under lib/jsons/other-documents/
|
||||
* with an intro.json and chapters/*.json.
|
||||
*
|
||||
* Book codes have NO prefix (e.g. "analysis-bg", "real-peace-formula").
|
||||
*
|
||||
* Usage:
|
||||
* npx tsx scripts/seed-other-documents.ts
|
||||
*/
|
||||
|
||||
import dotenv from "dotenv";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { MongoClient } from "mongodb";
|
||||
|
||||
// Load env
|
||||
dotenv.config({ path: ".env.local" });
|
||||
dotenv.config({ path: ".env" });
|
||||
|
||||
const DATABASE_URL = process.env.DATABASE_URL;
|
||||
if (!DATABASE_URL) {
|
||||
console.error("❌ DATABASE_URL is not set in .env");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const BASE_DIR = path.resolve(__dirname, "..");
|
||||
const OTHER_DOCS_DIR = path.join(BASE_DIR, "lib", "jsons", "other-documents");
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type IntroJson = {
|
||||
book: string;
|
||||
title: string;
|
||||
author: string;
|
||||
year: number | null;
|
||||
description: string | null;
|
||||
totalChapters: number;
|
||||
chapters: { number: number; title: string }[];
|
||||
};
|
||||
|
||||
type ChapterJson = {
|
||||
book: string;
|
||||
chapter: number;
|
||||
title: string;
|
||||
type: string;
|
||||
paragraphs: string[];
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function readJson<T>(filePath: string): T {
|
||||
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively sanitize all strings in a document to prevent BSON encoding errors.
|
||||
*/
|
||||
function sanitizeDoc(obj: unknown): unknown {
|
||||
if (typeof obj === "string") {
|
||||
let s = obj.normalize("NFC");
|
||||
s = s.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, "");
|
||||
s = s.replace(/[\uD800-\uDFFF]/g, "");
|
||||
s = s.replace(/[\u200B-\u200F\u2028-\u202F\uFEFF]/g, "");
|
||||
s = Buffer.from(s, "utf-8").toString("utf-8");
|
||||
return s;
|
||||
}
|
||||
if (Array.isArray(obj)) {
|
||||
return obj.map(sanitizeDoc);
|
||||
}
|
||||
if (obj !== null && typeof obj === "object") {
|
||||
const result: Record<string, unknown> = {};
|
||||
for (const [key, value] of Object.entries(obj as Record<string, unknown>)) {
|
||||
result[key] = sanitizeDoc(value);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Seed one document
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function seedOneDocument(
|
||||
db: ReturnType<MongoClient["db"]>,
|
||||
docDir: string,
|
||||
): Promise<{ bookCode: string; chapters: number }> {
|
||||
const introPath = path.join(docDir, "intro.json");
|
||||
if (!fs.existsSync(introPath)) {
|
||||
throw new Error(`No intro.json found in ${docDir}`);
|
||||
}
|
||||
|
||||
const intro = sanitizeDoc(readJson<IntroJson>(introPath)) as IntroJson;
|
||||
const bookCode = intro.book;
|
||||
|
||||
// ---- Document meta ----
|
||||
const metaCol = db.collection("other_document_meta");
|
||||
await metaCol.deleteMany({ book: bookCode });
|
||||
await metaCol.insertOne({ book: bookCode, data: intro });
|
||||
console.log(
|
||||
` 📖 Inserted other_document_meta for "${intro.title}" (${bookCode})`,
|
||||
);
|
||||
|
||||
// ---- Document chapters ----
|
||||
const chaptersCol = db.collection("other_document_chapters");
|
||||
await chaptersCol.deleteMany({ book: bookCode });
|
||||
|
||||
const docs: Record<string, unknown>[] = [];
|
||||
|
||||
const chaptersDir = path.join(docDir, "chapters");
|
||||
if (fs.existsSync(chaptersDir)) {
|
||||
const chapterFiles = fs
|
||||
.readdirSync(chaptersDir)
|
||||
.filter((f) => f.endsWith(".json"))
|
||||
.sort();
|
||||
|
||||
for (const file of chapterFiles) {
|
||||
const ch = sanitizeDoc(
|
||||
readJson<ChapterJson>(path.join(chaptersDir, file)),
|
||||
) as ChapterJson;
|
||||
|
||||
const chNum = ch.chapter ?? parseInt(file.replace(".json", ""), 10);
|
||||
docs.push({
|
||||
book: bookCode,
|
||||
chapter: chNum,
|
||||
title: ch.title,
|
||||
type: ch.type || "chapter",
|
||||
slug: String(chNum),
|
||||
order: 100 + chNum,
|
||||
paragraphs: ch.paragraphs,
|
||||
});
|
||||
}
|
||||
console.log(` 📄 Chapters: ${chapterFiles.length}`);
|
||||
}
|
||||
|
||||
// Insert all chapter docs
|
||||
if (docs.length > 0) {
|
||||
console.log(
|
||||
` 💾 Inserting ${docs.length} other_document_chapters docs...`,
|
||||
);
|
||||
for (const doc of docs) {
|
||||
await chaptersCol.insertOne(doc);
|
||||
}
|
||||
}
|
||||
|
||||
const totalDocs = await chaptersCol.countDocuments({ book: bookCode });
|
||||
console.log(` ✅ ${bookCode}: ${totalDocs} chapter entries in MongoDB`);
|
||||
|
||||
return { bookCode, chapters: docs.length };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Clean up old data from shared collections
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function cleanupOldData(db: ReturnType<MongoClient["db"]>) {
|
||||
// Remove od- prefixed entries from book_meta (old location)
|
||||
const bookMetaCol = db.collection("book_meta");
|
||||
const deletedMeta = await bookMetaCol.deleteMany({
|
||||
book: { $regex: /^od-/ },
|
||||
});
|
||||
if (deletedMeta.deletedCount > 0) {
|
||||
console.log(
|
||||
` 🗑️ Deleted ${deletedMeta.deletedCount} od-* doc(s) from book_meta`,
|
||||
);
|
||||
}
|
||||
|
||||
// Remove od- prefixed entries from prose_chapters (old location)
|
||||
const proseCol = db.collection("prose_chapters");
|
||||
const deletedProse = await proseCol.deleteMany({
|
||||
book: { $regex: /^od-/ },
|
||||
});
|
||||
if (deletedProse.deletedCount > 0) {
|
||||
console.log(
|
||||
` 🗑️ Deleted ${deletedProse.deletedCount} od-* doc(s) from prose_chapters`,
|
||||
);
|
||||
}
|
||||
|
||||
// Clean up legacy other_documents collection
|
||||
const collections = await db
|
||||
.listCollections({ name: "other_documents" })
|
||||
.toArray();
|
||||
if (collections.length > 0) {
|
||||
const col = db.collection("other_documents");
|
||||
const count = await col.countDocuments();
|
||||
if (count > 0) {
|
||||
await col.deleteMany({});
|
||||
console.log(
|
||||
` 🗑️ Deleted ${count} doc(s) from legacy other_documents collection`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function main() {
|
||||
console.log("🌱 Seeding Other Documents...\n");
|
||||
|
||||
const client = new MongoClient(DATABASE_URL!);
|
||||
await client.connect();
|
||||
console.log("📡 Connected to MongoDB\n");
|
||||
|
||||
const db = client.db();
|
||||
const start = Date.now();
|
||||
|
||||
// Discover all subdirectories
|
||||
if (!fs.existsSync(OTHER_DOCS_DIR)) {
|
||||
console.error(`❌ Directory not found: ${OTHER_DOCS_DIR}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const subdirs = fs
|
||||
.readdirSync(OTHER_DOCS_DIR)
|
||||
.filter((entry) => {
|
||||
const fullPath = path.join(OTHER_DOCS_DIR, entry);
|
||||
return (
|
||||
fs.statSync(fullPath).isDirectory() &&
|
||||
fs.existsSync(path.join(fullPath, "intro.json"))
|
||||
);
|
||||
})
|
||||
.sort();
|
||||
|
||||
if (subdirs.length === 0) {
|
||||
console.error(`❌ No document directories found in ${OTHER_DOCS_DIR}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`📂 Found ${subdirs.length} document(s)\n`);
|
||||
|
||||
// Seed each document
|
||||
let totalChapters = 0;
|
||||
const bookCodes: string[] = [];
|
||||
|
||||
for (const subdir of subdirs) {
|
||||
console.log(`\n📝 Processing: ${subdir}`);
|
||||
const docDir = path.join(OTHER_DOCS_DIR, subdir);
|
||||
const result = await seedOneDocument(db, docDir);
|
||||
bookCodes.push(result.bookCode);
|
||||
totalChapters += result.chapters;
|
||||
}
|
||||
|
||||
// Clean up old data from shared collections
|
||||
console.log("\n🧹 Cleaning up old data from shared collections...");
|
||||
await cleanupOldData(db);
|
||||
|
||||
// Summary
|
||||
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
|
||||
console.log("\n" + "─".repeat(50));
|
||||
console.log(`📊 Summary:`);
|
||||
console.log(` Documents: ${subdirs.length}`);
|
||||
console.log(` Book codes: ${bookCodes.join(", ")}`);
|
||||
console.log(` Total chapters: ${totalChapters}`);
|
||||
console.log(`\n🎉 Done in ${elapsed}s`);
|
||||
|
||||
await client.close();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("❌ Seed failed:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user