/** * seed-other-documents.ts โ€” Push "Other Documents" into MongoDB. * * Uses dedicated collections for other documents (not shared with books): * - other_document_meta โ†’ one doc per document { book: "", data: } * - other_document_chapters โ†’ N chapter docs per document * * Each "other document" is a subdirectory under lib/jsons/other-documents/ * with an intro.json and chapters/*.json. * * Book codes have NO prefix (e.g. "analysis-bg", "real-peace-formula"). * * Usage: * npx tsx scripts/seed-other-documents.ts */ import dotenv from "dotenv"; import fs from "fs"; import path from "path"; import { MongoClient } from "mongodb"; // Load env dotenv.config({ path: ".env.local" }); dotenv.config({ path: ".env" }); const DATABASE_URL = process.env.DATABASE_URL; if (!DATABASE_URL) { console.error("โŒ DATABASE_URL is not set in .env"); process.exit(1); } const BASE_DIR = path.resolve(__dirname, ".."); const OTHER_DOCS_DIR = path.join(BASE_DIR, "lib", "jsons", "other-documents"); // --------------------------------------------------------------------------- // Types // --------------------------------------------------------------------------- type IntroJson = { book: string; title: string; author: string; year: number | null; description: string | null; totalChapters: number; chapters: { number: number; title: string }[]; }; type ChapterJson = { book: string; chapter: number; title: string; type: string; paragraphs: string[]; }; // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- function readJson(filePath: string): T { return JSON.parse(fs.readFileSync(filePath, "utf-8")); } /** * Recursively sanitize all strings in a document to prevent BSON encoding errors. */ function sanitizeDoc(obj: unknown): unknown { if (typeof obj === "string") { let s = obj.normalize("NFC"); s = s.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, ""); s = s.replace(/[\uD800-\uDFFF]/g, ""); s = s.replace(/[\u200B-\u200F\u2028-\u202F\uFEFF]/g, ""); s = Buffer.from(s, "utf-8").toString("utf-8"); return s; } if (Array.isArray(obj)) { return obj.map(sanitizeDoc); } if (obj !== null && typeof obj === "object") { const result: Record = {}; for (const [key, value] of Object.entries(obj as Record)) { result[key] = sanitizeDoc(value); } return result; } return obj; } // --------------------------------------------------------------------------- // Seed one document // --------------------------------------------------------------------------- async function seedOneDocument( db: ReturnType, docDir: string, ): Promise<{ bookCode: string; chapters: number }> { const introPath = path.join(docDir, "intro.json"); if (!fs.existsSync(introPath)) { throw new Error(`No intro.json found in ${docDir}`); } const intro = sanitizeDoc(readJson(introPath)) as IntroJson; const bookCode = intro.book; // ---- Document meta ---- const metaCol = db.collection("other_document_meta"); await metaCol.deleteMany({ book: bookCode }); await metaCol.insertOne({ book: bookCode, data: intro }); console.log( ` ๐Ÿ“– Inserted other_document_meta for "${intro.title}" (${bookCode})`, ); // ---- Document chapters ---- const chaptersCol = db.collection("other_document_chapters"); await chaptersCol.deleteMany({ book: bookCode }); const docs: Record[] = []; const chaptersDir = path.join(docDir, "chapters"); if (fs.existsSync(chaptersDir)) { const chapterFiles = fs .readdirSync(chaptersDir) .filter((f) => f.endsWith(".json")) .sort(); for (const file of chapterFiles) { const ch = sanitizeDoc( readJson(path.join(chaptersDir, file)), ) as ChapterJson; const chNum = ch.chapter ?? parseInt(file.replace(".json", ""), 10); docs.push({ book: bookCode, chapter: chNum, title: ch.title, type: ch.type || "chapter", slug: String(chNum), order: 100 + chNum, paragraphs: ch.paragraphs, }); } console.log(` ๐Ÿ“„ Chapters: ${chapterFiles.length}`); } // Insert all chapter docs if (docs.length > 0) { console.log( ` ๐Ÿ’พ Inserting ${docs.length} other_document_chapters docs...`, ); for (const doc of docs) { await chaptersCol.insertOne(doc); } } const totalDocs = await chaptersCol.countDocuments({ book: bookCode }); console.log(` โœ… ${bookCode}: ${totalDocs} chapter entries in MongoDB`); return { bookCode, chapters: docs.length }; } // --------------------------------------------------------------------------- // Clean up old data from shared collections // --------------------------------------------------------------------------- async function cleanupOldData(db: ReturnType) { // Remove od- prefixed entries from book_meta (old location) const bookMetaCol = db.collection("book_meta"); const deletedMeta = await bookMetaCol.deleteMany({ book: { $regex: /^od-/ }, }); if (deletedMeta.deletedCount > 0) { console.log( ` ๐Ÿ—‘๏ธ Deleted ${deletedMeta.deletedCount} od-* doc(s) from book_meta`, ); } // Remove od- prefixed entries from prose_chapters (old location) const proseCol = db.collection("prose_chapters"); const deletedProse = await proseCol.deleteMany({ book: { $regex: /^od-/ }, }); if (deletedProse.deletedCount > 0) { console.log( ` ๐Ÿ—‘๏ธ Deleted ${deletedProse.deletedCount} od-* doc(s) from prose_chapters`, ); } // Clean up legacy other_documents collection const collections = await db .listCollections({ name: "other_documents" }) .toArray(); if (collections.length > 0) { const col = db.collection("other_documents"); const count = await col.countDocuments(); if (count > 0) { await col.deleteMany({}); console.log( ` ๐Ÿ—‘๏ธ Deleted ${count} doc(s) from legacy other_documents collection`, ); } } } // --------------------------------------------------------------------------- // Main // --------------------------------------------------------------------------- async function main() { console.log("๐ŸŒฑ Seeding Other Documents...\n"); const client = new MongoClient(DATABASE_URL!); await client.connect(); console.log("๐Ÿ“ก Connected to MongoDB\n"); const db = client.db(); const start = Date.now(); // Discover all subdirectories if (!fs.existsSync(OTHER_DOCS_DIR)) { console.error(`โŒ Directory not found: ${OTHER_DOCS_DIR}`); process.exit(1); } const subdirs = fs .readdirSync(OTHER_DOCS_DIR) .filter((entry) => { const fullPath = path.join(OTHER_DOCS_DIR, entry); return ( fs.statSync(fullPath).isDirectory() && fs.existsSync(path.join(fullPath, "intro.json")) ); }) .sort(); if (subdirs.length === 0) { console.error(`โŒ No document directories found in ${OTHER_DOCS_DIR}`); process.exit(1); } console.log(`๐Ÿ“‚ Found ${subdirs.length} document(s)\n`); // Seed each document let totalChapters = 0; const bookCodes: string[] = []; for (const subdir of subdirs) { console.log(`\n๐Ÿ“ Processing: ${subdir}`); const docDir = path.join(OTHER_DOCS_DIR, subdir); const result = await seedOneDocument(db, docDir); bookCodes.push(result.bookCode); totalChapters += result.chapters; } // Clean up old data from shared collections console.log("\n๐Ÿงน Cleaning up old data from shared collections..."); await cleanupOldData(db); // Summary const elapsed = ((Date.now() - start) / 1000).toFixed(1); console.log("\n" + "โ”€".repeat(50)); console.log(`๐Ÿ“Š Summary:`); console.log(` Documents: ${subdirs.length}`); console.log(` Book codes: ${bookCodes.join(", ")}`); console.log(` Total chapters: ${totalChapters}`); console.log(`\n๐ŸŽ‰ Done in ${elapsed}s`); await client.close(); } main().catch((err) => { console.error("โŒ Seed failed:", err); process.exit(1); });