/** * seed-magazines.ts — Push Magazines into MongoDB. * * Collections written: * - magazine_meta → one doc per magazine * - magazine_issues → N issue docs per magazine * * Usage: * npx tsx scripts/seed-magazines.ts */ import dotenv from "dotenv"; import fs from "fs"; import path from "path"; import { MongoClient } from "mongodb"; dotenv.config({ path: ".env.local" }); dotenv.config({ path: ".env" }); const DATABASE_URL = process.env.DATABASE_URL; if (!DATABASE_URL) { console.error("❌ DATABASE_URL is not set"); process.exit(1); } const BASE_DIR = path.resolve(__dirname, ".."); const MAGAZINES_DIR = path.join(BASE_DIR, "lib", "jsons", "magazines"); function readJson(filePath: string): T { return JSON.parse(fs.readFileSync(filePath, "utf-8")); } function sanitizeString(s: string): string { // Normalize to NFC s = s.normalize("NFC"); // Remove null bytes and C0 control chars (keep \t \n \r) s = s.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, ""); // Remove C1 control chars s = s.replace(/[\x7f-\x9f]/g, ""); // Remove lone surrogates (U+D800–U+DFFF) that are invalid in BSON s = s.replace(/[\uD800-\uDFFF]/g, ""); // Remove zero-width characters that can confuse BSON length calculations s = s.replace(/[\u200B-\u200F\u2028-\u202F\uFEFF]/g, ""); // Remove non-characters (U+FDD0–U+FDEF, U+FFFE, U+FFFF) s = s.replace(/[\uFDD0-\uFDEF\uFFFE\uFFFF]/g, ""); // Remove any remaining non-BMP characters that could cause issues // (keep common supplementary planes like emoji, but remove PUA etc.) s = s.replace(/[\uDB80-\uDBFF][\uDC00-\uDFFF]/g, ""); // Supplementary PUA // Round-trip through Buffer to guarantee valid UTF-8 byte sequence s = Buffer.from(s, "utf-8").toString("utf-8"); // Final safety: ensure the string is null-terminated compatible // by removing any embedded nulls that survived s = s.split("\0").join(""); return s; } function sanitizeDoc(obj: unknown): unknown { if (typeof obj === "string") { return sanitizeString(obj); } if (Array.isArray(obj)) return obj.map(sanitizeDoc); if (obj !== null && typeof obj === "object") { const result: Record = {}; for (const [key, value] of Object.entries(obj as Record)) result[key] = sanitizeDoc(value); return result; } return obj; } async function seedOneMagazine( db: ReturnType, magDir: string, ): Promise<{ bookCode: string; issues: number }> { const introPath = path.join(magDir, "intro.json"); if (!fs.existsSync(introPath)) throw new Error(`No intro.json in ${magDir}`); const intro = sanitizeDoc( readJson>(introPath), ) as Record; const bookCode = intro.book as string; const metaCol = db.collection("magazine_meta"); await metaCol.replaceOne( { book: bookCode }, { book: bookCode, data: intro }, { upsert: true }, ); console.log(` 📖 Upserted magazine_meta for "${intro.title}" (${bookCode})`); const issuesCol = db.collection("magazine_issues"); // Delete old issues for this magazine await issuesCol.deleteMany({ book: bookCode }); const issuesDir = path.join(magDir, "issues"); let inserted = 0; if (fs.existsSync(issuesDir)) { const files = fs .readdirSync(issuesDir) .filter((f) => f.endsWith(".json")) .sort(); for (const file of files) { const issue = sanitizeDoc( readJson>(path.join(issuesDir, file)), ) as Record; // Deep-sanitize each paragraph string const rawParas = (issue.paragraphs as string[]) || []; const cleanParas = rawParas.map((p: string) => sanitizeString(String(p))); // Insert the base document without paragraphs first, then $set paragraphs. // This avoids the MongoDB BSON wire protocol "Not null terminated string" // error that occurs with large string arrays in insertOne/replaceOne. const baseDoc = { book: bookCode, volume: issue.volume, part: issue.part, title: issue.title, slug: issue.slug, order: issue.order, paragraphs: [] as string[], }; await issuesCol.replaceOne( { book: bookCode, slug: issue.slug as string }, baseDoc, { upsert: true }, ); // Now set paragraphs in batches using $push to avoid BSON limits const BATCH_SIZE = 200; for (let bi = 0; bi < cleanParas.length; bi += BATCH_SIZE) { const batch = cleanParas.slice(bi, bi + BATCH_SIZE); await issuesCol.updateOne( { book: bookCode, slug: issue.slug as string }, { $push: { paragraphs: { $each: batch } } }, ); } inserted++; } console.log(` 📄 Issues: ${inserted}`); } const total = await issuesCol.countDocuments({ book: bookCode }); console.log(` ✅ ${bookCode}: ${total} issue entries`); return { bookCode, issues: inserted }; } async function main() { console.log("🌱 Seeding Magazines...\n"); const client = new MongoClient(DATABASE_URL!); await client.connect(); console.log("📡 Connected to MongoDB\n"); const db = client.db(); const start = Date.now(); if (!fs.existsSync(MAGAZINES_DIR)) { console.error(`❌ ${MAGAZINES_DIR} not found`); process.exit(1); } const subdirs = fs .readdirSync(MAGAZINES_DIR) .filter((e) => { const p = path.join(MAGAZINES_DIR, e); return ( fs.statSync(p).isDirectory() && fs.existsSync(path.join(p, "intro.json")) ); }) .sort(); if (subdirs.length === 0) { console.error("❌ No magazine directories found"); process.exit(1); } console.log(`📂 Found ${subdirs.length} magazine(s)\n`); let totalIssues = 0; const bookCodes: string[] = []; for (const subdir of subdirs) { console.log(`\n📝 Processing: ${subdir}`); const result = await seedOneMagazine(db, path.join(MAGAZINES_DIR, subdir)); bookCodes.push(result.bookCode); totalIssues += result.issues; } const elapsed = ((Date.now() - start) / 1000).toFixed(1); console.log("\n" + "─".repeat(50)); console.log(`📊 Summary: ${subdirs.length} magazines, ${totalIssues} issues`); console.log(` Book codes: ${bookCodes.join(", ")}`); console.log(`\n🎉 Done in ${elapsed}s`); await client.close(); } main().catch((err) => { console.error("❌ Seed failed:", err); process.exit(1); });