/** * seed-disciples.ts — Combine local JSON files and seed disciples into MongoDB. * * Reads pre-scraped disciple data from JSON files in data/ directory, * combines them, and inserts into the "disciples" MongoDB collection. * * JSON sources: * - data/disciples_a_b_c_d_e.json * - data/disciples_g_to_l.json * - data/disciples_m_n_o_p.json * - data/disciples_r_s_t.json * - data/disciples_u_v_y.json * * Collection written: * - disciples * * Usage: * npx tsx scripts/seed-disciples.ts */ import dotenv from "dotenv"; import fs from "fs"; import path from "path"; import { MongoClient } from "mongodb"; dotenv.config({ path: ".env.local" }); dotenv.config({ path: ".env" }); const DATABASE_URL = process.env.DATABASE_URL; if (!DATABASE_URL) { console.error("āŒ DATABASE_URL is not set"); process.exit(1); } const BASE_DIR = path.resolve(__dirname, ".."); const DATA_DIR = path.join(BASE_DIR, "data"); const JSON_FILES = [ "disciples_a_b_c_d_e.json", "disciples_g_to_l.json", "disciples_m_n_o_p.json", "disciples_r_s_t.json", "disciples_u_v_y.json", ]; const MONTH_MAP: Record = { january: 1, february: 2, march: 3, april: 4, may: 5, june: 6, july: 7, august: 8, september: 9, october: 10, november: 11, december: 12, }; type RawDisciple = { initiatedName: string; suffix: string; place: string; month: string; year: string; }; type DiscipleDoc = { initiatedName: string; suffix: string; initiationDate: Date | null; initiationPlace: string | null; karmiName: string | null; avatar: string | null; about: string | null; tags: string[]; isVerified: boolean; }; /** * Parse initiation date from month name + year string. * Day is always set to 1 (exact date unknown from this source). * Returns null if year is missing or unparseable. */ function parseInitiationDate(monthStr: string, yearStr: string): Date | null { if (!yearStr || yearStr.trim() === "") return null; const year = parseInt(yearStr.trim(), 10); if (isNaN(year)) return null; let month = 1; // Default to January if month is missing if (monthStr && monthStr.trim() !== "") { const m = MONTH_MAP[monthStr.toLowerCase().trim()]; if (m) month = m; } // Use UTC to avoid timezone issues return new Date(Date.UTC(year, month - 1, 1)); } /** * Clean up a place string. * - Returns null for empty, "?", or missing values * - Strips trailing "?" from uncertain places (e.g., "LA?" → "LA") */ function cleanPlace(place: string): string | null { if (!place || place.trim() === "" || place.trim() === "?") return null; // Keep the "?" — it indicates uncertainty; might be useful for display return place.trim(); } /** * Clean up an initiated name. * - Strips trailing "?" from uncertain names */ function cleanName(name: string): string { return name.replace(/\?$/, "").trim(); } function readJson(filePath: string): T { return JSON.parse(fs.readFileSync(filePath, "utf-8")); } async function main() { console.log("🌱 Seeding Disciples from local JSON files...\n"); // ─── Load all JSON files ─────────────────────────────────────────── const allRaw: RawDisciple[] = []; for (const file of JSON_FILES) { const filePath = path.join(DATA_DIR, file); if (!fs.existsSync(filePath)) { console.error(` āŒ Missing file: ${filePath}`); process.exit(1); } const data = readJson(filePath); console.log(` šŸ“‚ ${file}: ${data.length} entries`); allRaw.push(...data); } console.log(`\n šŸ“Š Total raw entries: ${allRaw.length}\n`); // ─── Transform to document shape ────────────────────────────────── const docs: DiscipleDoc[] = allRaw .filter((r) => r.initiatedName && r.initiatedName.trim() !== "") .map((r) => ({ initiatedName: cleanName(r.initiatedName), suffix: r.suffix && r.suffix.trim() !== "" ? r.suffix.trim() : "Dasa", initiationDate: parseInitiationDate(r.month, r.year), initiationPlace: cleanPlace(r.place), karmiName: null, avatar: null, about: null, tags: [], isVerified: false, })); console.log(` šŸ“Š Documents to insert: ${docs.length}\n`); // ─── Connect to MongoDB ─────────────────────────────────────────── const client = new MongoClient(DATABASE_URL!); await client.connect(); console.log(" šŸ“” Connected to MongoDB\n"); const db = client.db(); const col = db.collection("disciples"); const start = Date.now(); // Delete all existing disciples const deleted = await col.deleteMany({}); console.log(` šŸ—‘ļø Deleted ${deleted.deletedCount} existing disciples`); // Create indexes await col.createIndex({ initiatedName: 1 }); await col.createIndex({ initiationPlace: 1 }); await col.createIndex({ initiationDate: 1 }); await col.createIndex({ suffix: 1 }); await col.createIndex({ tags: 1 }); // ─── Insert in batches ──────────────────────────────────────────── let totalInserted = 0; let totalErrors = 0; const BATCH_SIZE = 500; for (let i = 0; i < docs.length; i += BATCH_SIZE) { const batch = docs.slice(i, i + BATCH_SIZE); const batchNum = Math.floor(i / BATCH_SIZE) + 1; try { const result = await col.insertMany(batch, { ordered: false }); totalInserted += result.insertedCount; console.log( ` āœ… Batch ${batchNum}: inserted ${result.insertedCount} docs`, ); } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); console.error(` āŒ Batch ${batchNum} error: ${msg.substring(0, 150)}`); totalErrors++; } } // ─── Verify ─────────────────────────────────────────────────────── const totalDocs = await col.countDocuments(); // Quick stats const suffixStats = await col .aggregate([ { $group: { _id: "$suffix", count: { $sum: 1 } } }, { $sort: { count: -1 } }, ]) .toArray(); const withDate = await col.countDocuments({ initiationDate: { $ne: null }, }); const withPlace = await col.countDocuments({ initiationPlace: { $ne: null }, }); const elapsed = ((Date.now() - start) / 1000).toFixed(1); console.log("\n" + "─".repeat(50)); console.log("šŸ“Š Summary:"); console.log(` Raw entries loaded: ${allRaw.length}`); console.log(` Documents inserted: ${totalInserted}`); console.log(` Batch errors: ${totalErrors}`); console.log(` Total in collection: ${totalDocs}`); console.log(` With initiation date: ${withDate}`); console.log(` With initiation place: ${withPlace}`); console.log(`\n By suffix:`); for (const s of suffixStats) { console.log(` ${s._id}: ${s.count}`); } console.log(`\nšŸŽ‰ Done in ${elapsed}s`); await client.close(); } main().catch((err) => { console.error("āŒ Seed failed:", err); process.exit(1); });