feat: add disciples list
This commit is contained in:
gosangam
@@ -0,0 +1,244 @@
|
||||
/**
|
||||
* seed-disciples.ts — Combine local JSON files and seed disciples into MongoDB.
|
||||
*
|
||||
* Reads pre-scraped disciple data from JSON files in data/ directory,
|
||||
* combines them, and inserts into the "disciples" MongoDB collection.
|
||||
*
|
||||
* JSON sources:
|
||||
* - data/disciples_a_b_c_d_e.json
|
||||
* - data/disciples_g_to_l.json
|
||||
* - data/disciples_m_n_o_p.json
|
||||
* - data/disciples_r_s_t.json
|
||||
* - data/disciples_u_v_y.json
|
||||
*
|
||||
* Collection written:
|
||||
* - disciples
|
||||
*
|
||||
* Usage:
|
||||
* npx tsx scripts/seed-disciples.ts
|
||||
*/
|
||||
|
||||
import dotenv from "dotenv";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { MongoClient } from "mongodb";
|
||||
|
||||
dotenv.config({ path: ".env.local" });
|
||||
dotenv.config({ path: ".env" });
|
||||
|
||||
const DATABASE_URL = process.env.DATABASE_URL;
|
||||
if (!DATABASE_URL) {
|
||||
console.error("❌ DATABASE_URL is not set");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const BASE_DIR = path.resolve(__dirname, "..");
|
||||
const DATA_DIR = path.join(BASE_DIR, "data");
|
||||
|
||||
const JSON_FILES = [
|
||||
"disciples_a_b_c_d_e.json",
|
||||
"disciples_g_to_l.json",
|
||||
"disciples_m_n_o_p.json",
|
||||
"disciples_r_s_t.json",
|
||||
"disciples_u_v_y.json",
|
||||
];
|
||||
|
||||
const MONTH_MAP: Record<string, number> = {
|
||||
january: 1,
|
||||
february: 2,
|
||||
march: 3,
|
||||
april: 4,
|
||||
may: 5,
|
||||
june: 6,
|
||||
july: 7,
|
||||
august: 8,
|
||||
september: 9,
|
||||
october: 10,
|
||||
november: 11,
|
||||
december: 12,
|
||||
};
|
||||
|
||||
type RawDisciple = {
|
||||
initiatedName: string;
|
||||
suffix: string;
|
||||
place: string;
|
||||
month: string;
|
||||
year: string;
|
||||
};
|
||||
|
||||
type DiscipleDoc = {
|
||||
initiatedName: string;
|
||||
suffix: string;
|
||||
initiationDate: Date | null;
|
||||
initiationPlace: string | null;
|
||||
karmiName: string | null;
|
||||
avatar: string | null;
|
||||
about: string | null;
|
||||
tags: string[];
|
||||
isVerified: boolean;
|
||||
};
|
||||
|
||||
/**
|
||||
* Parse initiation date from month name + year string.
|
||||
* Day is always set to 1 (exact date unknown from this source).
|
||||
* Returns null if year is missing or unparseable.
|
||||
*/
|
||||
function parseInitiationDate(monthStr: string, yearStr: string): Date | null {
|
||||
if (!yearStr || yearStr.trim() === "") return null;
|
||||
|
||||
const year = parseInt(yearStr.trim(), 10);
|
||||
if (isNaN(year)) return null;
|
||||
|
||||
let month = 1; // Default to January if month is missing
|
||||
if (monthStr && monthStr.trim() !== "") {
|
||||
const m = MONTH_MAP[monthStr.toLowerCase().trim()];
|
||||
if (m) month = m;
|
||||
}
|
||||
|
||||
// Use UTC to avoid timezone issues
|
||||
return new Date(Date.UTC(year, month - 1, 1));
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up a place string.
|
||||
* - Returns null for empty, "?", or missing values
|
||||
* - Strips trailing "?" from uncertain places (e.g., "LA?" → "LA")
|
||||
*/
|
||||
function cleanPlace(place: string): string | null {
|
||||
if (!place || place.trim() === "" || place.trim() === "?") return null;
|
||||
// Keep the "?" — it indicates uncertainty; might be useful for display
|
||||
return place.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up an initiated name.
|
||||
* - Strips trailing "?" from uncertain names
|
||||
*/
|
||||
function cleanName(name: string): string {
|
||||
return name.replace(/\?$/, "").trim();
|
||||
}
|
||||
|
||||
function readJson<T>(filePath: string): T {
|
||||
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log("🌱 Seeding Disciples from local JSON files...\n");
|
||||
|
||||
// ─── Load all JSON files ───────────────────────────────────────────
|
||||
const allRaw: RawDisciple[] = [];
|
||||
|
||||
for (const file of JSON_FILES) {
|
||||
const filePath = path.join(DATA_DIR, file);
|
||||
if (!fs.existsSync(filePath)) {
|
||||
console.error(` ❌ Missing file: ${filePath}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const data = readJson<RawDisciple[]>(filePath);
|
||||
console.log(` 📂 ${file}: ${data.length} entries`);
|
||||
allRaw.push(...data);
|
||||
}
|
||||
|
||||
console.log(`\n 📊 Total raw entries: ${allRaw.length}\n`);
|
||||
|
||||
// ─── Transform to document shape ──────────────────────────────────
|
||||
const docs: DiscipleDoc[] = allRaw
|
||||
.filter((r) => r.initiatedName && r.initiatedName.trim() !== "")
|
||||
.map((r) => ({
|
||||
initiatedName: cleanName(r.initiatedName),
|
||||
suffix: r.suffix && r.suffix.trim() !== "" ? r.suffix.trim() : "Dasa",
|
||||
initiationDate: parseInitiationDate(r.month, r.year),
|
||||
initiationPlace: cleanPlace(r.place),
|
||||
karmiName: null,
|
||||
avatar: null,
|
||||
about: null,
|
||||
tags: [],
|
||||
isVerified: false,
|
||||
}));
|
||||
|
||||
console.log(` 📊 Documents to insert: ${docs.length}\n`);
|
||||
|
||||
// ─── Connect to MongoDB ───────────────────────────────────────────
|
||||
const client = new MongoClient(DATABASE_URL!);
|
||||
await client.connect();
|
||||
console.log(" 📡 Connected to MongoDB\n");
|
||||
|
||||
const db = client.db();
|
||||
const col = db.collection("disciples");
|
||||
const start = Date.now();
|
||||
|
||||
// Delete all existing disciples
|
||||
const deleted = await col.deleteMany({});
|
||||
console.log(` 🗑️ Deleted ${deleted.deletedCount} existing disciples`);
|
||||
|
||||
// Create indexes
|
||||
await col.createIndex({ initiatedName: 1 });
|
||||
await col.createIndex({ initiationPlace: 1 });
|
||||
await col.createIndex({ initiationDate: 1 });
|
||||
await col.createIndex({ suffix: 1 });
|
||||
await col.createIndex({ tags: 1 });
|
||||
|
||||
// ─── Insert in batches ────────────────────────────────────────────
|
||||
let totalInserted = 0;
|
||||
let totalErrors = 0;
|
||||
const BATCH_SIZE = 500;
|
||||
|
||||
for (let i = 0; i < docs.length; i += BATCH_SIZE) {
|
||||
const batch = docs.slice(i, i + BATCH_SIZE);
|
||||
const batchNum = Math.floor(i / BATCH_SIZE) + 1;
|
||||
|
||||
try {
|
||||
const result = await col.insertMany(batch, { ordered: false });
|
||||
totalInserted += result.insertedCount;
|
||||
console.log(
|
||||
` ✅ Batch ${batchNum}: inserted ${result.insertedCount} docs`,
|
||||
);
|
||||
} catch (err: unknown) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
console.error(` ❌ Batch ${batchNum} error: ${msg.substring(0, 150)}`);
|
||||
totalErrors++;
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Verify ───────────────────────────────────────────────────────
|
||||
const totalDocs = await col.countDocuments();
|
||||
|
||||
// Quick stats
|
||||
const suffixStats = await col
|
||||
.aggregate([
|
||||
{ $group: { _id: "$suffix", count: { $sum: 1 } } },
|
||||
{ $sort: { count: -1 } },
|
||||
])
|
||||
.toArray();
|
||||
|
||||
const withDate = await col.countDocuments({
|
||||
initiationDate: { $ne: null },
|
||||
});
|
||||
const withPlace = await col.countDocuments({
|
||||
initiationPlace: { $ne: null },
|
||||
});
|
||||
|
||||
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
|
||||
|
||||
console.log("\n" + "─".repeat(50));
|
||||
console.log("📊 Summary:");
|
||||
console.log(` Raw entries loaded: ${allRaw.length}`);
|
||||
console.log(` Documents inserted: ${totalInserted}`);
|
||||
console.log(` Batch errors: ${totalErrors}`);
|
||||
console.log(` Total in collection: ${totalDocs}`);
|
||||
console.log(` With initiation date: ${withDate}`);
|
||||
console.log(` With initiation place: ${withPlace}`);
|
||||
console.log(`\n By suffix:`);
|
||||
for (const s of suffixStats) {
|
||||
console.log(` ${s._id}: ${s.count}`);
|
||||
}
|
||||
console.log(`\n🎉 Done in ${elapsed}s`);
|
||||
|
||||
await client.close();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("❌ Seed failed:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user