spsays-training/scripts/seed-disciples.ts

/**
 * seed-disciples.ts — Combine local JSON files and seed disciples into MongoDB.
 *
 * Reads pre-scraped disciple data from JSON files in data/ directory,
 * combines them, and inserts into the "disciples" MongoDB collection.
 *
 * JSON sources:
 *   - data/disciples_a_b_c_d_e.json
 *   - data/disciples_g_to_l.json
 *   - data/disciples_m_n_o_p.json
 *   - data/disciples_r_s_t.json
 *   - data/disciples_u_v_y.json
 *
 * Collection written:
 *   - disciples
 *
 * Usage:
 *   npx tsx scripts/seed-disciples.ts
 */

import dotenv from "dotenv";
import fs from "fs";
import path from "path";
import { MongoClient } from "mongodb";

dotenv.config({ path: ".env.local" });
dotenv.config({ path: ".env" });

const DATABASE_URL = process.env.DATABASE_URL;
if (!DATABASE_URL) {
  console.error("❌ DATABASE_URL is not set");
  process.exit(1);
}

const BASE_DIR = path.resolve(__dirname, "..");
const DATA_DIR = path.join(BASE_DIR, "data");

const JSON_FILES = [
  "disciples_a_b_c_d_e.json",
  "disciples_g_to_l.json",
  "disciples_m_n_o_p.json",
  "disciples_r_s_t.json",
  "disciples_u_v_y.json",
];

const MONTH_MAP: Record<string, number> = {
  january: 1,
  february: 2,
  march: 3,
  april: 4,
  may: 5,
  june: 6,
  july: 7,
  august: 8,
  september: 9,
  october: 10,
  november: 11,
  december: 12,
};

type RawDisciple = {
  initiatedName: string;
  suffix: string;
  place: string;
  month: string;
  year: string;
};

type DiscipleDoc = {
  initiatedName: string;
  suffix: string;
  initiationDate: Date | null;
  initiationPlace: string | null;
  karmiName: string | null;
  avatar: string | null;
  about: string | null;
  tags: string[];
  isVerified: boolean;
};

/**
 * Parse initiation date from month name + year string.
 * Day is always set to 1 (exact date unknown from this source).
 * Returns null if year is missing or unparseable.
 */
function parseInitiationDate(monthStr: string, yearStr: string): Date | null {
  if (!yearStr || yearStr.trim() === "") return null;

  const year = parseInt(yearStr.trim(), 10);
  if (isNaN(year)) return null;

  let month = 1; // Default to January if month is missing
  if (monthStr && monthStr.trim() !== "") {
    const m = MONTH_MAP[monthStr.toLowerCase().trim()];
    if (m) month = m;
  }

  // Use UTC to avoid timezone issues
  return new Date(Date.UTC(year, month - 1, 1));
}

/**
 * Clean up a place string.
 * - Returns null for empty, "?", or missing values
 * - Strips trailing "?" from uncertain places (e.g., "LA?" → "LA")
 */
function cleanPlace(place: string): string | null {
  if (!place || place.trim() === "" || place.trim() === "?") return null;
  // Keep the "?" — it indicates uncertainty; might be useful for display
  return place.trim();
}

/**
 * Clean up an initiated name.
 * - Strips trailing "?" from uncertain names
 */
function cleanName(name: string): string {
  return name.replace(/\?$/, "").trim();
}

function readJson<T>(filePath: string): T {
  return JSON.parse(fs.readFileSync(filePath, "utf-8"));
}

async function main() {
  console.log("🌱 Seeding Disciples from local JSON files...\n");

  // ─── Load all JSON files ───────────────────────────────────────────
  const allRaw: RawDisciple[] = [];

  for (const file of JSON_FILES) {
    const filePath = path.join(DATA_DIR, file);
    if (!fs.existsSync(filePath)) {
      console.error(`  ❌ Missing file: ${filePath}`);
      process.exit(1);
    }

    const data = readJson<RawDisciple[]>(filePath);
    console.log(`  📂 ${file}: ${data.length} entries`);
    allRaw.push(...data);
  }

  console.log(`\n  📊 Total raw entries: ${allRaw.length}\n`);

  // ─── Transform to document shape ──────────────────────────────────
  const docs: DiscipleDoc[] = allRaw
    .filter((r) => r.initiatedName && r.initiatedName.trim() !== "")
    .map((r) => ({
      initiatedName: cleanName(r.initiatedName),
      suffix: r.suffix && r.suffix.trim() !== "" ? r.suffix.trim() : "Dasa",
      initiationDate: parseInitiationDate(r.month, r.year),
      initiationPlace: cleanPlace(r.place),
      karmiName: null,
      avatar: null,
      about: null,
      tags: [],
      isVerified: false,
    }));

  console.log(`  📊 Documents to insert: ${docs.length}\n`);

  // ─── Connect to MongoDB ───────────────────────────────────────────
  const client = new MongoClient(DATABASE_URL!);
  await client.connect();
  console.log("  📡 Connected to MongoDB\n");

  const db = client.db();
  const col = db.collection("disciples");
  const start = Date.now();

  // Delete all existing disciples
  const deleted = await col.deleteMany({});
  console.log(`  🗑️  Deleted ${deleted.deletedCount} existing disciples`);

  // Create indexes
  await col.createIndex({ initiatedName: 1 });
  await col.createIndex({ initiationPlace: 1 });
  await col.createIndex({ initiationDate: 1 });
  await col.createIndex({ suffix: 1 });
  await col.createIndex({ tags: 1 });

  // ─── Insert in batches ────────────────────────────────────────────
  let totalInserted = 0;
  let totalErrors = 0;
  const BATCH_SIZE = 500;

  for (let i = 0; i < docs.length; i += BATCH_SIZE) {
    const batch = docs.slice(i, i + BATCH_SIZE);
    const batchNum = Math.floor(i / BATCH_SIZE) + 1;

    try {
      const result = await col.insertMany(batch, { ordered: false });
      totalInserted += result.insertedCount;
      console.log(
        `    ✅ Batch ${batchNum}: inserted ${result.insertedCount} docs`,
      );
    } catch (err: unknown) {
      const msg = err instanceof Error ? err.message : String(err);
      console.error(`    ❌ Batch ${batchNum} error: ${msg.substring(0, 150)}`);
      totalErrors++;
    }
  }

  // ─── Verify ───────────────────────────────────────────────────────
  const totalDocs = await col.countDocuments();

  // Quick stats
  const suffixStats = await col
    .aggregate([
      { $group: { _id: "$suffix", count: { $sum: 1 } } },
      { $sort: { count: -1 } },
    ])
    .toArray();

  const withDate = await col.countDocuments({
    initiationDate: { $ne: null },
  });
  const withPlace = await col.countDocuments({
    initiationPlace: { $ne: null },
  });

  const elapsed = ((Date.now() - start) / 1000).toFixed(1);

  console.log("\n" + "─".repeat(50));
  console.log("📊 Summary:");
  console.log(`   Raw entries loaded:    ${allRaw.length}`);
  console.log(`   Documents inserted:    ${totalInserted}`);
  console.log(`   Batch errors:          ${totalErrors}`);
  console.log(`   Total in collection:   ${totalDocs}`);
  console.log(`   With initiation date:  ${withDate}`);
  console.log(`   With initiation place: ${withPlace}`);
  console.log(`\n   By suffix:`);
  for (const s of suffixStats) {
    console.log(`     ${s._id}: ${s.count}`);
  }
  console.log(`\n🎉 Done in ${elapsed}s`);

  await client.close();
}

main().catch((err) => {
  console.error("❌ Seed failed:", err);
  process.exit(1);
});