spsays-training/scripts/seed-letters.ts

/**
 * seed-letters.ts — Push letters into MongoDB.
 *
 * Collection written:
 *   - letters → 6,866 individual letter documents
 *
 * JSON sources:
 *   - lib/jsons/letters/1947.json through 1977.json
 *   - lib/jsons/letters/undated.json
 *
 * Usage:
 *   npx tsx scripts/seed-letters.ts
 */

import dotenv from "dotenv";
import fs from "fs";
import path from "path";
import { MongoClient } from "mongodb";

dotenv.config({ path: ".env.local" });
dotenv.config({ path: ".env" });

const DATABASE_URL = process.env.DATABASE_URL;
if (!DATABASE_URL) {
  console.error("❌ DATABASE_URL is not set");
  process.exit(1);
}

const BASE_DIR = path.resolve(__dirname, "..");
const LETTERS_DIR = path.join(BASE_DIR, "lib", "jsons", "letters");

function readJson<T>(filePath: string): T {
  return JSON.parse(fs.readFileSync(filePath, "utf-8"));
}

function sanitizeString(s: string): string {
  s = s.normalize("NFC");
  s = s.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, "");
  s = s.replace(/[\x7f-\x9f]/g, "");
  s = s.replace(/[\uD800-\uDFFF]/g, "");
  s = s.replace(/[\u200B-\u200F\u2028-\u202F\uFEFF]/g, "");
  s = s.replace(/[\uFDD0-\uFDEF\uFFFE\uFFFF]/g, "");
  s = Buffer.from(s, "utf-8").toString("utf-8");
  s = s.split("\0").join("");
  return s;
}

function sanitizeDoc(obj: unknown): unknown {
  if (typeof obj === "string") return sanitizeString(obj);
  if (Array.isArray(obj)) return obj.map(sanitizeDoc);
  if (obj !== null && typeof obj === "object") {
    const result: Record<string, unknown> = {};
    for (const [key, value] of Object.entries(obj as Record<string, unknown>))
      result[key] = sanitizeDoc(value);
    return result;
  }
  return obj;
}

type LetterJson = {
  slug: string;
  recipients: string[];
  recipientDisplay: string;
  date: string | null;
  dateRef: string | null;
  year: number | null;
  month: number | null;
  day: number | null;
  location: string | null;
  salutation: string | null;
  paragraphs: string[];
};

async function main() {
  console.log("🌱 Seeding Letters...\n");

  const client = new MongoClient(DATABASE_URL!);
  await client.connect();
  console.log("📡 Connected to MongoDB\n");

  const db = client.db();
  const col = db.collection("letters");
  const start = Date.now();

  // Delete all existing letters
  const deleted = await col.deleteMany({});
  console.log(`  🗑️  Deleted ${deleted.deletedCount} existing letters`);

  // Ensure indexes
  await col.createIndex({ slug: 1 }, { unique: true });
  await col.createIndex({ year: 1 });
  await col.createIndex({ recipient: 1 });
  await col.createIndex({ location: 1 });
  await col.createIndex({ year: 1, month: 1 });

  // Read all year files
  const files = fs
    .readdirSync(LETTERS_DIR)
    .filter((f) => f.endsWith(".json") && f !== "index.json")
    .sort();

  console.log(`\n  📂 Found ${files.length} letter files`);

  let inserted = 0;
  let errors = 0;

  for (const file of files) {
    const letters = readJson<LetterJson[]>(path.join(LETTERS_DIR, file));
    console.log(`\n  📄 ${file}: ${letters.length} letters`);

    for (const letter of letters) {
      const doc = sanitizeDoc(letter) as Record<string, unknown>;

      try {
        await col.insertOne(doc);
        inserted++;

        if (inserted % 500 === 0) {
          console.log(`    ... ${inserted} letters inserted`);
        }
      } catch (e: unknown) {
        const msg = e instanceof Error ? e.message : String(e);
        if (msg.includes("duplicate key")) {
          // Skip duplicates silently
          continue;
        }
        console.error(
          `    ❌ Error inserting ${letter.slug}: ${msg.substring(0, 100)}`,
        );
        errors++;
        if (errors > 10) {
          console.error("    Too many errors, stopping.");
          break;
        }
      }
    }

    if (errors > 10) break;
  }

  // Verify
  const totalDocs = await col.countDocuments();

  const elapsed = ((Date.now() - start) / 1000).toFixed(1);
  console.log("\n" + "─".repeat(50));
  console.log(`📊 Summary:`);
  console.log(`   Inserted: ${inserted} letters`);
  console.log(`   Errors: ${errors}`);
  console.log(`   Total in collection: ${totalDocs}`);
  console.log(`\n🎉 Done in ${elapsed}s`);

  await client.close();
}

main().catch((err) => {
  console.error("❌ Seed failed:", err);
  process.exit(1);
});