feat: letters added

2026-05-04 00:08:31 +05:30
parent 35d1d14f37
commit 401766928a
20 changed files with 4272 additions and 26 deletions
@@ -0,0 +1,158 @@
+/**
+ * seed-letters.ts — Push letters into MongoDB.
+ *
+ * Collection written:
+ *   - letters → 6,866 individual letter documents
+ *
+ * JSON sources:
+ *   - lib/jsons/letters/1947.json through 1977.json
+ *   - lib/jsons/letters/undated.json
+ *
+ * Usage:
+ *   npx tsx scripts/seed-letters.ts
+ */
+
+import dotenv from "dotenv";
+import fs from "fs";
+import path from "path";
+import { MongoClient } from "mongodb";
+
+dotenv.config({ path: ".env.local" });
+dotenv.config({ path: ".env" });
+
+const DATABASE_URL = process.env.DATABASE_URL;
+if (!DATABASE_URL) {
+  console.error("❌ DATABASE_URL is not set");
+  process.exit(1);
+}
+
+const BASE_DIR = path.resolve(__dirname, "..");
+const LETTERS_DIR = path.join(BASE_DIR, "lib", "jsons", "letters");
+
+function readJson<T>(filePath: string): T {
+  return JSON.parse(fs.readFileSync(filePath, "utf-8"));
+}
+
+function sanitizeString(s: string): string {
+  s = s.normalize("NFC");
+  s = s.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, "");
+  s = s.replace(/[\x7f-\x9f]/g, "");
+  s = s.replace(/[\uD800-\uDFFF]/g, "");
+  s = s.replace(/[\u200B-\u200F\u2028-\u202F\uFEFF]/g, "");
+  s = s.replace(/[\uFDD0-\uFDEF\uFFFE\uFFFF]/g, "");
+  s = Buffer.from(s, "utf-8").toString("utf-8");
+  s = s.split("\0").join("");
+  return s;
+}
+
+function sanitizeDoc(obj: unknown): unknown {
+  if (typeof obj === "string") return sanitizeString(obj);
+  if (Array.isArray(obj)) return obj.map(sanitizeDoc);
+  if (obj !== null && typeof obj === "object") {
+    const result: Record<string, unknown> = {};
+    for (const [key, value] of Object.entries(obj as Record<string, unknown>))
+      result[key] = sanitizeDoc(value);
+    return result;
+  }
+  return obj;
+}
+
+type LetterJson = {
+  slug: string;
+  recipients: string[];
+  recipientDisplay: string;
+  date: string | null;
+  dateRef: string | null;
+  year: number | null;
+  month: number | null;
+  day: number | null;
+  location: string | null;
+  salutation: string | null;
+  paragraphs: string[];
+};
+
+async function main() {
+  console.log("🌱 Seeding Letters...\n");
+
+  const client = new MongoClient(DATABASE_URL!);
+  await client.connect();
+  console.log("📡 Connected to MongoDB\n");
+
+  const db = client.db();
+  const col = db.collection("letters");
+  const start = Date.now();
+
+  // Delete all existing letters
+  const deleted = await col.deleteMany({});
+  console.log(`  🗑️  Deleted ${deleted.deletedCount} existing letters`);
+
+  // Ensure indexes
+  await col.createIndex({ slug: 1 }, { unique: true });
+  await col.createIndex({ year: 1 });
+  await col.createIndex({ recipient: 1 });
+  await col.createIndex({ location: 1 });
+  await col.createIndex({ year: 1, month: 1 });
+
+  // Read all year files
+  const files = fs
+    .readdirSync(LETTERS_DIR)
+    .filter((f) => f.endsWith(".json") && f !== "index.json")
+    .sort();
+
+  console.log(`\n  📂 Found ${files.length} letter files`);
+
+  let inserted = 0;
+  let errors = 0;
+
+  for (const file of files) {
+    const letters = readJson<LetterJson[]>(path.join(LETTERS_DIR, file));
+    console.log(`\n  📄 ${file}: ${letters.length} letters`);
+
+    for (const letter of letters) {
+      const doc = sanitizeDoc(letter) as Record<string, unknown>;
+
+      try {
+        await col.insertOne(doc);
+        inserted++;
+
+        if (inserted % 500 === 0) {
+          console.log(`    ... ${inserted} letters inserted`);
+        }
+      } catch (e: unknown) {
+        const msg = e instanceof Error ? e.message : String(e);
+        if (msg.includes("duplicate key")) {
+          // Skip duplicates silently
+          continue;
+        }
+        console.error(
+          `    ❌ Error inserting ${letter.slug}: ${msg.substring(0, 100)}`,
+        );
+        errors++;
+        if (errors > 10) {
+          console.error("    Too many errors, stopping.");
+          break;
+        }
+      }
+    }
+
+    if (errors > 10) break;
+  }
+
+  // Verify
+  const totalDocs = await col.countDocuments();
+
+  const elapsed = ((Date.now() - start) / 1000).toFixed(1);
+  console.log("\n" + "─".repeat(50));
+  console.log(`📊 Summary:`);
+  console.log(`   Inserted: ${inserted} letters`);
+  console.log(`   Errors: ${errors}`);
+  console.log(`   Total in collection: ${totalDocs}`);
+  console.log(`\n🎉 Done in ${elapsed}s`);
+
+  await client.close();
+}
+
+main().catch((err) => {
+  console.error("❌ Seed failed:", err);
+  process.exit(1);
+});