feat: letters added

2026-05-04 00:08:31 +05:30
parent 35d1d14f37
commit 401766928a
20 changed files with 4272 additions and 26 deletions
@@ -0,0 +1,189 @@
+/**
+ * seed-magazines.ts — Push Magazines into MongoDB.
+ *
+ * Collections written:
+ *   - magazine_meta    → one doc per magazine
+ *   - magazine_issues  → N issue docs per magazine
+ *
+ * Usage:
+ *   npx tsx scripts/seed-magazines.ts
+ */
+
+import dotenv from "dotenv";
+import fs from "fs";
+import path from "path";
+import { MongoClient } from "mongodb";
+
+dotenv.config({ path: ".env.local" });
+dotenv.config({ path: ".env" });
+
+const DATABASE_URL = process.env.DATABASE_URL;
+if (!DATABASE_URL) {
+  console.error("❌ DATABASE_URL is not set");
+  process.exit(1);
+}
+
+const BASE_DIR = path.resolve(__dirname, "..");
+const MAGAZINES_DIR = path.join(BASE_DIR, "lib", "jsons", "magazines");
+
+function readJson<T>(filePath: string): T {
+  return JSON.parse(fs.readFileSync(filePath, "utf-8"));
+}
+
+function sanitizeString(s: string): string {
+  // Normalize to NFC
+  s = s.normalize("NFC");
+  // Remove null bytes and C0 control chars (keep \t \n \r)
+  s = s.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, "");
+  // Remove C1 control chars
+  s = s.replace(/[\x7f-\x9f]/g, "");
+  // Remove lone surrogates (U+D800–U+DFFF) that are invalid in BSON
+  s = s.replace(/[\uD800-\uDFFF]/g, "");
+  // Remove zero-width characters that can confuse BSON length calculations
+  s = s.replace(/[\u200B-\u200F\u2028-\u202F\uFEFF]/g, "");
+  // Remove non-characters (U+FDD0–U+FDEF, U+FFFE, U+FFFF)
+  s = s.replace(/[\uFDD0-\uFDEF\uFFFE\uFFFF]/g, "");
+  // Remove any remaining non-BMP characters that could cause issues
+  // (keep common supplementary planes like emoji, but remove PUA etc.)
+  s = s.replace(/[\uDB80-\uDBFF][\uDC00-\uDFFF]/g, ""); // Supplementary PUA
+  // Round-trip through Buffer to guarantee valid UTF-8 byte sequence
+  s = Buffer.from(s, "utf-8").toString("utf-8");
+  // Final safety: ensure the string is null-terminated compatible
+  // by removing any embedded nulls that survived
+  s = s.split("\0").join("");
+  return s;
+}
+
+function sanitizeDoc(obj: unknown): unknown {
+  if (typeof obj === "string") {
+    return sanitizeString(obj);
+  }
+  if (Array.isArray(obj)) return obj.map(sanitizeDoc);
+  if (obj !== null && typeof obj === "object") {
+    const result: Record<string, unknown> = {};
+    for (const [key, value] of Object.entries(obj as Record<string, unknown>))
+      result[key] = sanitizeDoc(value);
+    return result;
+  }
+  return obj;
+}
+
+async function seedOneMagazine(
+  db: ReturnType<MongoClient["db"]>,
+  magDir: string,
+): Promise<{ bookCode: string; issues: number }> {
+  const introPath = path.join(magDir, "intro.json");
+  if (!fs.existsSync(introPath)) throw new Error(`No intro.json in ${magDir}`);
+  const intro = sanitizeDoc(
+    readJson<Record<string, unknown>>(introPath),
+  ) as Record<string, unknown>;
+  const bookCode = intro.book as string;
+
+  const metaCol = db.collection("magazine_meta");
+  await metaCol.replaceOne(
+    { book: bookCode },
+    { book: bookCode, data: intro },
+    { upsert: true },
+  );
+  console.log(`  📖 Upserted magazine_meta for "${intro.title}" (${bookCode})`);
+
+  const issuesCol = db.collection("magazine_issues");
+  // Delete old issues for this magazine
+  await issuesCol.deleteMany({ book: bookCode });
+
+  const issuesDir = path.join(magDir, "issues");
+  let inserted = 0;
+  if (fs.existsSync(issuesDir)) {
+    const files = fs
+      .readdirSync(issuesDir)
+      .filter((f) => f.endsWith(".json"))
+      .sort();
+    for (const file of files) {
+      const issue = sanitizeDoc(
+        readJson<Record<string, unknown>>(path.join(issuesDir, file)),
+      ) as Record<string, unknown>;
+
+      // Deep-sanitize each paragraph string
+      const rawParas = (issue.paragraphs as string[]) || [];
+      const cleanParas = rawParas.map((p: string) => sanitizeString(String(p)));
+
+      // Store paragraphs as a single JSON string to avoid MongoDB BSON
+      // wire protocol issues with large string arrays.
+      // The data layer will JSON.parse() this back into an array on read.
+      const paragraphsJson = JSON.stringify(cleanParas);
+
+      const doc = {
+        book: bookCode,
+        volume: issue.volume,
+        part: issue.part,
+        title: issue.title,
+        slug: issue.slug,
+        order: issue.order,
+        paragraphsJson,
+        paragraphs: [] as string[], // Keep empty array for schema compat
+      };
+
+      await issuesCol.replaceOne({ book: bookCode, slug: issue.slug }, doc, {
+        upsert: true,
+      });
+      inserted++;
+    }
+    console.log(`  📄 Issues: ${inserted}`);
+  }
+
+  const total = await issuesCol.countDocuments({ book: bookCode });
+  console.log(`  ✅ ${bookCode}: ${total} issue entries`);
+  return { bookCode, issues: inserted };
+}
+
+async function main() {
+  console.log("🌱 Seeding Magazines...\n");
+  const client = new MongoClient(DATABASE_URL!);
+  await client.connect();
+  console.log("📡 Connected to MongoDB\n");
+  const db = client.db();
+  const start = Date.now();
+
+  if (!fs.existsSync(MAGAZINES_DIR)) {
+    console.error(`❌ ${MAGAZINES_DIR} not found`);
+    process.exit(1);
+  }
+
+  const subdirs = fs
+    .readdirSync(MAGAZINES_DIR)
+    .filter((e) => {
+      const p = path.join(MAGAZINES_DIR, e);
+      return (
+        fs.statSync(p).isDirectory() &&
+        fs.existsSync(path.join(p, "intro.json"))
+      );
+    })
+    .sort();
+
+  if (subdirs.length === 0) {
+    console.error("❌ No magazine directories found");
+    process.exit(1);
+  }
+  console.log(`📂 Found ${subdirs.length} magazine(s)\n`);
+
+  let totalIssues = 0;
+  const bookCodes: string[] = [];
+  for (const subdir of subdirs) {
+    console.log(`\n📝 Processing: ${subdir}`);
+    const result = await seedOneMagazine(db, path.join(MAGAZINES_DIR, subdir));
+    bookCodes.push(result.bookCode);
+    totalIssues += result.issues;
+  }
+
+  const elapsed = ((Date.now() - start) / 1000).toFixed(1);
+  console.log("\n" + "─".repeat(50));
+  console.log(`📊 Summary: ${subdirs.length} magazines, ${totalIssues} issues`);
+  console.log(`   Book codes: ${bookCodes.join(", ")}`);
+  console.log(`\n🎉 Done in ${elapsed}s`);
+  await client.close();
+}
+
+main().catch((err) => {
+  console.error("❌ Seed failed:", err);
+  process.exit(1);
+});