spsays-training/scripts/seed-kb-vol2.ts

import dotenv from "dotenv";
import fs from "fs";
import path from "path";
import { MongoClient } from "mongodb";

dotenv.config({ path: ".env.local" });
dotenv.config({ path: ".env" });

const DATABASE_URL = process.env.DATABASE_URL;
if (!DATABASE_URL) {
  console.error("❌ DATABASE_URL is not set");
  process.exit(1);
}

const LIB_DIR = path.join(process.cwd(), "lib", "jsons");

async function main() {
  const client = new MongoClient(DATABASE_URL!);
  await client.connect();
  console.log("📡 Connected to MongoDB");

  const db = client.db();
  const col = db.collection("prose_chapters");

  // Delete existing Vol 2 data
  const deleted = await col.deleteMany({ book: "kb", volume: 2 });
  console.log(`🗑  Cleared ${deleted.deletedCount} existing Vol 2 docs`);

  const chapDir = path.join(LIB_DIR, "books", "kb", "vol-2", "chapters");
  if (!fs.existsSync(chapDir)) {
    console.error(`ERROR: ${chapDir} not found!`);
    await client.close();
    process.exit(1);
  }

  const files = fs
    .readdirSync(chapDir)
    .filter((f) => f.endsWith(".json"))
    .sort();

  console.log(`\n📝 Seeding ${files.length} Vol 2 chapters...\n`);

  let success = 0;
  let failed = 0;

  for (const file of files) {
    const data = JSON.parse(fs.readFileSync(path.join(chapDir, file), "utf-8"));
    const slug = String(data.chapter).padStart(2, "0");
    const filter = { book: "kb" as const, volume: 2, slug };

    // Sanitize title: encode non-ASCII chars as HTML entities to avoid
    // BSON wire-protocol corruption on the initial insertOne
    const safeTitle = (data.title as string).replace(
      /[^\x20-\x7E]/g,
      (ch: string) => `&#${ch.codePointAt(0)};`,
    );

    try {
      // Step 1: Insert empty document with ASCII-safe title (small, won't corrupt)
      await col.insertOne({
        book: "kb",
        volume: 2,
        chapter: data.chapter ?? null,
        title: safeTitle,
        type: "chapter",
        slug,
        order: 100 + (data.chapter ?? 0),
        paragraphs: [] as string[],
      });

      // Step 1b: Restore the real Unicode title via $set update
      // (small update, less likely to corrupt than the initial insert)
      try {
        await col.updateOne(filter, { $set: { title: data.title } });
      } catch {
        // If title update fails, keep the HTML-entity version — it's readable
      }

      // Step 2: Push paragraphs one at a time via $push
      // Each update is tiny so it avoids BSON wire corruption.
      // If a paragraph fails (BSON encoding error), retry with ASCII-safe version.
      for (let i = 0; i < data.paragraphs.length; i++) {
        const para: string = data.paragraphs[i];
        try {
          await col.updateOne(filter, { $push: { paragraphs: para } });
        } catch {
          // Fallback: strip non-ASCII characters that may cause wire corruption
          const safePara = para.replace(
            /[^\x20-\x7E\n\r\t<>/='"&;.\-!?:,()[\]{}#@*+%$^~`|\\]/g,
            function (ch) {
              // Keep common IAST diacritical characters by encoding as HTML entities
              const cp = ch.codePointAt(0);
              if (cp !== undefined) return `&#${cp};`;
              return "";
            },
          );
          try {
            await col.updateOne(filter, { $push: { paragraphs: safePara } });
          } catch {
            // Last resort: push a placeholder
            await col.updateOne(filter, {
              $push: {
                paragraphs: `[Paragraph ${i + 1} could not be stored due to encoding issues]`,
              },
            });
            console.warn(
              `    ⚠ Ch ${data.chapter} para ${i + 1}: encoding fallback`,
            );
          }
        }
      }

      // Step 3: Verify round-trip
      const readBack = await col.findOne(filter);
      if (
        readBack &&
        Array.isArray(readBack.paragraphs) &&
        readBack.paragraphs.length === data.paragraphs.length
      ) {
        success++;
        console.log(
          `  ✓ Ch ${data.chapter}: ${data.paragraphs.length} paragraphs — ${data.title}`,
        );
      } else {
        failed++;
        console.log(
          `  ✗ Ch ${data.chapter}: verification failed (expected ${data.paragraphs.length}, got ${readBack?.paragraphs?.length ?? "null"})`,
        );
      }
    } catch (err: unknown) {
      failed++;
      const msg = err instanceof Error ? err.message : String(err);
      console.log(`  ✗ Ch ${data.chapter}: ${msg.substring(0, 100)}`);
    }
  }

  // Final count
  const total = await col.countDocuments({ book: "kb" });
  console.log(`\n--- Results ---`);
  console.log(`Success: ${success} | Failed: ${failed}`);
  console.log(`Total KB prose docs in DB: ${total}`);

  await client.close();
}

main().catch((e) => {
  console.error("❌ FATAL:", e.message);
  process.exit(1);
});