spsays-training/scripts/seed-magazines.ts

/**
 * seed-magazines.ts — Push Magazines into MongoDB.
 *
 * Collections written:
 *   - magazine_meta    → one doc per magazine
 *   - magazine_issues  → N issue docs per magazine
 *
 * Usage:
 *   npx tsx scripts/seed-magazines.ts
 */

import dotenv from "dotenv";
import fs from "fs";
import path from "path";
import { MongoClient } from "mongodb";

dotenv.config({ path: ".env.local" });
dotenv.config({ path: ".env" });

const DATABASE_URL = process.env.DATABASE_URL;
if (!DATABASE_URL) {
  console.error("❌ DATABASE_URL is not set");
  process.exit(1);
}

const BASE_DIR = path.resolve(__dirname, "..");
const MAGAZINES_DIR = path.join(BASE_DIR, "lib", "jsons", "magazines");

function readJson<T>(filePath: string): T {
  return JSON.parse(fs.readFileSync(filePath, "utf-8"));
}

function sanitizeString(s: string): string {
  // Normalize to NFC
  s = s.normalize("NFC");
  // Remove null bytes and C0 control chars (keep \t \n \r)
  s = s.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, "");
  // Remove C1 control chars
  s = s.replace(/[\x7f-\x9f]/g, "");
  // Remove lone surrogates (U+D800–U+DFFF) that are invalid in BSON
  s = s.replace(/[\uD800-\uDFFF]/g, "");
  // Remove zero-width characters that can confuse BSON length calculations
  s = s.replace(/[\u200B-\u200F\u2028-\u202F\uFEFF]/g, "");
  // Remove non-characters (U+FDD0–U+FDEF, U+FFFE, U+FFFF)
  s = s.replace(/[\uFDD0-\uFDEF\uFFFE\uFFFF]/g, "");
  // Remove any remaining non-BMP characters that could cause issues
  // (keep common supplementary planes like emoji, but remove PUA etc.)
  s = s.replace(/[\uDB80-\uDBFF][\uDC00-\uDFFF]/g, ""); // Supplementary PUA
  // Round-trip through Buffer to guarantee valid UTF-8 byte sequence
  s = Buffer.from(s, "utf-8").toString("utf-8");
  // Final safety: ensure the string is null-terminated compatible
  // by removing any embedded nulls that survived
  s = s.split("\0").join("");
  return s;
}

function sanitizeDoc(obj: unknown): unknown {
  if (typeof obj === "string") {
    return sanitizeString(obj);
  }
  if (Array.isArray(obj)) return obj.map(sanitizeDoc);
  if (obj !== null && typeof obj === "object") {
    const result: Record<string, unknown> = {};
    for (const [key, value] of Object.entries(obj as Record<string, unknown>))
      result[key] = sanitizeDoc(value);
    return result;
  }
  return obj;
}

async function seedOneMagazine(
  db: ReturnType<MongoClient["db"]>,
  magDir: string,
): Promise<{ bookCode: string; issues: number }> {
  const introPath = path.join(magDir, "intro.json");
  if (!fs.existsSync(introPath)) throw new Error(`No intro.json in ${magDir}`);
  const intro = sanitizeDoc(
    readJson<Record<string, unknown>>(introPath),
  ) as Record<string, unknown>;
  const bookCode = intro.book as string;

  const metaCol = db.collection("magazine_meta");
  await metaCol.replaceOne(
    { book: bookCode },
    { book: bookCode, data: intro },
    { upsert: true },
  );
  console.log(`  📖 Upserted magazine_meta for "${intro.title}" (${bookCode})`);

  const issuesCol = db.collection("magazine_issues");
  // Delete old issues for this magazine
  await issuesCol.deleteMany({ book: bookCode });

  const issuesDir = path.join(magDir, "issues");
  let inserted = 0;
  if (fs.existsSync(issuesDir)) {
    const files = fs
      .readdirSync(issuesDir)
      .filter((f) => f.endsWith(".json"))
      .sort();
    for (const file of files) {
      const issue = sanitizeDoc(
        readJson<Record<string, unknown>>(path.join(issuesDir, file)),
      ) as Record<string, unknown>;

      // Deep-sanitize each paragraph string
      const rawParas = (issue.paragraphs as string[]) || [];
      const cleanParas = rawParas.map((p: string) => sanitizeString(String(p)));

      // Insert the base document without paragraphs first, then $set paragraphs.
      // This avoids the MongoDB BSON wire protocol "Not null terminated string"
      // error that occurs with large string arrays in insertOne/replaceOne.
      const baseDoc = {
        book: bookCode,
        volume: issue.volume,
        part: issue.part,
        title: issue.title,
        slug: issue.slug,
        order: issue.order,
        paragraphs: [] as string[],
      };

      await issuesCol.replaceOne(
        { book: bookCode, slug: issue.slug as string },
        baseDoc,
        { upsert: true },
      );

      // Now set paragraphs in batches using $push to avoid BSON limits
      const BATCH_SIZE = 200;
      for (let bi = 0; bi < cleanParas.length; bi += BATCH_SIZE) {
        const batch = cleanParas.slice(bi, bi + BATCH_SIZE);
        await issuesCol.updateOne(
          { book: bookCode, slug: issue.slug as string },
          { $push: { paragraphs: { $each: batch } } },
        );
      }

      inserted++;
    }
    console.log(`  📄 Issues: ${inserted}`);
  }

  const total = await issuesCol.countDocuments({ book: bookCode });
  console.log(`  ✅ ${bookCode}: ${total} issue entries`);
  return { bookCode, issues: inserted };
}

async function main() {
  console.log("🌱 Seeding Magazines...\n");
  const client = new MongoClient(DATABASE_URL!);
  await client.connect();
  console.log("📡 Connected to MongoDB\n");
  const db = client.db();
  const start = Date.now();

  if (!fs.existsSync(MAGAZINES_DIR)) {
    console.error(`❌ ${MAGAZINES_DIR} not found`);
    process.exit(1);
  }

  const subdirs = fs
    .readdirSync(MAGAZINES_DIR)
    .filter((e) => {
      const p = path.join(MAGAZINES_DIR, e);
      return (
        fs.statSync(p).isDirectory() &&
        fs.existsSync(path.join(p, "intro.json"))
      );
    })
    .sort();

  if (subdirs.length === 0) {
    console.error("❌ No magazine directories found");
    process.exit(1);
  }
  console.log(`📂 Found ${subdirs.length} magazine(s)\n`);

  let totalIssues = 0;
  const bookCodes: string[] = [];
  for (const subdir of subdirs) {
    console.log(`\n📝 Processing: ${subdir}`);
    const result = await seedOneMagazine(db, path.join(MAGAZINES_DIR, subdir));
    bookCodes.push(result.bookCode);
    totalIssues += result.issues;
  }

  const elapsed = ((Date.now() - start) / 1000).toFixed(1);
  console.log("\n" + "─".repeat(50));
  console.log(`📊 Summary: ${subdirs.length} magazines, ${totalIssues} issues`);
  console.log(`   Book codes: ${bookCodes.join(", ")}`);
  console.log(`\n🎉 Done in ${elapsed}s`);
  await client.close();
}

main().catch((err) => {
  console.error("❌ Seed failed:", err);
  process.exit(1);
});