spsays-training/scripts/seed-sva.ts

/**
 * seed-sva.ts — Push Songs of the Vaiṣṇava Ācāryas JSON data into MongoDB.
 *
 * Uses the native MongoDB driver (same pattern as seed-ssr.ts) to avoid
 * Prisma replica-set write requirements.
 *
 * SVA is a song book with 4 parent parts, each containing multiple songs
 * (sub-chapters). Each song is stored as its own prose_chapters document.
 *
 * Sub-chapters are stored with slugs like "1-1", "1-2", "2-1", etc.
 * Each sub-chapter JSON has parentChapter and subChapter fields used to
 * compute a stable ordering: parentChapter * 100 + subChapter.
 *
 * Collections written:
 *   - book_meta       → one doc { book: "sva", data: <intro.json> }
 *   - prose_chapters  → 3 intro segments + N sub-chapters (songs)
 *
 * Usage:
 *   npx tsx scripts/seed-sva.ts
 */

import dotenv from "dotenv";
import fs from "fs";
import path from "path";
import { MongoClient } from "mongodb";

// Load env
dotenv.config({ path: ".env.local" });
dotenv.config({ path: ".env" });

const DATABASE_URL = process.env.DATABASE_URL;
if (!DATABASE_URL) {
  console.error("❌ DATABASE_URL is not set in .env");
  process.exit(1);
}

const BASE_DIR = path.resolve(__dirname, "..");
const SVA_DIR = path.join(BASE_DIR, "lib", "jsons", "books", "sva");

/** Volume sentinel for books without volumes. */
const NO_VOLUME = 0;

// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------

function readJson<T>(filePath: string): T {
  return JSON.parse(fs.readFileSync(filePath, "utf-8"));
}

/**
 * Recursively sanitize all strings in a document to prevent BSON encoding errors.
 *
 * Strategy: NFC-normalize every string, strip control characters and lone
 * surrogates, then round-trip through Buffer to guarantee the bytes are
 * well-formed UTF-8 that the MongoDB C driver can handle.
 */
function sanitizeDoc(obj: unknown): unknown {
  if (typeof obj === "string") {
    // 1. NFC normalize (composes combining characters)
    let s = obj.normalize("NFC");
    // 2. Remove null bytes and C0 control chars (keep \t \n \r)
    s = s.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, "");
    // 3. Remove lone surrogates (U+D800–U+DFFF) that are invalid in BSON
    s = s.replace(/[\uD800-\uDFFF]/g, "");
    // 4. Remove zero-width characters that can confuse BSON length calculations
    s = s.replace(/[\u200B-\u200F\u2028-\u202F\uFEFF]/g, "");
    // 5. Round-trip through Buffer to guarantee valid UTF-8 byte sequence
    s = Buffer.from(s, "utf-8").toString("utf-8");
    return s;
  }
  if (Array.isArray(obj)) {
    return obj.map(sanitizeDoc);
  }
  if (obj !== null && typeof obj === "object") {
    const result: Record<string, unknown> = {};
    for (const [key, value] of Object.entries(obj as Record<string, unknown>)) {
      result[key] = sanitizeDoc(value);
    }
    return result;
  }
  return obj;
}

/**
 * Insert documents one at a time with a small delay every N docs.
 * Uses insertOne to avoid BSON wire-protocol corruption.
 */
async function insertBatched(
  col: ReturnType<ReturnType<MongoClient["db"]>["collection"]>,
  docs: Record<string, unknown>[],
) {
  for (let i = 0; i < docs.length; i++) {
    const sanitized = sanitizeDoc(docs[i]) as Record<string, unknown>;

    // Work around BSON driver bug with large string arrays:
    // Store paragraphs as a single JSON-encoded string instead of a native
    // array. The data layer (lib/sva.ts) already handles stringified JSON
    // via normalizeParagraphs().
    if (Array.isArray(sanitized.paragraphs)) {
      // Stringify, then sanitize the resulting string to ensure clean UTF-8
      let jsonStr = JSON.stringify(sanitized.paragraphs);
      // NFC normalize the whole JSON string
      jsonStr = jsonStr.normalize("NFC");
      // Remove any stray control chars or surrogates
      jsonStr = jsonStr.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, "");
      jsonStr = jsonStr.replace(/[\uD800-\uDFFF]/g, "");
      jsonStr = jsonStr.replace(/[\u200B-\u200F\u2028-\u202F\uFEFF]/g, "");
      // Round-trip through Buffer to guarantee valid UTF-8 byte sequence
      jsonStr = Buffer.from(jsonStr, "utf-8").toString("utf-8");
      sanitized.paragraphs = jsonStr;
    }

    try {
      await col.insertOne(sanitized);
    } catch (err) {
      const slug = (sanitized.slug as string) || "unknown";
      const title = (sanitized.title as string) || "unknown";
      console.error(
        `\n  ❌ Failed inserting doc ${i} (slug="${slug}", title="${title}")`,
      );
      throw err;
    }
    if ((i + 1) % 100 === 0) {
      await new Promise((r) => setTimeout(r, 100));
    }
  }
}

// ---------------------------------------------------------------------------
// Types for the JSON files
// ---------------------------------------------------------------------------

type SvaIntroJson = {
  book: string;
  title: string;
  author: string;
  year: number;
  description: string;
  totalChapters: number;
  chapters: {
    number: number;
    title: string;
    subChapters: { slug: string; title: string }[];
  }[];
};

type SvaIntroSegmentJson = {
  slug: string;
  title: string;
  type: string;
  book: string;
  order: number;
  paragraphs: string[];
};

type SvaSubChapterJson = {
  book: string;
  slug: string;
  parentChapter: number;
  subChapter: number;
  title: string;
  parentTitle: string;
  type: string;
  paragraphs: string[];
};

// ---------------------------------------------------------------------------
// Seed function
// ---------------------------------------------------------------------------

async function seedSva(db: ReturnType<MongoClient["db"]>) {
  const introPath = path.join(SVA_DIR, "intro.json");
  if (!fs.existsSync(introPath)) {
    console.error("❌ SVA intro.json not found at", introPath);
    process.exit(1);
  }

  const intro = readJson<SvaIntroJson>(introPath);

  // ---- 1. Book meta ----
  const metaCol = db.collection("book_meta");
  await metaCol.deleteMany({ book: "sva" });
  await metaCol.insertOne({ book: "sva", data: intro });
  console.log("  📖 SVA book meta inserted");

  // ---- 2. Prose chapters collection ----
  const proseCol = db.collection("prose_chapters");
  await proseCol.deleteMany({ book: "sva" });

  const docs: Record<string, unknown>[] = [];

  // Intro segments (front-pages, foreword, introduction)
  const introSegmentFiles = [
    "front-pages.json",
    "foreword.json",
    "introduction.json",
  ];

  const introDir = path.join(SVA_DIR, "intro");
  for (const file of introSegmentFiles) {
    const filePath = path.join(introDir, file);
    if (!fs.existsSync(filePath)) {
      console.warn(`  ⚠️  Intro segment not found, skipping: ${file}`);
      continue;
    }

    const seg = readJson<SvaIntroSegmentJson>(filePath);

    docs.push({
      book: "sva",
      volume: NO_VOLUME,
      chapter: null,
      title: seg.title,
      type: "intro-segment",
      slug: `intro-${seg.slug}`,
      order: seg.order,
      paragraphs: seg.paragraphs,
    });
  }
  console.log(`  📄 Intro segments: ${docs.length}`);

  // Sub-chapters (songs): 1-1.json, 1-2.json, 2-1.json, etc.
  const chaptersDir = path.join(SVA_DIR, "chapters");
  let songCount = 0;
  if (fs.existsSync(chaptersDir)) {
    const chapterFiles = fs
      .readdirSync(chaptersDir)
      .filter((f) => f.endsWith(".json"))
      .sort((a, b) => {
        // Sort by parentChapter then subChapter numerically
        const parseSlug = (name: string) => {
          const slug = name.replace(".json", "");
          const parts = slug.split("-");
          return {
            parent: parseInt(parts[0], 10),
            sub: parseInt(parts[1], 10),
          };
        };
        const aa = parseSlug(a);
        const bb = parseSlug(b);
        if (aa.parent !== bb.parent) return aa.parent - bb.parent;
        return aa.sub - bb.sub;
      });

    for (const file of chapterFiles) {
      const sc = readJson<SvaSubChapterJson>(path.join(chaptersDir, file));
      const order = sc.parentChapter * 100 + sc.subChapter;

      docs.push({
        book: "sva",
        volume: NO_VOLUME,
        chapter: null, // null since slug is a string like "1-1"
        title: sc.title,
        type: "sub-chapter",
        slug: sc.slug,
        order,
        paragraphs: sc.paragraphs,
      });
      songCount++;
    }
    console.log(`  🎵 Songs (sub-chapters): ${songCount}`);
  }

  // Insert all docs
  console.log(`\n  Inserting ${docs.length} documents...`);
  await insertBatched(proseCol, docs);

  const totalDocs = await proseCol.countDocuments({ book: "sva" });
  console.log(`  ✅ SVA: ${totalDocs} prose entries in MongoDB`);
}

// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------

async function main() {
  console.log("🌱 Seeding Songs of the Vaiṣṇava Ācāryas...\n");

  const client = new MongoClient(DATABASE_URL!);
  await client.connect();
  console.log("📡 Connected to MongoDB\n");

  const db = client.db();

  const start = Date.now();

  await seedSva(db);

  const elapsed = ((Date.now() - start) / 1000).toFixed(1);
  console.log(`\n🎉 Done in ${elapsed}s`);

  await client.close();
}

main().catch((err) => {
  console.error("❌ Seed failed:", err);
  process.exit(1);
});