275 lines
8.1 KiB
TypeScript
275 lines
8.1 KiB
TypeScript
/**
|
|
* seed-other-documents.ts — Push "Other Documents" into MongoDB.
|
|
*
|
|
* Uses dedicated collections for other documents (not shared with books):
|
|
* - other_document_meta → one doc per document { book: "<slug>", data: <intro.json> }
|
|
* - other_document_chapters → N chapter docs per document
|
|
*
|
|
* Each "other document" is a subdirectory under lib/jsons/other-documents/
|
|
* with an intro.json and chapters/*.json.
|
|
*
|
|
* Book codes have NO prefix (e.g. "analysis-bg", "real-peace-formula").
|
|
*
|
|
* Usage:
|
|
* npx tsx scripts/seed-other-documents.ts
|
|
*/
|
|
|
|
import dotenv from "dotenv";
|
|
import fs from "fs";
|
|
import path from "path";
|
|
import { MongoClient } from "mongodb";
|
|
|
|
// Load env
|
|
dotenv.config({ path: ".env.local" });
|
|
dotenv.config({ path: ".env" });
|
|
|
|
const DATABASE_URL = process.env.DATABASE_URL;
|
|
if (!DATABASE_URL) {
|
|
console.error("❌ DATABASE_URL is not set in .env");
|
|
process.exit(1);
|
|
}
|
|
|
|
const BASE_DIR = path.resolve(__dirname, "..");
|
|
const OTHER_DOCS_DIR = path.join(BASE_DIR, "lib", "jsons", "other-documents");
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Types
|
|
// ---------------------------------------------------------------------------
|
|
|
|
type IntroJson = {
|
|
book: string;
|
|
title: string;
|
|
author: string;
|
|
year: number | null;
|
|
description: string | null;
|
|
totalChapters: number;
|
|
chapters: { number: number; title: string }[];
|
|
};
|
|
|
|
type ChapterJson = {
|
|
book: string;
|
|
chapter: number;
|
|
title: string;
|
|
type: string;
|
|
paragraphs: string[];
|
|
};
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Helpers
|
|
// ---------------------------------------------------------------------------
|
|
|
|
function readJson<T>(filePath: string): T {
|
|
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
|
|
}
|
|
|
|
/**
|
|
* Recursively sanitize all strings in a document to prevent BSON encoding errors.
|
|
*/
|
|
function sanitizeDoc(obj: unknown): unknown {
|
|
if (typeof obj === "string") {
|
|
let s = obj.normalize("NFC");
|
|
s = s.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, "");
|
|
s = s.replace(/[\uD800-\uDFFF]/g, "");
|
|
s = s.replace(/[\u200B-\u200F\u2028-\u202F\uFEFF]/g, "");
|
|
s = Buffer.from(s, "utf-8").toString("utf-8");
|
|
return s;
|
|
}
|
|
if (Array.isArray(obj)) {
|
|
return obj.map(sanitizeDoc);
|
|
}
|
|
if (obj !== null && typeof obj === "object") {
|
|
const result: Record<string, unknown> = {};
|
|
for (const [key, value] of Object.entries(obj as Record<string, unknown>)) {
|
|
result[key] = sanitizeDoc(value);
|
|
}
|
|
return result;
|
|
}
|
|
return obj;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Seed one document
|
|
// ---------------------------------------------------------------------------
|
|
|
|
async function seedOneDocument(
|
|
db: ReturnType<MongoClient["db"]>,
|
|
docDir: string,
|
|
): Promise<{ bookCode: string; chapters: number }> {
|
|
const introPath = path.join(docDir, "intro.json");
|
|
if (!fs.existsSync(introPath)) {
|
|
throw new Error(`No intro.json found in ${docDir}`);
|
|
}
|
|
|
|
const intro = sanitizeDoc(readJson<IntroJson>(introPath)) as IntroJson;
|
|
const bookCode = intro.book;
|
|
|
|
// ---- Document meta ----
|
|
const metaCol = db.collection("other_document_meta");
|
|
await metaCol.deleteMany({ book: bookCode });
|
|
await metaCol.insertOne({ book: bookCode, data: intro });
|
|
console.log(
|
|
` 📖 Inserted other_document_meta for "${intro.title}" (${bookCode})`,
|
|
);
|
|
|
|
// ---- Document chapters ----
|
|
const chaptersCol = db.collection("other_document_chapters");
|
|
await chaptersCol.deleteMany({ book: bookCode });
|
|
|
|
const docs: Record<string, unknown>[] = [];
|
|
|
|
const chaptersDir = path.join(docDir, "chapters");
|
|
if (fs.existsSync(chaptersDir)) {
|
|
const chapterFiles = fs
|
|
.readdirSync(chaptersDir)
|
|
.filter((f) => f.endsWith(".json"))
|
|
.sort();
|
|
|
|
for (const file of chapterFiles) {
|
|
const ch = sanitizeDoc(
|
|
readJson<ChapterJson>(path.join(chaptersDir, file)),
|
|
) as ChapterJson;
|
|
|
|
const chNum = ch.chapter ?? parseInt(file.replace(".json", ""), 10);
|
|
docs.push({
|
|
book: bookCode,
|
|
chapter: chNum,
|
|
title: ch.title,
|
|
type: ch.type || "chapter",
|
|
slug: String(chNum),
|
|
order: 100 + chNum,
|
|
paragraphs: ch.paragraphs,
|
|
});
|
|
}
|
|
console.log(` 📄 Chapters: ${chapterFiles.length}`);
|
|
}
|
|
|
|
// Insert all chapter docs
|
|
if (docs.length > 0) {
|
|
console.log(
|
|
` 💾 Inserting ${docs.length} other_document_chapters docs...`,
|
|
);
|
|
for (const doc of docs) {
|
|
await chaptersCol.insertOne(doc);
|
|
}
|
|
}
|
|
|
|
const totalDocs = await chaptersCol.countDocuments({ book: bookCode });
|
|
console.log(` ✅ ${bookCode}: ${totalDocs} chapter entries in MongoDB`);
|
|
|
|
return { bookCode, chapters: docs.length };
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Clean up old data from shared collections
|
|
// ---------------------------------------------------------------------------
|
|
|
|
async function cleanupOldData(db: ReturnType<MongoClient["db"]>) {
|
|
// Remove od- prefixed entries from book_meta (old location)
|
|
const bookMetaCol = db.collection("book_meta");
|
|
const deletedMeta = await bookMetaCol.deleteMany({
|
|
book: { $regex: /^od-/ },
|
|
});
|
|
if (deletedMeta.deletedCount > 0) {
|
|
console.log(
|
|
` 🗑️ Deleted ${deletedMeta.deletedCount} od-* doc(s) from book_meta`,
|
|
);
|
|
}
|
|
|
|
// Remove od- prefixed entries from prose_chapters (old location)
|
|
const proseCol = db.collection("prose_chapters");
|
|
const deletedProse = await proseCol.deleteMany({
|
|
book: { $regex: /^od-/ },
|
|
});
|
|
if (deletedProse.deletedCount > 0) {
|
|
console.log(
|
|
` 🗑️ Deleted ${deletedProse.deletedCount} od-* doc(s) from prose_chapters`,
|
|
);
|
|
}
|
|
|
|
// Clean up legacy other_documents collection
|
|
const collections = await db
|
|
.listCollections({ name: "other_documents" })
|
|
.toArray();
|
|
if (collections.length > 0) {
|
|
const col = db.collection("other_documents");
|
|
const count = await col.countDocuments();
|
|
if (count > 0) {
|
|
await col.deleteMany({});
|
|
console.log(
|
|
` 🗑️ Deleted ${count} doc(s) from legacy other_documents collection`,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Main
|
|
// ---------------------------------------------------------------------------
|
|
|
|
async function main() {
|
|
console.log("🌱 Seeding Other Documents...\n");
|
|
|
|
const client = new MongoClient(DATABASE_URL!);
|
|
await client.connect();
|
|
console.log("📡 Connected to MongoDB\n");
|
|
|
|
const db = client.db();
|
|
const start = Date.now();
|
|
|
|
// Discover all subdirectories
|
|
if (!fs.existsSync(OTHER_DOCS_DIR)) {
|
|
console.error(`❌ Directory not found: ${OTHER_DOCS_DIR}`);
|
|
process.exit(1);
|
|
}
|
|
|
|
const subdirs = fs
|
|
.readdirSync(OTHER_DOCS_DIR)
|
|
.filter((entry) => {
|
|
const fullPath = path.join(OTHER_DOCS_DIR, entry);
|
|
return (
|
|
fs.statSync(fullPath).isDirectory() &&
|
|
fs.existsSync(path.join(fullPath, "intro.json"))
|
|
);
|
|
})
|
|
.sort();
|
|
|
|
if (subdirs.length === 0) {
|
|
console.error(`❌ No document directories found in ${OTHER_DOCS_DIR}`);
|
|
process.exit(1);
|
|
}
|
|
|
|
console.log(`📂 Found ${subdirs.length} document(s)\n`);
|
|
|
|
// Seed each document
|
|
let totalChapters = 0;
|
|
const bookCodes: string[] = [];
|
|
|
|
for (const subdir of subdirs) {
|
|
console.log(`\n📝 Processing: ${subdir}`);
|
|
const docDir = path.join(OTHER_DOCS_DIR, subdir);
|
|
const result = await seedOneDocument(db, docDir);
|
|
bookCodes.push(result.bookCode);
|
|
totalChapters += result.chapters;
|
|
}
|
|
|
|
// Clean up old data from shared collections
|
|
console.log("\n🧹 Cleaning up old data from shared collections...");
|
|
await cleanupOldData(db);
|
|
|
|
// Summary
|
|
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
|
|
console.log("\n" + "─".repeat(50));
|
|
console.log(`📊 Summary:`);
|
|
console.log(` Documents: ${subdirs.length}`);
|
|
console.log(` Book codes: ${bookCodes.join(", ")}`);
|
|
console.log(` Total chapters: ${totalChapters}`);
|
|
console.log(`\n🎉 Done in ${elapsed}s`);
|
|
|
|
await client.close();
|
|
}
|
|
|
|
main().catch((err) => {
|
|
console.error("❌ Seed failed:", err);
|
|
process.exit(1);
|
|
});
|