1173 lines
34 KiB
Go
1173 lines
34 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"flag"
|
|
"fmt"
|
|
"net/url"
|
|
"os"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/prabhupadasays/spsays-bff/internal/config"
|
|
"github.com/prabhupadasays/spsays-bff/internal/repository"
|
|
"github.com/prabhupadasays/spsays-bff/pkg/revalidate"
|
|
"github.com/typesense/typesense-go/v2/typesense"
|
|
"github.com/typesense/typesense-go/v2/typesense/api"
|
|
"go.mongodb.org/mongo-driver/v2/bson"
|
|
"go.uber.org/zap"
|
|
)
|
|
|
|
// CLI flags
|
|
var (
|
|
fullReindex = flag.Bool("full", false, "Drop all collections and reindex everything")
|
|
onlyFlag = flag.String("only", "", "Comma-separated list of collections to index (e.g., verses,letters)")
|
|
noDrop = flag.Bool("no-drop", false, "Do not drop the collection first — upsert into the existing one (resume a partial import)")
|
|
)
|
|
|
|
// All supported collection names
|
|
var allCollections = []string{
|
|
"verses", "prose", "letters", "transcriptions", "glossary",
|
|
"essays", "journals", "magazines", "other_documents",
|
|
}
|
|
|
|
// hyphenSeparators is applied to every collection so that hyphenated
|
|
// Sanskrit/Vedic terms (e.g. "Dīkṣā-guru", "Hare-Kṛṣṇa", "sarva-dharman")
|
|
// are tokenised into individual words. Without this, a query like
|
|
// "Diksa guru" only matches one of the two query tokens against the
|
|
// indexed term "diksa-guru", which dramatically lowers its score.
|
|
var hyphenSeparators = []string{"-"}
|
|
|
|
// HTML stripping helper
|
|
var htmlTagRe = regexp.MustCompile(`<[^>]*>`)
|
|
|
|
func stripHTML(s string) string {
|
|
return htmlTagRe.ReplaceAllString(s, "")
|
|
}
|
|
|
|
func joinParagraphs(paragraphs []string) string {
|
|
var sb strings.Builder
|
|
for i, p := range paragraphs {
|
|
if i > 0 {
|
|
sb.WriteString(" ")
|
|
}
|
|
sb.WriteString(stripHTML(p))
|
|
}
|
|
return sb.String()
|
|
}
|
|
|
|
// IAST diacritical → ASCII mapping for typo-tolerant search.
|
|
// Allows "Krsna" to match "Kṛṣṇa", "Prabhupada" to match "Prabhupāda", etc.
|
|
var diacriticReplacer = strings.NewReplacer(
|
|
"ā", "a", "Ā", "A",
|
|
"ī", "i", "Ī", "I",
|
|
"ū", "u", "Ū", "U",
|
|
"ṛ", "r", "Ṛ", "R",
|
|
"ṝ", "r", "Ṝ", "R",
|
|
"ṣ", "s", "Ṣ", "S",
|
|
"ś", "s", "Ś", "S",
|
|
"ṭ", "t", "Ṭ", "T",
|
|
"ḍ", "d", "Ḍ", "D",
|
|
"ṇ", "n", "Ṇ", "N",
|
|
"ñ", "n", "Ñ", "N",
|
|
"ṅ", "n", "Ṅ", "N",
|
|
"ṁ", "m", "Ṁ", "M",
|
|
"ḥ", "h", "Ḥ", "H",
|
|
"ḷ", "l", "Ḷ", "L",
|
|
"é", "e", "É", "E",
|
|
"è", "e",
|
|
"ö", "o",
|
|
"ü", "u",
|
|
"ä", "a",
|
|
// Common ligatures
|
|
"æ", "ae", "œ", "oe",
|
|
)
|
|
|
|
// stripDiacritics converts IAST/Unicode diacritical characters to plain ASCII.
|
|
func stripDiacritics(s string) string {
|
|
return diacriticReplacer.Replace(s)
|
|
}
|
|
|
|
// Pointer helpers
|
|
func stringPtr(s string) *string { return &s }
|
|
func boolPtr(b bool) *bool { return &b }
|
|
|
|
func main() {
|
|
flag.Parse()
|
|
|
|
// Load config
|
|
cfg, err := config.Load()
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "failed to load config: %v\n", err)
|
|
os.Exit(1)
|
|
}
|
|
|
|
// Initialize logger
|
|
logger, _ := zap.NewDevelopment()
|
|
defer logger.Sync()
|
|
sugar := logger.Sugar()
|
|
|
|
// Connect to MongoDB
|
|
mongoDB, err := repository.NewMongoDB(cfg.MongoURI, cfg.MongoDB, logger)
|
|
if err != nil {
|
|
sugar.Fatalf("failed to connect to MongoDB: %v", err)
|
|
}
|
|
defer mongoDB.Close()
|
|
|
|
// Create Typesense client.
|
|
// Default connection timeout is 5s — too short for large content imports
|
|
// (magazines and other_documents can ship multi-MB batches).
|
|
tsClient := typesense.NewClient(
|
|
typesense.WithServer(cfg.TypesenseURL),
|
|
typesense.WithAPIKey(cfg.TypesenseAPIKey),
|
|
typesense.WithConnectionTimeout(5*time.Minute),
|
|
)
|
|
|
|
// Determine which collections to index
|
|
collectionsToIndex := allCollections
|
|
if *onlyFlag != "" {
|
|
collectionsToIndex = strings.Split(*onlyFlag, ",")
|
|
for i := range collectionsToIndex {
|
|
collectionsToIndex[i] = strings.TrimSpace(collectionsToIndex[i])
|
|
}
|
|
}
|
|
|
|
// Get existing collections
|
|
existingCollections := make(map[string]bool)
|
|
existing, err := tsClient.Collections().Retrieve(context.Background())
|
|
if err != nil {
|
|
sugar.Warnf("could not retrieve existing collections: %v", err)
|
|
} else {
|
|
for _, col := range existing {
|
|
existingCollections[col.Name] = true
|
|
}
|
|
}
|
|
|
|
for _, name := range collectionsToIndex {
|
|
sugar.Infof("=== Processing collection: %s ===", name)
|
|
|
|
shouldDrop := (*fullReindex || *onlyFlag != "") && !*noDrop
|
|
// On a bare run (no targeting), skip collections that already exist.
|
|
// An explicit --full / --only / --no-drop always (re)processes them.
|
|
targeted := *fullReindex || *onlyFlag != "" || *noDrop
|
|
if !targeted && existingCollections[name] {
|
|
sugar.Infof("[%s] Collection already exists, skipping (use --full to force reindex)", name)
|
|
continue
|
|
}
|
|
|
|
// Drop if exists and we need to recreate
|
|
if existingCollections[name] && shouldDrop {
|
|
sugar.Infof("[%s] Dropping existing collection...", name)
|
|
_, err := tsClient.Collection(name).Delete(context.Background())
|
|
if err != nil {
|
|
sugar.Warnf("[%s] Failed to drop collection: %v", name, err)
|
|
}
|
|
}
|
|
|
|
// Create schema and index
|
|
switch name {
|
|
case "verses":
|
|
indexVerses(sugar, tsClient, mongoDB)
|
|
case "prose":
|
|
indexProse(sugar, tsClient, mongoDB)
|
|
case "letters":
|
|
indexLetters(sugar, tsClient, mongoDB)
|
|
case "transcriptions":
|
|
indexTranscriptions(sugar, tsClient, mongoDB)
|
|
case "glossary":
|
|
indexGlossary(sugar, tsClient, mongoDB)
|
|
case "essays":
|
|
indexEssays(sugar, tsClient, mongoDB)
|
|
case "journals":
|
|
indexJournals(sugar, tsClient, mongoDB)
|
|
case "magazines":
|
|
indexMagazines(sugar, tsClient, mongoDB)
|
|
case "other_documents":
|
|
indexOtherDocuments(sugar, tsClient, mongoDB)
|
|
default:
|
|
sugar.Warnf("Unknown collection: %s", name)
|
|
}
|
|
}
|
|
|
|
sugar.Info("=== Indexing complete ===")
|
|
|
|
// Content changed → refresh the frontend sitemap cache (synchronous: this
|
|
// CLI exits right after, so a background goroutine wouldn't run). No-op
|
|
// when FRONTEND_URL / REVALIDATE_SECRET are unset.
|
|
if err := revalidate.New(cfg.FrontendURL, cfg.RevalidateSecret, logger).TriggerSync("sitemap"); err != nil {
|
|
sugar.Warnf("sitemap revalidation ping failed: %v", err)
|
|
} else if cfg.RevalidateSecret != "" {
|
|
sugar.Info("sitemap revalidation pinged")
|
|
}
|
|
}
|
|
|
|
// importBatch imports documents into Typesense in batches of batchSize.
|
|
func importBatch(sugar *zap.SugaredLogger, tsClient *typesense.Client, collectionName string, docs []interface{}, batchSize int) {
|
|
total := len(docs)
|
|
// Upsert (not create) so a re-run over an existing collection is
|
|
// idempotent and additive — essential for resuming a partial import
|
|
// when a flaky link drops batches mid-run.
|
|
action := "upsert"
|
|
for i := 0; i < total; i += batchSize {
|
|
end := i + batchSize
|
|
if end > total {
|
|
end = total
|
|
}
|
|
batch := docs[i:end]
|
|
// Retry the batch on transient transport errors (flaky networks can
|
|
// drop large TLS writes — "broken pipe" / "bad record MAC").
|
|
var responses []*api.ImportDocumentResponse
|
|
var err error
|
|
for attempt := 1; attempt <= 8; attempt++ {
|
|
responses, err = tsClient.Collection(collectionName).Documents().Import(context.Background(), batch, &api.ImportDocumentsParams{
|
|
Action: &action,
|
|
})
|
|
if err == nil {
|
|
break
|
|
}
|
|
sugar.Warnf("[%s] batch %d-%d attempt %d/5 failed: %v", collectionName, i, end, attempt, err)
|
|
time.Sleep(time.Duration(attempt) * time.Second)
|
|
}
|
|
if err != nil {
|
|
sugar.Errorf("[%s] Import error at batch %d-%d (gave up): %v", collectionName, i, end, err)
|
|
continue
|
|
}
|
|
// Check for per-document errors
|
|
errCount := 0
|
|
for _, resp := range responses {
|
|
if !resp.Success {
|
|
errCount++
|
|
if errCount <= 3 {
|
|
sugar.Warnf("[%s] Document import error: %s", collectionName, resp.Error)
|
|
}
|
|
}
|
|
}
|
|
if errCount > 3 {
|
|
sugar.Warnf("[%s] ... and %d more document errors in this batch", collectionName, errCount-3)
|
|
}
|
|
sugar.Infof("[%s] %d/%d indexed...", collectionName, end, total)
|
|
}
|
|
}
|
|
|
|
// ==================== VERSES ====================
|
|
|
|
func buildVerseURL(book string, division *string, chapter, verse int) string {
|
|
switch book {
|
|
case "bg":
|
|
return fmt.Sprintf("/books/bg/%d/%d", chapter, verse)
|
|
case "sb":
|
|
if division != nil {
|
|
return fmt.Sprintf("/books/sb/%s/%d/%d", *division, chapter, verse)
|
|
}
|
|
return fmt.Sprintf("/books/sb/%d/%d", chapter, verse)
|
|
case "cc":
|
|
if division != nil {
|
|
return fmt.Sprintf("/books/cc/%s/%d/%d", *division, chapter, verse)
|
|
}
|
|
return fmt.Sprintf("/books/cc/%d/%d", chapter, verse)
|
|
case "sb1962":
|
|
return fmt.Sprintf("/books/sb1962/%d/%d", chapter, verse)
|
|
case "iso":
|
|
return fmt.Sprintf("/books/iso/%d", chapter)
|
|
case "noi":
|
|
return fmt.Sprintf("/books/noi/%d", chapter)
|
|
case "bs":
|
|
return fmt.Sprintf("/books/bs/5/%d", verse)
|
|
case "mm":
|
|
return fmt.Sprintf("/books/mm/%d/%d", chapter, verse)
|
|
case "nbs":
|
|
return fmt.Sprintf("/books/nbs/%d/%d", chapter, verse)
|
|
default:
|
|
return fmt.Sprintf("/books/%s/%d/%d", book, chapter, verse)
|
|
}
|
|
}
|
|
|
|
func indexVerses(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *repository.MongoDB) {
|
|
defaultSortingField := "chapter"
|
|
schema := &api.CollectionSchema{
|
|
Name: "verses",
|
|
Fields: []api.Field{
|
|
{Name: "book", Type: "string", Facet: boolPtr(true)},
|
|
{Name: "division", Type: "string", Facet: boolPtr(true), Optional: boolPtr(true)},
|
|
{Name: "chapter", Type: "int32", Facet: boolPtr(true)},
|
|
{Name: "verse", Type: "int32"},
|
|
{Name: "reference", Type: "string"},
|
|
{Name: "book_title", Type: "string"},
|
|
{Name: "translation", Type: "string"},
|
|
{Name: "purport", Type: "string"},
|
|
{Name: "transliteration", Type: "string"},
|
|
{Name: "translation_ascii", Type: "string"},
|
|
{Name: "purport_ascii", Type: "string"},
|
|
{Name: "transliteration_ascii", Type: "string"},
|
|
{Name: "url", Type: "string"},
|
|
},
|
|
DefaultSortingField: &defaultSortingField,
|
|
TokenSeparators: &hyphenSeparators,
|
|
}
|
|
|
|
_, err := tsClient.Collections().Create(context.Background(), schema)
|
|
if err != nil {
|
|
sugar.Fatalf("[verses] Failed to create collection: %v", err)
|
|
}
|
|
sugar.Info("[verses] Collection created")
|
|
|
|
// Read from MongoDB
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
|
defer cancel()
|
|
|
|
coll := mongoDB.Collection("verses")
|
|
cursor, err := coll.Find(ctx, bson.M{})
|
|
if err != nil {
|
|
sugar.Fatalf("[verses] Failed to query MongoDB: %v", err)
|
|
}
|
|
defer cursor.Close(ctx)
|
|
|
|
var docs []interface{}
|
|
for cursor.Next(ctx) {
|
|
var raw bson.M
|
|
if err := cursor.Decode(&raw); err != nil {
|
|
sugar.Warnf("[verses] Decode error: %v", err)
|
|
continue
|
|
}
|
|
|
|
id := objectIDToHex(raw["_id"])
|
|
book, _ := raw["book"].(string)
|
|
var division *string
|
|
if d, ok := raw["division"].(string); ok && d != "" {
|
|
division = &d
|
|
}
|
|
chapter := toInt32(raw["chapter"])
|
|
verse := toInt32(raw["verse"])
|
|
reference, _ := raw["reference"].(string)
|
|
bookTitle, _ := raw["bookTitle"].(string)
|
|
translation := stripHTML(toString(raw["translation"]))
|
|
transliteration := toString(raw["transliteration"])
|
|
|
|
// Join purport paragraphs
|
|
purport := ""
|
|
if paragraphs, ok := raw["purport"].(bson.A); ok {
|
|
strs := bsonAToStrings(paragraphs)
|
|
purport = joinParagraphs(strs)
|
|
}
|
|
|
|
verseURL := buildVerseURL(book, division, int(chapter), int(verse))
|
|
|
|
doc := map[string]interface{}{
|
|
"id": id,
|
|
"book": book,
|
|
"chapter": chapter,
|
|
"verse": verse,
|
|
"reference": reference,
|
|
"book_title": bookTitle,
|
|
"translation": translation,
|
|
"purport": purport,
|
|
"transliteration": transliteration,
|
|
"translation_ascii": stripDiacritics(translation),
|
|
"purport_ascii": stripDiacritics(purport),
|
|
"transliteration_ascii": stripDiacritics(transliteration),
|
|
"url": verseURL,
|
|
}
|
|
if division != nil {
|
|
doc["division"] = *division
|
|
}
|
|
|
|
docs = append(docs, doc)
|
|
}
|
|
|
|
sugar.Infof("[verses] Read %d documents from MongoDB", len(docs))
|
|
importBatch(sugar, tsClient, "verses", docs, 500)
|
|
}
|
|
|
|
// ==================== PROSE ====================
|
|
|
|
func indexProse(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *repository.MongoDB) {
|
|
schema := &api.CollectionSchema{
|
|
Name: "prose",
|
|
Fields: []api.Field{
|
|
{Name: "book", Type: "string", Facet: boolPtr(true)},
|
|
{Name: "title", Type: "string"},
|
|
{Name: "content", Type: "string"},
|
|
{Name: "title_ascii", Type: "string"},
|
|
{Name: "content_ascii", Type: "string"},
|
|
{Name: "type", Type: "string", Facet: boolPtr(true)},
|
|
{Name: "url", Type: "string"},
|
|
},
|
|
TokenSeparators: &hyphenSeparators,
|
|
}
|
|
|
|
_, err := tsClient.Collections().Create(context.Background(), schema)
|
|
if err != nil {
|
|
// An already-existing collection is fine: with --no-drop we resume an
|
|
// upsert into it (the schema is unchanged).
|
|
sugar.Warnf("[prose] Create collection: %v (continuing — assuming it exists)", err)
|
|
} else {
|
|
sugar.Info("[prose] Collection created")
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
|
defer cancel()
|
|
|
|
coll := mongoDB.Collection("prose_chapters")
|
|
cursor, err := coll.Find(ctx, bson.M{})
|
|
if err != nil {
|
|
sugar.Fatalf("[prose] Failed to query MongoDB: %v", err)
|
|
}
|
|
defer cursor.Close(ctx)
|
|
|
|
var docs []interface{}
|
|
for cursor.Next(ctx) {
|
|
var raw bson.M
|
|
if err := cursor.Decode(&raw); err != nil {
|
|
sugar.Warnf("[prose] Decode error: %v", err)
|
|
continue
|
|
}
|
|
|
|
id := objectIDToHex(raw["_id"])
|
|
book, _ := raw["book"].(string)
|
|
title, _ := raw["title"].(string)
|
|
docType, _ := raw["type"].(string)
|
|
slug, _ := raw["slug"].(string)
|
|
volume := toInt32(raw["volume"])
|
|
|
|
content := ""
|
|
if paragraphs, ok := raw["paragraphs"].(bson.A); ok {
|
|
strs := bsonAToStrings(paragraphs)
|
|
content = joinParagraphs(strs)
|
|
}
|
|
|
|
// Build URL: KB uses /books/kb/{volume}/{slug}; chapter summaries map
|
|
// their "<lila>-<n>" slug to the chapter landing page; others use
|
|
// /books/{book}/{slug}.
|
|
var proseURL string
|
|
switch {
|
|
case book == "kb":
|
|
proseURL = fmt.Sprintf("/books/kb/%d/%s", volume, slug)
|
|
case docType == "chapter-summary":
|
|
if parts := strings.SplitN(slug, "-", 2); len(parts) == 2 {
|
|
proseURL = fmt.Sprintf("/books/%s/%s/%s", book, parts[0], parts[1])
|
|
} else {
|
|
proseURL = fmt.Sprintf("/books/%s/%s", book, slug)
|
|
}
|
|
default:
|
|
proseURL = fmt.Sprintf("/books/%s/%s", book, slug)
|
|
}
|
|
|
|
doc := map[string]interface{}{
|
|
"id": id,
|
|
"book": book,
|
|
"title": title,
|
|
"content": content,
|
|
"title_ascii": stripDiacritics(title),
|
|
"content_ascii": stripDiacritics(content),
|
|
"type": docType,
|
|
"url": proseURL,
|
|
}
|
|
docs = append(docs, doc)
|
|
}
|
|
|
|
sugar.Infof("[prose] Read %d documents from MongoDB", len(docs))
|
|
// Small batches keep each import POST modest in size — large multi-MB
|
|
// bodies (front-matter intro segments) are prone to drops on flaky links.
|
|
importBatch(sugar, tsClient, "prose", docs, 10)
|
|
}
|
|
|
|
// ==================== LETTERS ====================
|
|
|
|
func indexLetters(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *repository.MongoDB) {
|
|
schema := &api.CollectionSchema{
|
|
Name: "letters",
|
|
Fields: []api.Field{
|
|
{Name: "recipient_display", Type: "string"},
|
|
{Name: "recipients", Type: "string[]", Facet: boolPtr(true)},
|
|
{Name: "date", Type: "string", Optional: boolPtr(true)},
|
|
{Name: "year", Type: "int32", Facet: boolPtr(true), Optional: boolPtr(true)},
|
|
{Name: "location", Type: "string", Facet: boolPtr(true), Optional: boolPtr(true)},
|
|
{Name: "content", Type: "string"},
|
|
{Name: "content_ascii", Type: "string"},
|
|
{Name: "recipient_display_ascii", Type: "string"},
|
|
{Name: "url", Type: "string"},
|
|
},
|
|
TokenSeparators: &hyphenSeparators,
|
|
}
|
|
|
|
_, err := tsClient.Collections().Create(context.Background(), schema)
|
|
if err != nil {
|
|
sugar.Fatalf("[letters] Failed to create collection: %v", err)
|
|
}
|
|
sugar.Info("[letters] Collection created")
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
|
defer cancel()
|
|
|
|
coll := mongoDB.Collection("letters")
|
|
cursor, err := coll.Find(ctx, bson.M{})
|
|
if err != nil {
|
|
sugar.Fatalf("[letters] Failed to query MongoDB: %v", err)
|
|
}
|
|
defer cursor.Close(ctx)
|
|
|
|
var docs []interface{}
|
|
for cursor.Next(ctx) {
|
|
var raw bson.M
|
|
if err := cursor.Decode(&raw); err != nil {
|
|
sugar.Warnf("[letters] Decode error: %v", err)
|
|
continue
|
|
}
|
|
|
|
id := objectIDToHex(raw["_id"])
|
|
slug, _ := raw["slug"].(string)
|
|
recipientDisplay, _ := raw["recipientDisplay"].(string)
|
|
|
|
recipients := []string{}
|
|
if arr, ok := raw["recipients"].(bson.A); ok {
|
|
recipients = bsonAToStrings(arr)
|
|
}
|
|
|
|
content := ""
|
|
if paragraphs, ok := raw["paragraphs"].(bson.A); ok {
|
|
strs := bsonAToStrings(paragraphs)
|
|
content = joinParagraphs(strs)
|
|
}
|
|
|
|
doc := map[string]interface{}{
|
|
"id": id,
|
|
"recipient_display": recipientDisplay,
|
|
"recipients": recipients,
|
|
"content": content,
|
|
"content_ascii": stripDiacritics(content),
|
|
"recipient_display_ascii": stripDiacritics(recipientDisplay),
|
|
"url": fmt.Sprintf("/letters/%s", slug),
|
|
}
|
|
|
|
if date, ok := raw["date"].(string); ok && date != "" {
|
|
doc["date"] = date
|
|
}
|
|
if year, ok := toOptionalInt32(raw["year"]); ok {
|
|
doc["year"] = year
|
|
}
|
|
if location, ok := raw["location"].(string); ok && location != "" {
|
|
doc["location"] = location
|
|
}
|
|
|
|
docs = append(docs, doc)
|
|
}
|
|
|
|
sugar.Infof("[letters] Read %d documents from MongoDB", len(docs))
|
|
importBatch(sugar, tsClient, "letters", docs, 500)
|
|
}
|
|
|
|
// ==================== TRANSCRIPTIONS ====================
|
|
|
|
func indexTranscriptions(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *repository.MongoDB) {
|
|
schema := &api.CollectionSchema{
|
|
Name: "transcriptions",
|
|
Fields: []api.Field{
|
|
{Name: "name", Type: "string"},
|
|
{Name: "type", Type: "string", Facet: boolPtr(true)},
|
|
{Name: "date", Type: "string", Optional: boolPtr(true)},
|
|
{Name: "year", Type: "int32", Facet: boolPtr(true), Optional: boolPtr(true)},
|
|
{Name: "place", Type: "string", Facet: boolPtr(true), Optional: boolPtr(true)},
|
|
{Name: "reference", Type: "string", Optional: boolPtr(true)},
|
|
{Name: "participants", Type: "string[]", Facet: boolPtr(true)},
|
|
{Name: "content", Type: "string"},
|
|
{Name: "name_ascii", Type: "string"},
|
|
{Name: "content_ascii", Type: "string"},
|
|
{Name: "url", Type: "string"},
|
|
},
|
|
TokenSeparators: &hyphenSeparators,
|
|
}
|
|
|
|
_, err := tsClient.Collections().Create(context.Background(), schema)
|
|
if err != nil {
|
|
sugar.Fatalf("[transcriptions] Failed to create collection: %v", err)
|
|
}
|
|
sugar.Info("[transcriptions] Collection created")
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
|
defer cancel()
|
|
|
|
coll := mongoDB.Collection("transcriptions")
|
|
cursor, err := coll.Find(ctx, bson.M{"isPublished": true})
|
|
if err != nil {
|
|
sugar.Fatalf("[transcriptions] Failed to query MongoDB: %v", err)
|
|
}
|
|
defer cursor.Close(ctx)
|
|
|
|
var docs []interface{}
|
|
for cursor.Next(ctx) {
|
|
var raw bson.M
|
|
if err := cursor.Decode(&raw); err != nil {
|
|
sugar.Warnf("[transcriptions] Decode error: %v", err)
|
|
continue
|
|
}
|
|
|
|
id := objectIDToHex(raw["_id"])
|
|
slug, _ := raw["slug"].(string)
|
|
name, _ := raw["name"].(string)
|
|
docType, _ := raw["type"].(string)
|
|
|
|
participants := []string{}
|
|
if arr, ok := raw["participants"].(bson.A); ok {
|
|
participants = bsonAToStrings(arr)
|
|
}
|
|
|
|
// Build content from content blocks or contentJson
|
|
content := extractTranscriptionContent(raw)
|
|
|
|
doc := map[string]interface{}{
|
|
"id": id,
|
|
"name": name,
|
|
"type": docType,
|
|
"participants": participants,
|
|
"content": content,
|
|
"name_ascii": stripDiacritics(name),
|
|
"content_ascii": stripDiacritics(content),
|
|
"url": fmt.Sprintf("/transcriptions/%s", slug),
|
|
}
|
|
|
|
if date, ok := raw["date"].(string); ok && date != "" {
|
|
doc["date"] = date
|
|
}
|
|
if year, ok := toOptionalInt32(raw["year"]); ok {
|
|
doc["year"] = year
|
|
}
|
|
if place, ok := raw["place"].(string); ok && place != "" {
|
|
doc["place"] = place
|
|
}
|
|
if ref, ok := raw["reference"].(string); ok && ref != "" {
|
|
doc["reference"] = ref
|
|
}
|
|
|
|
docs = append(docs, doc)
|
|
}
|
|
|
|
sugar.Infof("[transcriptions] Read %d documents from MongoDB", len(docs))
|
|
importBatch(sugar, tsClient, "transcriptions", docs, 500)
|
|
}
|
|
|
|
func extractTranscriptionContent(raw bson.M) string {
|
|
// Try content array first
|
|
if contentArr, ok := raw["content"].(bson.A); ok && len(contentArr) > 0 {
|
|
var texts []string
|
|
for _, item := range contentArr {
|
|
if block, ok := item.(bson.M); ok {
|
|
if text, ok := block["text"].(string); ok && text != "" {
|
|
texts = append(texts, stripHTML(text))
|
|
}
|
|
}
|
|
}
|
|
if len(texts) > 0 {
|
|
return strings.Join(texts, " ")
|
|
}
|
|
}
|
|
|
|
// Fallback to contentJson
|
|
if contentJson, ok := raw["contentJson"].(string); ok && contentJson != "" {
|
|
var blocks []struct {
|
|
Text string `json:"text"`
|
|
}
|
|
if err := json.Unmarshal([]byte(contentJson), &blocks); err == nil {
|
|
var texts []string
|
|
for _, b := range blocks {
|
|
if b.Text != "" {
|
|
texts = append(texts, stripHTML(b.Text))
|
|
}
|
|
}
|
|
return strings.Join(texts, " ")
|
|
}
|
|
}
|
|
|
|
return ""
|
|
}
|
|
|
|
// ==================== GLOSSARY ====================
|
|
|
|
func indexGlossary(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *repository.MongoDB) {
|
|
schema := &api.CollectionSchema{
|
|
Name: "glossary",
|
|
Fields: []api.Field{
|
|
{Name: "term", Type: "string"},
|
|
{Name: "definition", Type: "string"},
|
|
{Name: "term_ascii", Type: "string"},
|
|
{Name: "definition_ascii", Type: "string"},
|
|
{Name: "first_letter", Type: "string", Facet: boolPtr(true)},
|
|
{Name: "url", Type: "string"},
|
|
},
|
|
TokenSeparators: &hyphenSeparators,
|
|
}
|
|
|
|
_, err := tsClient.Collections().Create(context.Background(), schema)
|
|
if err != nil {
|
|
sugar.Fatalf("[glossary] Failed to create collection: %v", err)
|
|
}
|
|
sugar.Info("[glossary] Collection created")
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
|
defer cancel()
|
|
|
|
coll := mongoDB.Collection("glossary")
|
|
cursor, err := coll.Find(ctx, bson.M{})
|
|
if err != nil {
|
|
sugar.Fatalf("[glossary] Failed to query MongoDB: %v", err)
|
|
}
|
|
defer cursor.Close(ctx)
|
|
|
|
var docs []interface{}
|
|
for cursor.Next(ctx) {
|
|
var raw bson.M
|
|
if err := cursor.Decode(&raw); err != nil {
|
|
sugar.Warnf("[glossary] Decode error: %v", err)
|
|
continue
|
|
}
|
|
|
|
id := objectIDToHex(raw["_id"])
|
|
term, _ := raw["term"].(string)
|
|
definition := stripHTML(toString(raw["definition"]))
|
|
|
|
firstLetter := getFirstLetter(term)
|
|
encodedTerm := url.QueryEscape(term)
|
|
|
|
doc := map[string]interface{}{
|
|
"id": id,
|
|
"term": term,
|
|
"definition": definition,
|
|
"term_ascii": stripDiacritics(term),
|
|
"definition_ascii": stripDiacritics(definition),
|
|
"first_letter": firstLetter,
|
|
"url": fmt.Sprintf("/glossary/%s?term=%s", strings.ToLower(firstLetter), encodedTerm),
|
|
}
|
|
docs = append(docs, doc)
|
|
}
|
|
|
|
sugar.Infof("[glossary] Read %d documents from MongoDB", len(docs))
|
|
importBatch(sugar, tsClient, "glossary", docs, 500)
|
|
}
|
|
|
|
// ==================== ESSAYS ====================
|
|
|
|
func indexEssays(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *repository.MongoDB) {
|
|
schema := &api.CollectionSchema{
|
|
Name: "essays",
|
|
Fields: []api.Field{
|
|
{Name: "title", Type: "string"},
|
|
{Name: "author", Type: "string"},
|
|
{Name: "year", Type: "int32", Optional: boolPtr(true)},
|
|
{Name: "content", Type: "string"},
|
|
{Name: "title_ascii", Type: "string"},
|
|
{Name: "content_ascii", Type: "string"},
|
|
{Name: "url", Type: "string"},
|
|
},
|
|
TokenSeparators: &hyphenSeparators,
|
|
}
|
|
|
|
_, err := tsClient.Collections().Create(context.Background(), schema)
|
|
if err != nil {
|
|
sugar.Fatalf("[essays] Failed to create collection: %v", err)
|
|
}
|
|
sugar.Info("[essays] Collection created")
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
|
defer cancel()
|
|
|
|
coll := mongoDB.Collection("essays")
|
|
cursor, err := coll.Find(ctx, bson.M{})
|
|
if err != nil {
|
|
sugar.Fatalf("[essays] Failed to query MongoDB: %v", err)
|
|
}
|
|
defer cursor.Close(ctx)
|
|
|
|
var docs []interface{}
|
|
for cursor.Next(ctx) {
|
|
var raw bson.M
|
|
if err := cursor.Decode(&raw); err != nil {
|
|
sugar.Warnf("[essays] Decode error: %v", err)
|
|
continue
|
|
}
|
|
|
|
id := objectIDToHex(raw["_id"])
|
|
slug, _ := raw["slug"].(string)
|
|
title, _ := raw["title"].(string)
|
|
author, _ := raw["author"].(string)
|
|
|
|
content := ""
|
|
if paragraphs, ok := raw["paragraphs"].(bson.A); ok {
|
|
strs := bsonAToStrings(paragraphs)
|
|
content = joinParagraphs(strs)
|
|
}
|
|
|
|
doc := map[string]interface{}{
|
|
"id": id,
|
|
"title": title,
|
|
"author": author,
|
|
"content": content,
|
|
"title_ascii": stripDiacritics(title),
|
|
"content_ascii": stripDiacritics(content),
|
|
"url": fmt.Sprintf("/essays/%s", slug),
|
|
}
|
|
|
|
if year, ok := toOptionalInt32(raw["year"]); ok {
|
|
doc["year"] = year
|
|
}
|
|
|
|
docs = append(docs, doc)
|
|
}
|
|
|
|
sugar.Infof("[essays] Read %d documents from MongoDB", len(docs))
|
|
importBatch(sugar, tsClient, "essays", docs, 500)
|
|
}
|
|
|
|
// ==================== JOURNALS ====================
|
|
|
|
func indexJournals(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *repository.MongoDB) {
|
|
schema := &api.CollectionSchema{
|
|
Name: "journals",
|
|
Fields: []api.Field{
|
|
{Name: "book", Type: "string", Facet: boolPtr(true)},
|
|
{Name: "title", Type: "string"},
|
|
{Name: "content", Type: "string"},
|
|
{Name: "title_ascii", Type: "string"},
|
|
{Name: "content_ascii", Type: "string"},
|
|
{Name: "url", Type: "string"},
|
|
},
|
|
TokenSeparators: &hyphenSeparators,
|
|
}
|
|
|
|
_, err := tsClient.Collections().Create(context.Background(), schema)
|
|
if err != nil {
|
|
sugar.Fatalf("[journals] Failed to create collection: %v", err)
|
|
}
|
|
sugar.Info("[journals] Collection created")
|
|
|
|
// Build book slug mapping from journal_meta
|
|
bookSlugs := buildBookSlugMap(sugar, mongoDB, "journal_meta")
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
|
defer cancel()
|
|
|
|
coll := mongoDB.Collection("journal_chapters")
|
|
cursor, err := coll.Find(ctx, bson.M{})
|
|
if err != nil {
|
|
sugar.Fatalf("[journals] Failed to query MongoDB: %v", err)
|
|
}
|
|
defer cursor.Close(ctx)
|
|
|
|
var docs []interface{}
|
|
for cursor.Next(ctx) {
|
|
var raw bson.M
|
|
if err := cursor.Decode(&raw); err != nil {
|
|
sugar.Warnf("[journals] Decode error: %v", err)
|
|
continue
|
|
}
|
|
|
|
id := objectIDToHex(raw["_id"])
|
|
book, _ := raw["book"].(string)
|
|
title, _ := raw["title"].(string)
|
|
chapterSlug, _ := raw["slug"].(string)
|
|
|
|
content := ""
|
|
if paragraphs, ok := raw["paragraphs"].(bson.A); ok {
|
|
strs := bsonAToStrings(paragraphs)
|
|
content = joinParagraphs(strs)
|
|
}
|
|
|
|
bookSlug := bookSlugs[book]
|
|
if bookSlug == "" {
|
|
bookSlug = book
|
|
}
|
|
|
|
doc := map[string]interface{}{
|
|
"id": id,
|
|
"book": book,
|
|
"title": title,
|
|
"content": content,
|
|
"title_ascii": stripDiacritics(title),
|
|
"content_ascii": stripDiacritics(content),
|
|
"url": fmt.Sprintf("/journals/%s/%s", bookSlug, chapterSlug),
|
|
}
|
|
docs = append(docs, doc)
|
|
}
|
|
|
|
sugar.Infof("[journals] Read %d documents from MongoDB", len(docs))
|
|
importBatch(sugar, tsClient, "journals", docs, 500)
|
|
}
|
|
|
|
// ==================== MAGAZINES ====================
|
|
|
|
func indexMagazines(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *repository.MongoDB) {
|
|
schema := &api.CollectionSchema{
|
|
Name: "magazines",
|
|
Fields: []api.Field{
|
|
{Name: "book", Type: "string", Facet: boolPtr(true)},
|
|
{Name: "title", Type: "string"},
|
|
{Name: "content", Type: "string"},
|
|
{Name: "title_ascii", Type: "string"},
|
|
{Name: "content_ascii", Type: "string"},
|
|
{Name: "url", Type: "string"},
|
|
},
|
|
TokenSeparators: &hyphenSeparators,
|
|
}
|
|
|
|
_, err := tsClient.Collections().Create(context.Background(), schema)
|
|
if err != nil {
|
|
sugar.Fatalf("[magazines] Failed to create collection: %v", err)
|
|
}
|
|
sugar.Info("[magazines] Collection created")
|
|
|
|
// Build book slug mapping from magazine_meta
|
|
bookSlugs := buildBookSlugMap(sugar, mongoDB, "magazine_meta")
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
|
defer cancel()
|
|
|
|
coll := mongoDB.Collection("magazine_issues")
|
|
cursor, err := coll.Find(ctx, bson.M{})
|
|
if err != nil {
|
|
sugar.Fatalf("[magazines] Failed to query MongoDB: %v", err)
|
|
}
|
|
defer cursor.Close(ctx)
|
|
|
|
var docs []interface{}
|
|
for cursor.Next(ctx) {
|
|
var raw bson.M
|
|
if err := cursor.Decode(&raw); err != nil {
|
|
sugar.Warnf("[magazines] Decode error: %v", err)
|
|
continue
|
|
}
|
|
|
|
id := objectIDToHex(raw["_id"])
|
|
book, _ := raw["book"].(string)
|
|
title, _ := raw["title"].(string)
|
|
issueSlug, _ := raw["slug"].(string)
|
|
|
|
content := ""
|
|
if paragraphs, ok := raw["paragraphs"].(bson.A); ok {
|
|
strs := bsonAToStrings(paragraphs)
|
|
content = joinParagraphs(strs)
|
|
}
|
|
|
|
bookSlug := bookSlugs[book]
|
|
if bookSlug == "" {
|
|
bookSlug = book
|
|
}
|
|
|
|
doc := map[string]interface{}{
|
|
"id": id,
|
|
"book": book,
|
|
"title": title,
|
|
"content": content,
|
|
"title_ascii": stripDiacritics(title),
|
|
"content_ascii": stripDiacritics(content),
|
|
"url": fmt.Sprintf("/magazines/%s/%s", bookSlug, issueSlug),
|
|
}
|
|
docs = append(docs, doc)
|
|
}
|
|
|
|
sugar.Infof("[magazines] Read %d documents from MongoDB", len(docs))
|
|
// Smaller batch size: each magazine issue can have very large paragraphs
|
|
importBatch(sugar, tsClient, "magazines", docs, 25)
|
|
}
|
|
|
|
// ==================== OTHER DOCUMENTS ====================
|
|
|
|
func indexOtherDocuments(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *repository.MongoDB) {
|
|
schema := &api.CollectionSchema{
|
|
Name: "other_documents",
|
|
Fields: []api.Field{
|
|
{Name: "book", Type: "string", Facet: boolPtr(true)},
|
|
{Name: "title", Type: "string"},
|
|
{Name: "content", Type: "string"},
|
|
{Name: "title_ascii", Type: "string"},
|
|
{Name: "content_ascii", Type: "string"},
|
|
{Name: "url", Type: "string"},
|
|
},
|
|
TokenSeparators: &hyphenSeparators,
|
|
}
|
|
|
|
_, err := tsClient.Collections().Create(context.Background(), schema)
|
|
if err != nil {
|
|
sugar.Fatalf("[other_documents] Failed to create collection: %v", err)
|
|
}
|
|
sugar.Info("[other_documents] Collection created")
|
|
|
|
// Build book slug mapping from other_document_meta
|
|
bookSlugs := buildBookSlugMap(sugar, mongoDB, "other_document_meta")
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
|
defer cancel()
|
|
|
|
coll := mongoDB.Collection("other_document_chapters")
|
|
cursor, err := coll.Find(ctx, bson.M{})
|
|
if err != nil {
|
|
sugar.Fatalf("[other_documents] Failed to query MongoDB: %v", err)
|
|
}
|
|
defer cursor.Close(ctx)
|
|
|
|
var docs []interface{}
|
|
for cursor.Next(ctx) {
|
|
var raw bson.M
|
|
if err := cursor.Decode(&raw); err != nil {
|
|
sugar.Warnf("[other_documents] Decode error: %v", err)
|
|
continue
|
|
}
|
|
|
|
id := objectIDToHex(raw["_id"])
|
|
book, _ := raw["book"].(string)
|
|
title, _ := raw["title"].(string)
|
|
chapterSlug, _ := raw["slug"].(string)
|
|
|
|
content := ""
|
|
if paragraphs, ok := raw["paragraphs"].(bson.A); ok {
|
|
strs := bsonAToStrings(paragraphs)
|
|
content = joinParagraphs(strs)
|
|
}
|
|
|
|
bookSlug := bookSlugs[book]
|
|
if bookSlug == "" {
|
|
bookSlug = book
|
|
}
|
|
|
|
doc := map[string]interface{}{
|
|
"id": id,
|
|
"book": book,
|
|
"title": title,
|
|
"content": content,
|
|
"title_ascii": stripDiacritics(title),
|
|
"content_ascii": stripDiacritics(content),
|
|
"url": fmt.Sprintf("/other-documents/%s/%s", bookSlug, chapterSlug),
|
|
}
|
|
docs = append(docs, doc)
|
|
}
|
|
|
|
sugar.Infof("[other_documents] Read %d documents from MongoDB", len(docs))
|
|
// Smaller batch size: each chapter can have very large paragraphs
|
|
importBatch(sugar, tsClient, "other_documents", docs, 25)
|
|
}
|
|
|
|
// ==================== HELPERS ====================
|
|
|
|
// buildBookSlugMap reads a meta collection and returns a map of book code -> slug.
|
|
// The slug is typically the same as the book code, lowercased and hyphenated.
|
|
func buildBookSlugMap(sugar *zap.SugaredLogger, mongoDB *repository.MongoDB, metaCollection string) map[string]string {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
|
|
coll := mongoDB.Collection(metaCollection)
|
|
cursor, err := coll.Find(ctx, bson.M{})
|
|
if err != nil {
|
|
sugar.Warnf("Failed to read %s: %v", metaCollection, err)
|
|
return map[string]string{}
|
|
}
|
|
defer cursor.Close(ctx)
|
|
|
|
slugMap := make(map[string]string)
|
|
for cursor.Next(ctx) {
|
|
var raw bson.M
|
|
if err := cursor.Decode(&raw); err != nil {
|
|
continue
|
|
}
|
|
book, _ := raw["book"].(string)
|
|
if book != "" {
|
|
// Use the book code directly as the slug; this matches the API routing pattern
|
|
slugMap[book] = book
|
|
}
|
|
}
|
|
return slugMap
|
|
}
|
|
|
|
// objectIDToHex safely converts a MongoDB ObjectID to hex string.
|
|
func objectIDToHex(v interface{}) string {
|
|
if oid, ok := v.(bson.ObjectID); ok {
|
|
return oid.Hex()
|
|
}
|
|
return fmt.Sprintf("%v", v)
|
|
}
|
|
|
|
// toInt32 converts a BSON numeric value to int32.
|
|
func toInt32(v interface{}) int32 {
|
|
switch n := v.(type) {
|
|
case int32:
|
|
return n
|
|
case int64:
|
|
return int32(n)
|
|
case float64:
|
|
return int32(n)
|
|
case int:
|
|
return int32(n)
|
|
default:
|
|
return 0
|
|
}
|
|
}
|
|
|
|
// toOptionalInt32 tries to convert a BSON value to int32, returning ok=false if nil.
|
|
func toOptionalInt32(v interface{}) (int32, bool) {
|
|
if v == nil {
|
|
return 0, false
|
|
}
|
|
switch n := v.(type) {
|
|
case int32:
|
|
return n, true
|
|
case int64:
|
|
return int32(n), true
|
|
case float64:
|
|
return int32(n), true
|
|
case int:
|
|
return int32(n), true
|
|
default:
|
|
return 0, false
|
|
}
|
|
}
|
|
|
|
// toString safely converts an interface to string.
|
|
func toString(v interface{}) string {
|
|
if s, ok := v.(string); ok {
|
|
return s
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// bsonAToStrings converts a bson.A to a []string.
|
|
func bsonAToStrings(arr bson.A) []string {
|
|
result := make([]string, 0, len(arr))
|
|
for _, v := range arr {
|
|
if s, ok := v.(string); ok {
|
|
result = append(result, s)
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
// getFirstLetter returns the base letter for glossary grouping.
|
|
// Maps diacritical characters to their ASCII base letter.
|
|
func getFirstLetter(term string) string {
|
|
if term == "" {
|
|
return "?"
|
|
}
|
|
first := []rune(term)[0]
|
|
diacriticToBase := map[rune]rune{
|
|
'Ā': 'A', 'Ä': 'A',
|
|
'Ī': 'I', 'É': 'I',
|
|
'Ū': 'U', 'Ü': 'U',
|
|
'Ṛ': 'R', 'Ṝ': 'R',
|
|
'Ś': 'S', 'Ç': 'S', 'Ṣ': 'S',
|
|
'Ñ': 'N', 'Ṇ': 'N', 'Ṅ': 'N',
|
|
'Ṭ': 'T',
|
|
'Ḍ': 'D',
|
|
'Ḥ': 'H',
|
|
'ā': 'A', 'ä': 'A',
|
|
'ī': 'I', 'é': 'I',
|
|
'ū': 'U', 'ü': 'U',
|
|
'ṛ': 'R', 'ṝ': 'R',
|
|
'ś': 'S', 'ç': 'S', 'ṣ': 'S',
|
|
'ñ': 'N', 'ṇ': 'N', 'ṅ': 'N',
|
|
'ṭ': 'T',
|
|
'ḍ': 'D',
|
|
'ḥ': 'H',
|
|
}
|
|
if base, ok := diacriticToBase[first]; ok {
|
|
return string(base)
|
|
}
|
|
return strings.ToUpper(string(first))
|
|
}
|