From 76a144e1ea72718972992761f7b4bc4a67b9ae36 Mon Sep 17 00:00:00 2001 From: Krispire Admin Date: Sun, 10 May 2026 20:41:40 +0530 Subject: [PATCH] search added --- PROJECT_CONTEXT.md | 116 +++- cmd/indexer/main.go | 1113 ++++++++++++++++++++++++++++++ go.mod | 5 + go.sum | 25 +- internal/config/config.go | 10 +- internal/handler/search.go | 512 ++++++++++++++ internal/repository/typesense.go | 19 + internal/server/server.go | 11 +- 8 files changed, 1787 insertions(+), 24 deletions(-) create mode 100644 cmd/indexer/main.go create mode 100644 internal/handler/search.go create mode 100644 internal/repository/typesense.go diff --git a/PROJECT_CONTEXT.md b/PROJECT_CONTEXT.md index c0b0b93..96b57a0 100644 --- a/PROJECT_CONTEXT.md +++ b/PROJECT_CONTEXT.md @@ -2,7 +2,7 @@ ## What this project is -A Go-based Backend for Frontend (BFF) API server for [prabhupadasays.org](https://prabhupadasays.org). It is the **sole data provider** for the production Next.js frontend (and future mobile apps). The knowledgebase frontend has removed all direct Prisma, MongoDB, and S3 dependencies — everything flows through this BFF. It aggregates content from MongoDB, user data from PostgreSQL, search from Meilisearch, caching from DragonflyDB, and media from S3 into a unified API surface. +A Go-based Backend for Frontend (BFF) API server for [prabhupadasays.org](https://prabhupadasays.org). It is the **sole data provider** for the production Next.js frontend (and future mobile apps). The knowledgebase frontend has removed all direct Prisma, MongoDB, and S3 dependencies — everything flows through this BFF. It aggregates content from MongoDB, user data from PostgreSQL, search from Typesense, caching from DragonflyDB, and media from S3 into a unified API surface. --- @@ -13,7 +13,7 @@ A Go-based Backend for Frontend (BFF) API server for [prabhupadasays.org](https: - **Zap** — structured logging (uber-go/zap) - **MongoDB** — content database (books, verses, letters, essays, glossary, disciples, transcriptions) - **PostgreSQL** — relational data (users, bookmarks, collections, mentions, URL shortener) -- **Meilisearch** — typo-tolerant full-text search across all content +- **Typesense** — typo-tolerant full-text search engine (https://typesense.org) - **DragonflyDB** — Redis-compatible cache (API responses, rate limiting, sessions) - **AWS S3** — media storage (avatars, images, audio files) - **FCM + APNs** — push notifications for Android and iOS @@ -37,7 +37,7 @@ A Go-based Backend for Frontend (BFF) API server for [prabhupadasays.org](https: └──────────────┘ │ │ │ │ ┌─────────────────────┼──────────┐ │ │ ▼ ▼ ▼ ▼ ▼ │ - │ Mongo Postgres Meili S3 Dragonfly│ + │ Mongo Postgres Typesense S3 Dragonfly│ │ (content)(users, (search)(media)(cache)│ │ mentions, │ │ URL short, │ @@ -52,15 +52,20 @@ A Go-based Backend for Frontend (BFF) API server for [prabhupadasays.org](https: ``` spsays-bff/ ├── cmd/ -│ └── api/ -│ └── main.go → Entry point (config, logger, server, graceful shutdown) +│ ├── api/ +│ │ └── main.go → Entry point (config, logger, server, graceful shutdown) +│ └── indexer/ +│ └── main.go → CLI tool for indexing MongoDB content into Typesense +│ • --full drop all collections and reindex +│ • --only=verses,letters index specific collections only ├── internal/ │ ├── config/ -│ │ └── config.go → Environment config (all services: Mongo, Postgres, Meili, S3, etc.) +│ │ └── config.go → Environment config (all services: Mongo, Postgres, Typesense, S3, etc.) │ ├── server/ │ │ └── server.go → Chi router setup, global middleware, route registration │ ├── handler/ → HTTP handlers (one file per domain) │ │ ├── health.go → Health check endpoint +│ │ ├── search.go → Federated search handler (multi-collection, typo-tolerant) │ │ ├── book.go → Book routes (SB, CC, KB, BG, generic verse & prose) │ │ ├── letter.go → Letter routes (by year, recipient, location) │ │ ├── essay.go → Essay routes @@ -71,7 +76,7 @@ spsays-bff/ │ │ ├── other_document.go → Other-document routes │ │ └── transcription.go → Transcription routes (by year, place, type, slug) │ ├── service/ → Business logic layer (orchestrates repositories) -│ ├── repository/ → Data access layer (Mongo, Postgres, Meili, S3, Dragonfly) +│ ├── repository/ → Data access layer (Mongo, Postgres, Typesense, S3, Dragonfly) │ │ ├── mongo.go → MongoDB connection + database handle │ │ ├── book_repo.go → Book / verse queries │ │ ├── letter_repo.go → Letter queries @@ -81,7 +86,8 @@ spsays-bff/ │ │ ├── journal_repo.go → Journal queries │ │ ├── magazine_repo.go → Magazine queries │ │ ├── other_document_repo.go → Other-document queries -│ │ └── transcription_repo.go → Transcription queries (with contentJson fallback) +│ │ ├── transcription_repo.go → Transcription queries (with contentJson fallback) +│ │ └── typesense.go → Typesense client wrapper (search, indexing helpers) │ ├── middleware/ │ │ ├── logger.go → Zap-based request logging (method, path, status, duration) │ │ ├── auth.go → JWT Bearer token authentication + context user extraction @@ -227,6 +233,46 @@ All routes are prefixed with `/api/v1`. | `GET` | `/api/v1/transcriptions/adjacent/{slug}` | Prev/next within same year (`?year=N`) | | `GET` | `/api/v1/transcriptions/{slug}` | Single transcription (full content + audio URL) | +### Search + +| Method | Route | Description | +|--------|-------|-------------| +| `GET` | `/api/v1/search?q=...&type=...&page=1&limit=20` | Federated typo-tolerant search across all content | + +**Query params:** +- `q` — search query (required) +- `type` — content type filter, comma-separated. Valid values: + - `all` (default) — search all 9 collections + - `verses`, `prose`, `letters`, `transcriptions`, `glossary`, `essays`, `journals`, `magazines`, `other_documents` + - Multiple: `verses,letters,transcriptions` +- `page` — page number (default 1) +- `limit` — results per page (default 20, max 100) + +**Response:** +```json +{ + "success": true, + "data": { + "query": "surrender unto", + "type": "all", + "results": [ + { + "collection": "verses", + "id": "...", + "title": "BG 18.66", + "snippet": "...just surrender unto Me...", + "url": "/books/bg/18/66", + "meta": { "book": "bg", "reference": "BG 18.66" } + } + ], + "total": 150, + "page": 1, + "limit": 20, + "total_pages": 8 + } +} +``` + ### Authenticated (JWT required) — planned | Method | Route | Description | @@ -247,6 +293,49 @@ All routes are prefixed with `/api/v1`. --- +## Typesense Indexing + +The BFF includes a CLI tool (`cmd/indexer/main.go`) for managing Typesense indexes. + +### Collections (9 total, ~35,500 documents) + +| Collection | MongoDB Source | Documents | Searchable Fields | +|---|---|---|---| +| `verses` | `verses` | 21,127 | translation, purport, transliteration, reference (+ ASCII variants) | +| `prose` | `prose_chapters` | 627 | title, content (+ ASCII variants) | +| `letters` | `letters` | 6,866 | content, recipient_display (+ ASCII variants) | +| `transcriptions` | `transcriptions` | 3,787 | content, name, reference (+ ASCII variants) | +| `glossary` | `glossary` | 2,749 | term, definition (+ ASCII variants) | +| `essays` | `essays` | 23 | title, content (+ ASCII variants) | +| `journals` | `journal_chapters` | 300 | title, content (+ ASCII variants) | +| `magazines` | `magazine_issues` | 59 | title, content (+ ASCII variants) | +| `other_documents` | `other_document_chapters` | 28 | title, content (+ ASCII variants) | + +### ASCII Shadow Fields + +For diacritic-insensitive search, every searchable text field has a `_ascii` shadow field with all IAST diacritics stripped (`ā→a`, `ṛ→r`, `ṣ→s`, `ṭ→t`, `ñ→n`, etc.). This allows queries like "Krsna" to match "Kṛṣṇa", "Yashodanandana" to match "Yaśodanandana". + +### Search Configuration + +- **`prefix=true`** — partial word matching (e.g., "sarv" matches "sarva") +- **`num_typos=2`** — up to 2 character substitutions allowed per word +- **HTML stripping** — all paragraph content has HTML tags removed before indexing + +### CLI Usage + +```/dev/null/shell.sh#L1-12 +# Full reindex (drops all collections and rebuilds) +go run cmd/indexer/main.go --full + +# Index specific collections only (drops + recreates them) +go run cmd/indexer/main.go --only=verses,letters + +# Index all, but skip collections that already exist +go run cmd/indexer/main.go +``` + +--- + ## Data Models ### Content Models (MongoDB) @@ -333,9 +422,9 @@ MONGO_DB=spsays # PostgreSQL (users, relations) POSTGRES_URI=postgres://... -# Meilisearch -MEILISEARCH_HOST=http://localhost:7700 -MEILISEARCH_API_KEY= +# Typesense +TYPESENSE_URL=https://search.prabhupadasays.org +TYPESENSE_API_KEY= # DragonflyDB / Redis REDIS_URI=redis://localhost:6379 @@ -402,7 +491,7 @@ make tidy Migration status (Prisma → BFF): 1. ✅ Done — BFF API routes matching all former Prisma queries 2. ✅ Done — Frontend migrated from Prisma to BFF API calls (Prisma, MongoDB driver, and S3 SDK removed from knowledgebase) -3. Add Meilisearch indexing + search endpoints +3. ✅ Done — Typesense indexing + federated search endpoints (`cmd/indexer/main.go`, `/api/v1/search`) 4. Add PostgreSQL for user features (auth, bookmarks, collections, mentions) 5. Add mobile app support (same API, push notifications) @@ -417,4 +506,5 @@ Migration status (Prisma → BFF): - **Service layer** — business logic between handlers and repositories (orchestration, validation, caching) - **Handler layer** — thin HTTP handlers that parse requests, call services, return responses - **No ORM** — use native drivers (`mongo-go-driver`, `pgx`) for full control -- **Explicit book routing** — SB, CC, and KB have dedicated route groups to avoid URL ambiguity with generic `{code}/{chapter}/{verse}` patterns \ No newline at end of file +- **Explicit book routing** — SB, CC, and KB have dedicated route groups to avoid URL ambiguity with generic `{code}/{chapter}/{verse}` patterns +- **Search gateway** — search is provided by Typesense, indexed from MongoDB via the indexer CLI; the BFF acts as the search gateway with multi-collection federated queries. \ No newline at end of file diff --git a/cmd/indexer/main.go b/cmd/indexer/main.go new file mode 100644 index 0000000..ff5ec8d --- /dev/null +++ b/cmd/indexer/main.go @@ -0,0 +1,1113 @@ +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "net/url" + "os" + "regexp" + "strings" + "time" + + "github.com/prabhupadasays/spsays-bff/internal/config" + "github.com/prabhupadasays/spsays-bff/internal/repository" + "github.com/typesense/typesense-go/v2/typesense" + "github.com/typesense/typesense-go/v2/typesense/api" + "go.mongodb.org/mongo-driver/v2/bson" + "go.uber.org/zap" +) + +// CLI flags +var ( + fullReindex = flag.Bool("full", false, "Drop all collections and reindex everything") + onlyFlag = flag.String("only", "", "Comma-separated list of collections to index (e.g., verses,letters)") +) + +// All supported collection names +var allCollections = []string{ + "verses", "prose", "letters", "transcriptions", "glossary", + "essays", "journals", "magazines", "other_documents", +} + +// HTML stripping helper +var htmlTagRe = regexp.MustCompile(`<[^>]*>`) + +func stripHTML(s string) string { + return htmlTagRe.ReplaceAllString(s, "") +} + +func joinParagraphs(paragraphs []string) string { + var sb strings.Builder + for i, p := range paragraphs { + if i > 0 { + sb.WriteString(" ") + } + sb.WriteString(stripHTML(p)) + } + return sb.String() +} + +// IAST diacritical → ASCII mapping for typo-tolerant search. +// Allows "Krsna" to match "Kṛṣṇa", "Prabhupada" to match "Prabhupāda", etc. +var diacriticReplacer = strings.NewReplacer( + "ā", "a", "Ā", "A", + "ī", "i", "Ī", "I", + "ū", "u", "Ū", "U", + "ṛ", "r", "Ṛ", "R", + "ṝ", "r", "Ṝ", "R", + "ṣ", "s", "Ṣ", "S", + "ś", "s", "Ś", "S", + "ṭ", "t", "Ṭ", "T", + "ḍ", "d", "Ḍ", "D", + "ṇ", "n", "Ṇ", "N", + "ñ", "n", "Ñ", "N", + "ṅ", "n", "Ṅ", "N", + "ṁ", "m", "Ṁ", "M", + "ḥ", "h", "Ḥ", "H", + "ḷ", "l", "Ḷ", "L", + "é", "e", "É", "E", + "è", "e", + "ö", "o", + "ü", "u", + "ä", "a", + // Common ligatures + "æ", "ae", "œ", "oe", +) + +// stripDiacritics converts IAST/Unicode diacritical characters to plain ASCII. +func stripDiacritics(s string) string { + return diacriticReplacer.Replace(s) +} + +// Pointer helpers +func stringPtr(s string) *string { return &s } +func boolPtr(b bool) *bool { return &b } + +func main() { + flag.Parse() + + // Load config + cfg, err := config.Load() + if err != nil { + fmt.Fprintf(os.Stderr, "failed to load config: %v\n", err) + os.Exit(1) + } + + // Initialize logger + logger, _ := zap.NewDevelopment() + defer logger.Sync() + sugar := logger.Sugar() + + // Connect to MongoDB + mongoDB, err := repository.NewMongoDB(cfg.MongoURI, cfg.MongoDB, logger) + if err != nil { + sugar.Fatalf("failed to connect to MongoDB: %v", err) + } + defer mongoDB.Close() + + // Create Typesense client + tsClient := typesense.NewClient( + typesense.WithServer(cfg.TypesenseURL), + typesense.WithAPIKey(cfg.TypesenseAPIKey), + ) + + // Determine which collections to index + collectionsToIndex := allCollections + if *onlyFlag != "" { + collectionsToIndex = strings.Split(*onlyFlag, ",") + for i := range collectionsToIndex { + collectionsToIndex[i] = strings.TrimSpace(collectionsToIndex[i]) + } + } + + // Get existing collections + existingCollections := make(map[string]bool) + existing, err := tsClient.Collections().Retrieve(context.Background()) + if err != nil { + sugar.Warnf("could not retrieve existing collections: %v", err) + } else { + for _, col := range existing { + existingCollections[col.Name] = true + } + } + + for _, name := range collectionsToIndex { + sugar.Infof("=== Processing collection: %s ===", name) + + shouldDrop := *fullReindex || *onlyFlag != "" + if !shouldDrop && existingCollections[name] { + sugar.Infof("[%s] Collection already exists, skipping (use --full to force reindex)", name) + continue + } + + // Drop if exists and we need to recreate + if existingCollections[name] && shouldDrop { + sugar.Infof("[%s] Dropping existing collection...", name) + _, err := tsClient.Collection(name).Delete(context.Background()) + if err != nil { + sugar.Warnf("[%s] Failed to drop collection: %v", name, err) + } + } + + // Create schema and index + switch name { + case "verses": + indexVerses(sugar, tsClient, mongoDB) + case "prose": + indexProse(sugar, tsClient, mongoDB) + case "letters": + indexLetters(sugar, tsClient, mongoDB) + case "transcriptions": + indexTranscriptions(sugar, tsClient, mongoDB) + case "glossary": + indexGlossary(sugar, tsClient, mongoDB) + case "essays": + indexEssays(sugar, tsClient, mongoDB) + case "journals": + indexJournals(sugar, tsClient, mongoDB) + case "magazines": + indexMagazines(sugar, tsClient, mongoDB) + case "other_documents": + indexOtherDocuments(sugar, tsClient, mongoDB) + default: + sugar.Warnf("Unknown collection: %s", name) + } + } + + sugar.Info("=== Indexing complete ===") +} + +// importBatch imports documents into Typesense in batches of batchSize. +func importBatch(sugar *zap.SugaredLogger, tsClient *typesense.Client, collectionName string, docs []interface{}, batchSize int) { + total := len(docs) + action := "create" + for i := 0; i < total; i += batchSize { + end := i + batchSize + if end > total { + end = total + } + batch := docs[i:end] + responses, err := tsClient.Collection(collectionName).Documents().Import(context.Background(), batch, &api.ImportDocumentsParams{ + Action: &action, + }) + if err != nil { + sugar.Errorf("[%s] Import error at batch %d-%d: %v", collectionName, i, end, err) + continue + } + // Check for per-document errors + errCount := 0 + for _, resp := range responses { + if !resp.Success { + errCount++ + if errCount <= 3 { + sugar.Warnf("[%s] Document import error: %s", collectionName, resp.Error) + } + } + } + if errCount > 3 { + sugar.Warnf("[%s] ... and %d more document errors in this batch", collectionName, errCount-3) + } + sugar.Infof("[%s] %d/%d indexed...", collectionName, end, total) + } +} + +// ==================== VERSES ==================== + +func buildVerseURL(book string, division *string, chapter, verse int) string { + switch book { + case "bg": + return fmt.Sprintf("/books/bg/%d/%d", chapter, verse) + case "sb": + if division != nil { + return fmt.Sprintf("/books/sb/%s/%d/%d", *division, chapter, verse) + } + return fmt.Sprintf("/books/sb/%d/%d", chapter, verse) + case "cc": + if division != nil { + return fmt.Sprintf("/books/cc/%s/%d/%d", *division, chapter, verse) + } + return fmt.Sprintf("/books/cc/%d/%d", chapter, verse) + case "sb1962": + return fmt.Sprintf("/books/sb1962/%d/%d", chapter, verse) + case "iso": + return fmt.Sprintf("/books/iso/%d", chapter) + case "noi": + return fmt.Sprintf("/books/noi/%d", chapter) + case "bs": + return fmt.Sprintf("/books/bs/5/%d", verse) + case "mm": + return fmt.Sprintf("/books/mm/%d/%d", chapter, verse) + case "nbs": + return fmt.Sprintf("/books/nbs/%d/%d", chapter, verse) + default: + return fmt.Sprintf("/books/%s/%d/%d", book, chapter, verse) + } +} + +func indexVerses(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *repository.MongoDB) { + defaultSortingField := "chapter" + // Treat hyphens as token separators so "sarva-dharman" indexes as + // two tokens ("sarva" and "dharman"), making queries like "sarva dharman" matchable. + tokenSeparators := []string{"-"} + schema := &api.CollectionSchema{ + Name: "verses", + Fields: []api.Field{ + {Name: "book", Type: "string", Facet: boolPtr(true)}, + {Name: "division", Type: "string", Facet: boolPtr(true), Optional: boolPtr(true)}, + {Name: "chapter", Type: "int32", Facet: boolPtr(true)}, + {Name: "verse", Type: "int32"}, + {Name: "reference", Type: "string"}, + {Name: "book_title", Type: "string"}, + {Name: "translation", Type: "string"}, + {Name: "purport", Type: "string"}, + {Name: "transliteration", Type: "string"}, + {Name: "translation_ascii", Type: "string"}, + {Name: "purport_ascii", Type: "string"}, + {Name: "transliteration_ascii", Type: "string"}, + {Name: "url", Type: "string"}, + }, + DefaultSortingField: &defaultSortingField, + TokenSeparators: &tokenSeparators, + } + + _, err := tsClient.Collections().Create(context.Background(), schema) + if err != nil { + sugar.Fatalf("[verses] Failed to create collection: %v", err) + } + sugar.Info("[verses] Collection created") + + // Read from MongoDB + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + coll := mongoDB.Collection("verses") + cursor, err := coll.Find(ctx, bson.M{}) + if err != nil { + sugar.Fatalf("[verses] Failed to query MongoDB: %v", err) + } + defer cursor.Close(ctx) + + var docs []interface{} + for cursor.Next(ctx) { + var raw bson.M + if err := cursor.Decode(&raw); err != nil { + sugar.Warnf("[verses] Decode error: %v", err) + continue + } + + id := objectIDToHex(raw["_id"]) + book, _ := raw["book"].(string) + var division *string + if d, ok := raw["division"].(string); ok && d != "" { + division = &d + } + chapter := toInt32(raw["chapter"]) + verse := toInt32(raw["verse"]) + reference, _ := raw["reference"].(string) + bookTitle, _ := raw["bookTitle"].(string) + translation := stripHTML(toString(raw["translation"])) + transliteration := toString(raw["transliteration"]) + + // Join purport paragraphs + purport := "" + if paragraphs, ok := raw["purport"].(bson.A); ok { + strs := bsonAToStrings(paragraphs) + purport = joinParagraphs(strs) + } + + verseURL := buildVerseURL(book, division, int(chapter), int(verse)) + + doc := map[string]interface{}{ + "id": id, + "book": book, + "chapter": chapter, + "verse": verse, + "reference": reference, + "book_title": bookTitle, + "translation": translation, + "purport": purport, + "transliteration": transliteration, + "translation_ascii": stripDiacritics(translation), + "purport_ascii": stripDiacritics(purport), + "transliteration_ascii": stripDiacritics(transliteration), + "url": verseURL, + } + if division != nil { + doc["division"] = *division + } + + docs = append(docs, doc) + } + + sugar.Infof("[verses] Read %d documents from MongoDB", len(docs)) + importBatch(sugar, tsClient, "verses", docs, 500) +} + +// ==================== PROSE ==================== + +func indexProse(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *repository.MongoDB) { + schema := &api.CollectionSchema{ + Name: "prose", + Fields: []api.Field{ + {Name: "book", Type: "string", Facet: boolPtr(true)}, + {Name: "title", Type: "string"}, + {Name: "content", Type: "string"}, + {Name: "title_ascii", Type: "string"}, + {Name: "content_ascii", Type: "string"}, + {Name: "type", Type: "string", Facet: boolPtr(true)}, + {Name: "url", Type: "string"}, + }, + } + + _, err := tsClient.Collections().Create(context.Background(), schema) + if err != nil { + sugar.Fatalf("[prose] Failed to create collection: %v", err) + } + sugar.Info("[prose] Collection created") + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + coll := mongoDB.Collection("prose_chapters") + cursor, err := coll.Find(ctx, bson.M{}) + if err != nil { + sugar.Fatalf("[prose] Failed to query MongoDB: %v", err) + } + defer cursor.Close(ctx) + + var docs []interface{} + for cursor.Next(ctx) { + var raw bson.M + if err := cursor.Decode(&raw); err != nil { + sugar.Warnf("[prose] Decode error: %v", err) + continue + } + + id := objectIDToHex(raw["_id"]) + book, _ := raw["book"].(string) + title, _ := raw["title"].(string) + docType, _ := raw["type"].(string) + slug, _ := raw["slug"].(string) + volume := toInt32(raw["volume"]) + + content := "" + if paragraphs, ok := raw["paragraphs"].(bson.A); ok { + strs := bsonAToStrings(paragraphs) + content = joinParagraphs(strs) + } + + // Build URL: KB uses /books/kb/{volume}/{slug}, others use /books/{book}/{slug} + var proseURL string + if book == "kb" { + proseURL = fmt.Sprintf("/books/kb/%d/%s", volume, slug) + } else { + proseURL = fmt.Sprintf("/books/%s/%s", book, slug) + } + + doc := map[string]interface{}{ + "id": id, + "book": book, + "title": title, + "content": content, + "title_ascii": stripDiacritics(title), + "content_ascii": stripDiacritics(content), + "type": docType, + "url": proseURL, + } + docs = append(docs, doc) + } + + sugar.Infof("[prose] Read %d documents from MongoDB", len(docs)) + importBatch(sugar, tsClient, "prose", docs, 500) +} + +// ==================== LETTERS ==================== + +func indexLetters(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *repository.MongoDB) { + schema := &api.CollectionSchema{ + Name: "letters", + Fields: []api.Field{ + {Name: "recipient_display", Type: "string"}, + {Name: "recipients", Type: "string[]", Facet: boolPtr(true)}, + {Name: "date", Type: "string", Optional: boolPtr(true)}, + {Name: "year", Type: "int32", Facet: boolPtr(true), Optional: boolPtr(true)}, + {Name: "location", Type: "string", Facet: boolPtr(true), Optional: boolPtr(true)}, + {Name: "content", Type: "string"}, + {Name: "content_ascii", Type: "string"}, + {Name: "recipient_display_ascii", Type: "string"}, + {Name: "url", Type: "string"}, + }, + } + + _, err := tsClient.Collections().Create(context.Background(), schema) + if err != nil { + sugar.Fatalf("[letters] Failed to create collection: %v", err) + } + sugar.Info("[letters] Collection created") + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + coll := mongoDB.Collection("letters") + cursor, err := coll.Find(ctx, bson.M{}) + if err != nil { + sugar.Fatalf("[letters] Failed to query MongoDB: %v", err) + } + defer cursor.Close(ctx) + + var docs []interface{} + for cursor.Next(ctx) { + var raw bson.M + if err := cursor.Decode(&raw); err != nil { + sugar.Warnf("[letters] Decode error: %v", err) + continue + } + + id := objectIDToHex(raw["_id"]) + slug, _ := raw["slug"].(string) + recipientDisplay, _ := raw["recipientDisplay"].(string) + + recipients := []string{} + if arr, ok := raw["recipients"].(bson.A); ok { + recipients = bsonAToStrings(arr) + } + + content := "" + if paragraphs, ok := raw["paragraphs"].(bson.A); ok { + strs := bsonAToStrings(paragraphs) + content = joinParagraphs(strs) + } + + doc := map[string]interface{}{ + "id": id, + "recipient_display": recipientDisplay, + "recipients": recipients, + "content": content, + "content_ascii": stripDiacritics(content), + "recipient_display_ascii": stripDiacritics(recipientDisplay), + "url": fmt.Sprintf("/letters/%s", slug), + } + + if date, ok := raw["date"].(string); ok && date != "" { + doc["date"] = date + } + if year, ok := toOptionalInt32(raw["year"]); ok { + doc["year"] = year + } + if location, ok := raw["location"].(string); ok && location != "" { + doc["location"] = location + } + + docs = append(docs, doc) + } + + sugar.Infof("[letters] Read %d documents from MongoDB", len(docs)) + importBatch(sugar, tsClient, "letters", docs, 500) +} + +// ==================== TRANSCRIPTIONS ==================== + +func indexTranscriptions(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *repository.MongoDB) { + schema := &api.CollectionSchema{ + Name: "transcriptions", + Fields: []api.Field{ + {Name: "name", Type: "string"}, + {Name: "type", Type: "string", Facet: boolPtr(true)}, + {Name: "date", Type: "string", Optional: boolPtr(true)}, + {Name: "year", Type: "int32", Facet: boolPtr(true), Optional: boolPtr(true)}, + {Name: "place", Type: "string", Facet: boolPtr(true), Optional: boolPtr(true)}, + {Name: "reference", Type: "string", Optional: boolPtr(true)}, + {Name: "participants", Type: "string[]", Facet: boolPtr(true)}, + {Name: "content", Type: "string"}, + {Name: "name_ascii", Type: "string"}, + {Name: "content_ascii", Type: "string"}, + {Name: "url", Type: "string"}, + }, + } + + _, err := tsClient.Collections().Create(context.Background(), schema) + if err != nil { + sugar.Fatalf("[transcriptions] Failed to create collection: %v", err) + } + sugar.Info("[transcriptions] Collection created") + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + coll := mongoDB.Collection("transcriptions") + cursor, err := coll.Find(ctx, bson.M{"isPublished": true}) + if err != nil { + sugar.Fatalf("[transcriptions] Failed to query MongoDB: %v", err) + } + defer cursor.Close(ctx) + + var docs []interface{} + for cursor.Next(ctx) { + var raw bson.M + if err := cursor.Decode(&raw); err != nil { + sugar.Warnf("[transcriptions] Decode error: %v", err) + continue + } + + id := objectIDToHex(raw["_id"]) + slug, _ := raw["slug"].(string) + name, _ := raw["name"].(string) + docType, _ := raw["type"].(string) + + participants := []string{} + if arr, ok := raw["participants"].(bson.A); ok { + participants = bsonAToStrings(arr) + } + + // Build content from content blocks or contentJson + content := extractTranscriptionContent(raw) + + doc := map[string]interface{}{ + "id": id, + "name": name, + "type": docType, + "participants": participants, + "content": content, + "name_ascii": stripDiacritics(name), + "content_ascii": stripDiacritics(content), + "url": fmt.Sprintf("/transcriptions/%s", slug), + } + + if date, ok := raw["date"].(string); ok && date != "" { + doc["date"] = date + } + if year, ok := toOptionalInt32(raw["year"]); ok { + doc["year"] = year + } + if place, ok := raw["place"].(string); ok && place != "" { + doc["place"] = place + } + if ref, ok := raw["reference"].(string); ok && ref != "" { + doc["reference"] = ref + } + + docs = append(docs, doc) + } + + sugar.Infof("[transcriptions] Read %d documents from MongoDB", len(docs)) + importBatch(sugar, tsClient, "transcriptions", docs, 500) +} + +func extractTranscriptionContent(raw bson.M) string { + // Try content array first + if contentArr, ok := raw["content"].(bson.A); ok && len(contentArr) > 0 { + var texts []string + for _, item := range contentArr { + if block, ok := item.(bson.M); ok { + if text, ok := block["text"].(string); ok && text != "" { + texts = append(texts, stripHTML(text)) + } + } + } + if len(texts) > 0 { + return strings.Join(texts, " ") + } + } + + // Fallback to contentJson + if contentJson, ok := raw["contentJson"].(string); ok && contentJson != "" { + var blocks []struct { + Text string `json:"text"` + } + if err := json.Unmarshal([]byte(contentJson), &blocks); err == nil { + var texts []string + for _, b := range blocks { + if b.Text != "" { + texts = append(texts, stripHTML(b.Text)) + } + } + return strings.Join(texts, " ") + } + } + + return "" +} + +// ==================== GLOSSARY ==================== + +func indexGlossary(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *repository.MongoDB) { + schema := &api.CollectionSchema{ + Name: "glossary", + Fields: []api.Field{ + {Name: "term", Type: "string"}, + {Name: "definition", Type: "string"}, + {Name: "term_ascii", Type: "string"}, + {Name: "definition_ascii", Type: "string"}, + {Name: "first_letter", Type: "string", Facet: boolPtr(true)}, + {Name: "url", Type: "string"}, + }, + } + + _, err := tsClient.Collections().Create(context.Background(), schema) + if err != nil { + sugar.Fatalf("[glossary] Failed to create collection: %v", err) + } + sugar.Info("[glossary] Collection created") + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + coll := mongoDB.Collection("glossary") + cursor, err := coll.Find(ctx, bson.M{}) + if err != nil { + sugar.Fatalf("[glossary] Failed to query MongoDB: %v", err) + } + defer cursor.Close(ctx) + + var docs []interface{} + for cursor.Next(ctx) { + var raw bson.M + if err := cursor.Decode(&raw); err != nil { + sugar.Warnf("[glossary] Decode error: %v", err) + continue + } + + id := objectIDToHex(raw["_id"]) + term, _ := raw["term"].(string) + definition := stripHTML(toString(raw["definition"])) + + firstLetter := getFirstLetter(term) + encodedTerm := url.QueryEscape(term) + + doc := map[string]interface{}{ + "id": id, + "term": term, + "definition": definition, + "term_ascii": stripDiacritics(term), + "definition_ascii": stripDiacritics(definition), + "first_letter": firstLetter, + "url": fmt.Sprintf("/glossary/%s?term=%s", strings.ToLower(firstLetter), encodedTerm), + } + docs = append(docs, doc) + } + + sugar.Infof("[glossary] Read %d documents from MongoDB", len(docs)) + importBatch(sugar, tsClient, "glossary", docs, 500) +} + +// ==================== ESSAYS ==================== + +func indexEssays(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *repository.MongoDB) { + schema := &api.CollectionSchema{ + Name: "essays", + Fields: []api.Field{ + {Name: "title", Type: "string"}, + {Name: "author", Type: "string"}, + {Name: "year", Type: "int32", Optional: boolPtr(true)}, + {Name: "content", Type: "string"}, + {Name: "title_ascii", Type: "string"}, + {Name: "content_ascii", Type: "string"}, + {Name: "url", Type: "string"}, + }, + } + + _, err := tsClient.Collections().Create(context.Background(), schema) + if err != nil { + sugar.Fatalf("[essays] Failed to create collection: %v", err) + } + sugar.Info("[essays] Collection created") + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + coll := mongoDB.Collection("essays") + cursor, err := coll.Find(ctx, bson.M{}) + if err != nil { + sugar.Fatalf("[essays] Failed to query MongoDB: %v", err) + } + defer cursor.Close(ctx) + + var docs []interface{} + for cursor.Next(ctx) { + var raw bson.M + if err := cursor.Decode(&raw); err != nil { + sugar.Warnf("[essays] Decode error: %v", err) + continue + } + + id := objectIDToHex(raw["_id"]) + slug, _ := raw["slug"].(string) + title, _ := raw["title"].(string) + author, _ := raw["author"].(string) + + content := "" + if paragraphs, ok := raw["paragraphs"].(bson.A); ok { + strs := bsonAToStrings(paragraphs) + content = joinParagraphs(strs) + } + + doc := map[string]interface{}{ + "id": id, + "title": title, + "author": author, + "content": content, + "title_ascii": stripDiacritics(title), + "content_ascii": stripDiacritics(content), + "url": fmt.Sprintf("/essays/%s", slug), + } + + if year, ok := toOptionalInt32(raw["year"]); ok { + doc["year"] = year + } + + docs = append(docs, doc) + } + + sugar.Infof("[essays] Read %d documents from MongoDB", len(docs)) + importBatch(sugar, tsClient, "essays", docs, 500) +} + +// ==================== JOURNALS ==================== + +func indexJournals(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *repository.MongoDB) { + schema := &api.CollectionSchema{ + Name: "journals", + Fields: []api.Field{ + {Name: "book", Type: "string", Facet: boolPtr(true)}, + {Name: "title", Type: "string"}, + {Name: "content", Type: "string"}, + {Name: "title_ascii", Type: "string"}, + {Name: "content_ascii", Type: "string"}, + {Name: "url", Type: "string"}, + }, + } + + _, err := tsClient.Collections().Create(context.Background(), schema) + if err != nil { + sugar.Fatalf("[journals] Failed to create collection: %v", err) + } + sugar.Info("[journals] Collection created") + + // Build book slug mapping from journal_meta + bookSlugs := buildBookSlugMap(sugar, mongoDB, "journal_meta") + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + coll := mongoDB.Collection("journal_chapters") + cursor, err := coll.Find(ctx, bson.M{}) + if err != nil { + sugar.Fatalf("[journals] Failed to query MongoDB: %v", err) + } + defer cursor.Close(ctx) + + var docs []interface{} + for cursor.Next(ctx) { + var raw bson.M + if err := cursor.Decode(&raw); err != nil { + sugar.Warnf("[journals] Decode error: %v", err) + continue + } + + id := objectIDToHex(raw["_id"]) + book, _ := raw["book"].(string) + title, _ := raw["title"].(string) + chapterSlug, _ := raw["slug"].(string) + + content := "" + if paragraphs, ok := raw["paragraphs"].(bson.A); ok { + strs := bsonAToStrings(paragraphs) + content = joinParagraphs(strs) + } + + bookSlug := bookSlugs[book] + if bookSlug == "" { + bookSlug = book + } + + doc := map[string]interface{}{ + "id": id, + "book": book, + "title": title, + "content": content, + "title_ascii": stripDiacritics(title), + "content_ascii": stripDiacritics(content), + "url": fmt.Sprintf("/journals/%s/%s", bookSlug, chapterSlug), + } + docs = append(docs, doc) + } + + sugar.Infof("[journals] Read %d documents from MongoDB", len(docs)) + importBatch(sugar, tsClient, "journals", docs, 500) +} + +// ==================== MAGAZINES ==================== + +func indexMagazines(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *repository.MongoDB) { + schema := &api.CollectionSchema{ + Name: "magazines", + Fields: []api.Field{ + {Name: "book", Type: "string", Facet: boolPtr(true)}, + {Name: "title", Type: "string"}, + {Name: "content", Type: "string"}, + {Name: "title_ascii", Type: "string"}, + {Name: "content_ascii", Type: "string"}, + {Name: "url", Type: "string"}, + }, + } + + _, err := tsClient.Collections().Create(context.Background(), schema) + if err != nil { + sugar.Fatalf("[magazines] Failed to create collection: %v", err) + } + sugar.Info("[magazines] Collection created") + + // Build book slug mapping from magazine_meta + bookSlugs := buildBookSlugMap(sugar, mongoDB, "magazine_meta") + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + coll := mongoDB.Collection("magazine_issues") + cursor, err := coll.Find(ctx, bson.M{}) + if err != nil { + sugar.Fatalf("[magazines] Failed to query MongoDB: %v", err) + } + defer cursor.Close(ctx) + + var docs []interface{} + for cursor.Next(ctx) { + var raw bson.M + if err := cursor.Decode(&raw); err != nil { + sugar.Warnf("[magazines] Decode error: %v", err) + continue + } + + id := objectIDToHex(raw["_id"]) + book, _ := raw["book"].(string) + title, _ := raw["title"].(string) + issueSlug, _ := raw["slug"].(string) + + content := "" + if paragraphs, ok := raw["paragraphs"].(bson.A); ok { + strs := bsonAToStrings(paragraphs) + content = joinParagraphs(strs) + } + + bookSlug := bookSlugs[book] + if bookSlug == "" { + bookSlug = book + } + + doc := map[string]interface{}{ + "id": id, + "book": book, + "title": title, + "content": content, + "title_ascii": stripDiacritics(title), + "content_ascii": stripDiacritics(content), + "url": fmt.Sprintf("/magazines/%s/%s", bookSlug, issueSlug), + } + docs = append(docs, doc) + } + + sugar.Infof("[magazines] Read %d documents from MongoDB", len(docs)) + importBatch(sugar, tsClient, "magazines", docs, 500) +} + +// ==================== OTHER DOCUMENTS ==================== + +func indexOtherDocuments(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *repository.MongoDB) { + schema := &api.CollectionSchema{ + Name: "other_documents", + Fields: []api.Field{ + {Name: "book", Type: "string", Facet: boolPtr(true)}, + {Name: "title", Type: "string"}, + {Name: "content", Type: "string"}, + {Name: "title_ascii", Type: "string"}, + {Name: "content_ascii", Type: "string"}, + {Name: "url", Type: "string"}, + }, + } + + _, err := tsClient.Collections().Create(context.Background(), schema) + if err != nil { + sugar.Fatalf("[other_documents] Failed to create collection: %v", err) + } + sugar.Info("[other_documents] Collection created") + + // Build book slug mapping from other_document_meta + bookSlugs := buildBookSlugMap(sugar, mongoDB, "other_document_meta") + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + coll := mongoDB.Collection("other_document_chapters") + cursor, err := coll.Find(ctx, bson.M{}) + if err != nil { + sugar.Fatalf("[other_documents] Failed to query MongoDB: %v", err) + } + defer cursor.Close(ctx) + + var docs []interface{} + for cursor.Next(ctx) { + var raw bson.M + if err := cursor.Decode(&raw); err != nil { + sugar.Warnf("[other_documents] Decode error: %v", err) + continue + } + + id := objectIDToHex(raw["_id"]) + book, _ := raw["book"].(string) + title, _ := raw["title"].(string) + chapterSlug, _ := raw["slug"].(string) + + content := "" + if paragraphs, ok := raw["paragraphs"].(bson.A); ok { + strs := bsonAToStrings(paragraphs) + content = joinParagraphs(strs) + } + + bookSlug := bookSlugs[book] + if bookSlug == "" { + bookSlug = book + } + + doc := map[string]interface{}{ + "id": id, + "book": book, + "title": title, + "content": content, + "title_ascii": stripDiacritics(title), + "content_ascii": stripDiacritics(content), + "url": fmt.Sprintf("/other-documents/%s/%s", bookSlug, chapterSlug), + } + docs = append(docs, doc) + } + + sugar.Infof("[other_documents] Read %d documents from MongoDB", len(docs)) + importBatch(sugar, tsClient, "other_documents", docs, 500) +} + +// ==================== HELPERS ==================== + +// buildBookSlugMap reads a meta collection and returns a map of book code -> slug. +// The slug is typically the same as the book code, lowercased and hyphenated. +func buildBookSlugMap(sugar *zap.SugaredLogger, mongoDB *repository.MongoDB, metaCollection string) map[string]string { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + coll := mongoDB.Collection(metaCollection) + cursor, err := coll.Find(ctx, bson.M{}) + if err != nil { + sugar.Warnf("Failed to read %s: %v", metaCollection, err) + return map[string]string{} + } + defer cursor.Close(ctx) + + slugMap := make(map[string]string) + for cursor.Next(ctx) { + var raw bson.M + if err := cursor.Decode(&raw); err != nil { + continue + } + book, _ := raw["book"].(string) + if book != "" { + // Use the book code directly as the slug; this matches the API routing pattern + slugMap[book] = book + } + } + return slugMap +} + +// objectIDToHex safely converts a MongoDB ObjectID to hex string. +func objectIDToHex(v interface{}) string { + if oid, ok := v.(bson.ObjectID); ok { + return oid.Hex() + } + return fmt.Sprintf("%v", v) +} + +// toInt32 converts a BSON numeric value to int32. +func toInt32(v interface{}) int32 { + switch n := v.(type) { + case int32: + return n + case int64: + return int32(n) + case float64: + return int32(n) + case int: + return int32(n) + default: + return 0 + } +} + +// toOptionalInt32 tries to convert a BSON value to int32, returning ok=false if nil. +func toOptionalInt32(v interface{}) (int32, bool) { + if v == nil { + return 0, false + } + switch n := v.(type) { + case int32: + return n, true + case int64: + return int32(n), true + case float64: + return int32(n), true + case int: + return int32(n), true + default: + return 0, false + } +} + +// toString safely converts an interface to string. +func toString(v interface{}) string { + if s, ok := v.(string); ok { + return s + } + return "" +} + +// bsonAToStrings converts a bson.A to a []string. +func bsonAToStrings(arr bson.A) []string { + result := make([]string, 0, len(arr)) + for _, v := range arr { + if s, ok := v.(string); ok { + result = append(result, s) + } + } + return result +} + +// getFirstLetter returns the base letter for glossary grouping. +// Maps diacritical characters to their ASCII base letter. +func getFirstLetter(term string) string { + if term == "" { + return "?" + } + first := []rune(term)[0] + diacriticToBase := map[rune]rune{ + 'Ā': 'A', 'Ä': 'A', + 'Ī': 'I', 'É': 'I', + 'Ū': 'U', 'Ü': 'U', + 'Ṛ': 'R', 'Ṝ': 'R', + 'Ś': 'S', 'Ç': 'S', 'Ṣ': 'S', + 'Ñ': 'N', 'Ṇ': 'N', 'Ṅ': 'N', + 'Ṭ': 'T', + 'Ḍ': 'D', + 'Ḥ': 'H', + 'ā': 'A', 'ä': 'A', + 'ī': 'I', 'é': 'I', + 'ū': 'U', 'ü': 'U', + 'ṛ': 'R', 'ṝ': 'R', + 'ś': 'S', 'ç': 'S', 'ṣ': 'S', + 'ñ': 'N', 'ṇ': 'N', 'ṅ': 'N', + 'ṭ': 'T', + 'ḍ': 'D', + 'ḥ': 'H', + } + if base, ok := diacriticToBase[first]; ok { + return string(base) + } + return strings.ToUpper(string(first)) +} diff --git a/go.mod b/go.mod index cb7038c..9bd579d 100644 --- a/go.mod +++ b/go.mod @@ -6,13 +6,18 @@ require ( github.com/go-chi/chi/v5 v5.2.1 github.com/go-chi/cors v1.2.1 github.com/joho/godotenv v1.5.1 + github.com/typesense/typesense-go/v2 v2.0.0 go.mongodb.org/mongo-driver/v2 v2.2.1 go.uber.org/zap v1.27.0 ) require ( + github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/golang/snappy v1.0.0 // indirect + github.com/google/uuid v1.5.0 // indirect github.com/klauspost/compress v1.16.7 // indirect + github.com/oapi-codegen/runtime v1.1.1 // indirect + github.com/sony/gobreaker v0.5.0 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect diff --git a/go.sum b/go.sum index 97d16e9..f47a02f 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,8 @@ +github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= +github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= +github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= +github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-chi/chi/v5 v5.2.1 h1:KOIHODQj58PmL80G2Eak4WdvUzjSJSm0vG72crDCqb8= @@ -8,14 +13,28 @@ github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jinzhu/copier v0.3.4 h1:mfU6jI9PtCeUjkjQ322dlff9ELjGDu975C2p/nrubVI= +github.com/jinzhu/copier v0.3.4/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro= +github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg= +github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= +github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/typesense/typesense-go/v2 v2.0.0 h1:+MksOnrVioDqsGpz8RXkOUqhVN+yFxZwJlGDQHr/64I= +github.com/typesense/typesense-go/v2 v2.0.0/go.mod h1:7V1ZBSfmdciL6yb2bPtWha+W53gV5WZhyOSpVgDJfao= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= @@ -29,6 +48,8 @@ go.mongodb.org/mongo-driver/v2 v2.2.1 h1:w5xra3yyu/sGrziMzK1D0cRRaH/b7lWCSsoN6+W go.mongodb.org/mongo-driver/v2 v2.2.1/go.mod h1:qQkDMhCGWl3FN509DfdPd4GRBLU/41zqF/k8eTRceps= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= diff --git a/internal/config/config.go b/internal/config/config.go index adac4a4..5c08180 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -21,9 +21,9 @@ type Config struct { // PostgreSQL (users, relations, URL shortener) PostgresURI string - // Meilisearch - MeilisearchHost string - MeilisearchAPIKey string + // Typesense + TypesenseURL string + TypesenseAPIKey string // DragonflyDB / Redis RedisURI string @@ -64,8 +64,8 @@ func Load() (*Config, error) { PostgresURI: getEnv("POSTGRES_URI", ""), - MeilisearchHost: getEnv("MEILISEARCH_HOST", "http://localhost:7700"), - MeilisearchAPIKey: getEnv("MEILISEARCH_API_KEY", ""), + TypesenseURL: getEnv("TYPESENSE_URL", "https://search.prabhupadasays.org"), + TypesenseAPIKey: getEnv("TYPESENSE_API_KEY", ""), RedisURI: getEnv("REDIS_URI", "redis://localhost:6379"), diff --git a/internal/handler/search.go b/internal/handler/search.go new file mode 100644 index 0000000..51035e5 --- /dev/null +++ b/internal/handler/search.go @@ -0,0 +1,512 @@ +package handler + +import ( + "context" + "fmt" + "math" + "net/http" + "sort" + "strconv" + "strings" + + "github.com/prabhupadasays/spsays-bff/internal/repository" + "github.com/prabhupadasays/spsays-bff/pkg/response" + "github.com/typesense/typesense-go/v2/typesense/api" +) + +// SearchHandler handles search requests via Typesense. +type SearchHandler struct { + ts *repository.TypesenseClient +} + +// NewSearchHandler creates a new SearchHandler. +func NewSearchHandler(ts *repository.TypesenseClient) *SearchHandler { + return &SearchHandler{ts: ts} +} + +// validSearchTypes lists the valid values for the "type" query parameter. +var validSearchTypes = map[string]bool{ + "all": true, + "verses": true, + "prose": true, + "letters": true, + "transcriptions": true, + "glossary": true, + "essays": true, + "journals": true, + "magazines": true, + "other_documents": true, +} + +// collectionQueryByFields maps each Typesense collection to the fields to search. +// Field order matters: most relevant fields first. Used with query_by_weights +// to control ranking — identifiers and primary text rank higher than secondary +// text like purport/content where references/citations may appear. +var collectionQueryByFields = map[string]string{ + // Note: purport intentionally excluded from default verse search — matches there + // (where verses are quoted in commentary) drown out actual verse-text matches. + "verses": "reference,transliteration,transliteration_ascii,translation,translation_ascii", + "prose": "title,title_ascii,content,content_ascii", + "letters": "recipient_display,recipient_display_ascii,content,content_ascii", + "transcriptions": "reference,name,name_ascii,content,content_ascii", + "glossary": "term,term_ascii,definition,definition_ascii", + "essays": "title,title_ascii,content,content_ascii", + "journals": "title,title_ascii,content,content_ascii", + "magazines": "title,title_ascii,content,content_ascii", + "other_documents": "title,title_ascii,content,content_ascii", +} + +// collectionQueryByWeights assigns weights matching the order of fields above. +// Higher weight = stronger ranking signal. Identifiers/primary text dominate. +var collectionQueryByWeights = map[string]string{ + "verses": "10,8,8,4,4", // reference, transliteration, transliteration_ascii, translation, translation_ascii + "prose": "10,10,2,2", + "letters": "10,10,2,2", + "transcriptions": "10,8,8,2,2", + "glossary": "10,10,4,4", + "essays": "10,10,2,2", + "journals": "10,10,2,2", + "magazines": "10,10,2,2", + "other_documents": "10,10,2,2", +} + +// searchResultItem represents a single search result in the API response. +type searchResultItem struct { + Collection string `json:"collection"` + Slug string `json:"slug,omitempty"` + Title string `json:"title"` + Snippet string `json:"snippet"` + URL string `json:"url"` + Meta map[string]interface{} `json:"meta,omitempty"` + score int64 // internal, not serialized +} + +// searchResponse is the shape of the data field in the API response. +type searchResponse struct { + Query string `json:"query"` + Type string `json:"type"` + Results []searchResultItem `json:"results"` + Total int `json:"total"` + Page int `json:"page"` + Limit int `json:"limit"` + TotalPages int `json:"total_pages"` +} + +// Search handles GET /api/v1/search?q=...&type=all&page=1&limit=20 +func (h *SearchHandler) Search(w http.ResponseWriter, r *http.Request) { + query := strings.TrimSpace(r.URL.Query().Get("q")) + if query == "" { + response.Error(w, http.StatusBadRequest, "query parameter 'q' is required") + return + } + + searchType := r.URL.Query().Get("type") + if searchType == "" { + searchType = "all" + } + + // Parse comma-separated types (e.g., "verses,letters,transcriptions") + var requestedTypes []string + if searchType != "all" { + for _, t := range strings.Split(searchType, ",") { + t = strings.TrimSpace(t) + if t == "" || t == "all" { + continue + } + if !validSearchTypes[t] { + response.Error(w, http.StatusBadRequest, fmt.Sprintf("invalid type '%s'", t)) + return + } + requestedTypes = append(requestedTypes, t) + } + if len(requestedTypes) == 0 { + searchType = "all" + } + } + + page := parseIntParam(r.URL.Query().Get("page"), 1) + if page < 1 { + page = 1 + } + + limit := parseIntParam(r.URL.Query().Get("limit"), 20) + if limit < 1 { + limit = 1 + } + if limit > 100 { + limit = 100 + } + + ctx := r.Context() + + var results []searchResultItem + var totalFound int + + if searchType == "all" { + // Search all 9 collections + results, totalFound = h.multiSearch(ctx, nil, query, page, limit) + } else if len(requestedTypes) == 1 { + // Single collection — use singleSearch for efficiency + results, totalFound = h.singleSearch(ctx, requestedTypes[0], query, page, limit) + } else { + // Multiple collections — use multiSearch with a subset + results, totalFound = h.multiSearch(ctx, requestedTypes, query, page, limit) + } + + totalPages := int(math.Ceil(float64(totalFound) / float64(limit))) + + resp := searchResponse{ + Query: query, + Type: searchType, + Results: results, + Total: totalFound, + Page: page, + Limit: limit, + TotalPages: totalPages, + } + + response.JSON(w, http.StatusOK, resp) +} + +// multiSearch searches across all 9 collections using Typesense multi_search, +// merges results by text_match score, and returns a single paginated list. +// multiSearch searches across multiple collections using Typesense multi_search. +// If collections is nil, searches all 9 collections. +func (h *SearchHandler) multiSearch(ctx context.Context, collections []string, query string, page, limit int) ([]searchResultItem, int) { + if collections == nil { + collections = []string{ + "verses", "prose", "letters", "transcriptions", "glossary", + "essays", "journals", "magazines", "other_documents", + } + } + + exhaustive := true + searches := make([]api.MultiSearchCollectionParameters, 0, len(collections)) + for _, col := range collections { + queryBy := collectionQueryByFields[col] + weights := collectionQueryByWeights[col] + perPage := limit + pg := page + params := api.MultiSearchCollectionParameters{ + Collection: col, + Q: stringPtr(query), + QueryBy: stringPtr(queryBy), + PerPage: &perPage, + Page: &pg, + Prefix: stringPtr("true"), + NumTypos: stringPtr("2"), + ExhaustiveSearch: &exhaustive, + } + if weights != "" { + params.QueryByWeights = stringPtr(weights) + } + searches = append(searches, params) + } + + searchParams := api.MultiSearchSearchesParameter{ + Searches: searches, + } + + commonParams := &api.MultiSearchParams{} + + msResult, err := h.ts.Client.MultiSearch.Perform(ctx, commonParams, searchParams) + if err != nil { + return []searchResultItem{}, 0 + } + + allItems := []searchResultItem{} + totalFound := 0 + + for i, sr := range msResult.Results { + collectionName := collections[i] + if sr.Found != nil { + totalFound += *sr.Found + } + if sr.Hits == nil { + continue + } + for _, hit := range *sr.Hits { + item := hitToResultItem(collectionName, &hit) + allItems = append(allItems, item) + } + } + + // Sort by text_match score descending + sort.Slice(allItems, func(i, j int) bool { + return allItems[i].score > allItems[j].score + }) + + // Paginate the merged results: since each sub-search already returned `limit` + // results for the given page, the merged set has at most 9*limit items. + // We take the top `limit` after sorting. + if len(allItems) > limit { + allItems = allItems[:limit] + } + + return allItems, totalFound +} + +// singleSearch searches a single collection. +func (h *SearchHandler) singleSearch(ctx context.Context, collection, query string, page, limit int) ([]searchResultItem, int) { + queryBy, ok := collectionQueryByFields[collection] + if !ok { + return []searchResultItem{}, 0 + } + + exhaustive := true + params := &api.SearchCollectionParams{ + Q: stringPtr(query), + QueryBy: stringPtr(queryBy), + Page: &page, + PerPage: &limit, + Prefix: stringPtr("true"), + NumTypos: stringPtr("2"), + ExhaustiveSearch: &exhaustive, + } + if weights := collectionQueryByWeights[collection]; weights != "" { + params.QueryByWeights = stringPtr(weights) + } + + sr, err := h.ts.Client.Collection(collection).Documents().Search(ctx, params) + if err != nil { + return []searchResultItem{}, 0 + } + + totalFound := 0 + if sr.Found != nil { + totalFound = *sr.Found + } + + items := []searchResultItem{} + if sr.Hits != nil { + for _, hit := range *sr.Hits { + item := hitToResultItem(collection, &hit) + items = append(items, item) + } + } + + return items, totalFound +} + +// hitToResultItem converts a Typesense SearchResultHit into our API result shape. +func hitToResultItem(collection string, hit *api.SearchResultHit) searchResultItem { + doc := make(map[string]interface{}) + if hit.Document != nil { + doc = *hit.Document + } + + var score int64 + if hit.TextMatch != nil { + score = *hit.TextMatch + } + + item := searchResultItem{ + Collection: collection, + URL: getStringField(doc, "url"), + score: score, + } + + // Extract title based on collection type + switch collection { + case "verses": + item.Title = getStringField(doc, "reference") + item.Slug = slugFromURL(item.URL) + item.Meta = map[string]interface{}{ + "book": getStringField(doc, "book"), + "reference": getStringField(doc, "reference"), + } + case "prose": + item.Title = getStringField(doc, "title") + item.Slug = slugFromURL(item.URL) + item.Meta = map[string]interface{}{ + "book": getStringField(doc, "book"), + "type": getStringField(doc, "type"), + } + case "letters": + recipient := getStringField(doc, "recipient_display") + if recipient != "" { + item.Title = "Letter to " + recipient + } else { + item.Title = "Letter" + } + item.Slug = slugFromURL(item.URL) + item.Meta = map[string]interface{}{ + "recipient_display": recipient, + } + if date, ok := doc["date"]; ok { + item.Meta["date"] = date + } + if location, ok := doc["location"]; ok { + item.Meta["location"] = location + } + case "transcriptions": + item.Title = getStringField(doc, "name") + item.Slug = slugFromURL(item.URL) + item.Meta = map[string]interface{}{ + "type": getStringField(doc, "type"), + } + if place, ok := doc["place"]; ok { + item.Meta["place"] = place + } + if date, ok := doc["date"]; ok { + item.Meta["date"] = date + } + if ref, ok := doc["reference"]; ok { + item.Meta["reference"] = ref + } + case "glossary": + item.Title = getStringField(doc, "term") + item.Slug = getStringField(doc, "term") + item.Meta = map[string]interface{}{ + "first_letter": getStringField(doc, "first_letter"), + } + case "essays": + item.Title = getStringField(doc, "title") + item.Slug = slugFromURL(item.URL) + item.Meta = map[string]interface{}{ + "author": getStringField(doc, "author"), + } + if year, ok := doc["year"]; ok { + item.Meta["year"] = year + } + case "journals": + item.Title = getStringField(doc, "title") + item.Slug = slugFromURL(item.URL) + item.Meta = map[string]interface{}{ + "book": getStringField(doc, "book"), + } + case "magazines": + item.Title = getStringField(doc, "title") + item.Slug = slugFromURL(item.URL) + item.Meta = map[string]interface{}{ + "book": getStringField(doc, "book"), + } + case "other_documents": + item.Title = getStringField(doc, "title") + item.Slug = slugFromURL(item.URL) + item.Meta = map[string]interface{}{ + "book": getStringField(doc, "book"), + } + } + + // Extract snippet from highlights + item.Snippet = extractSnippet(hit, collection) + + return item +} + +// extractSnippet builds a snippet string from Typesense highlight data. +// It prefers the newer highlight map, falling back to the deprecated highlights array. +func extractSnippet(hit *api.SearchResultHit, collection string) string { + // Try the highlights array (deprecated but widely available) + if hit.Highlights != nil && len(*hit.Highlights) > 0 { + for _, hl := range *hit.Highlights { + if hl.Snippet != nil && *hl.Snippet != "" { + return *hl.Snippet + } + if hl.Snippets != nil && len(*hl.Snippets) > 0 { + return (*hl.Snippets)[0] + } + } + } + + // Try the highlight map (v2 style) + if hit.Highlight != nil { + hlMap := *hit.Highlight + // Prioritize content/text fields for snippet + priorityFields := snippetPriorityFields(collection) + for _, field := range priorityFields { + if val, ok := hlMap[field]; ok { + if s := extractHighlightValue(val); s != "" { + return s + } + } + } + // Fallback: return any highlighted field + for _, val := range hlMap { + if s := extractHighlightValue(val); s != "" { + return s + } + } + } + + return "" +} + +// snippetPriorityFields returns the fields to check first for snippets, per collection. +func snippetPriorityFields(collection string) []string { + switch collection { + case "verses": + return []string{"translation", "purport", "transliteration", "reference"} + case "prose": + return []string{"content", "title"} + case "letters": + return []string{"content", "recipient_display"} + case "transcriptions": + return []string{"content", "name", "reference"} + case "glossary": + return []string{"definition", "term"} + case "essays": + return []string{"content", "title"} + case "journals", "magazines", "other_documents": + return []string{"content", "title"} + default: + return []string{"content", "title"} + } +} + +// extractHighlightValue tries to pull a string snippet from a highlight value, +// which may be a string, a map with "snippet" or "matched_tokens", etc. +func extractHighlightValue(val interface{}) string { + switch v := val.(type) { + case string: + if v != "" { + return v + } + case map[string]interface{}: + if snippet, ok := v["snippet"].(string); ok && snippet != "" { + return snippet + } + if value, ok := v["value"].(string); ok && value != "" { + return value + } + } + return "" +} + +// getStringField safely extracts a string value from a map. +func getStringField(doc map[string]interface{}, key string) string { + if val, ok := doc[key]; ok { + if s, ok := val.(string); ok { + return s + } + } + return "" +} + +// slugFromURL extracts the last path segment from a URL as a slug. +func slugFromURL(u string) string { + u = strings.TrimRight(u, "/") + parts := strings.Split(u, "/") + if len(parts) > 0 { + return parts[len(parts)-1] + } + return "" +} + +// parseIntParam parses an integer from a query param string with a default fallback. +func parseIntParam(s string, defaultVal int) int { + if s == "" { + return defaultVal + } + v, err := strconv.Atoi(s) + if err != nil { + return defaultVal + } + return v +} + +// stringPtr returns a pointer to a string value. +func stringPtr(s string) *string { + return &s +} diff --git a/internal/repository/typesense.go b/internal/repository/typesense.go new file mode 100644 index 0000000..f34e95d --- /dev/null +++ b/internal/repository/typesense.go @@ -0,0 +1,19 @@ +package repository + +import ( + "github.com/typesense/typesense-go/v2/typesense" +) + +// TypesenseClient wraps the Typesense client for search operations. +type TypesenseClient struct { + Client *typesense.Client +} + +// NewTypesenseClient creates a new Typesense client with the given URL and API key. +func NewTypesenseClient(url, apiKey string) *TypesenseClient { + client := typesense.NewClient( + typesense.WithServer(url), + typesense.WithAPIKey(apiKey), + ) + return &TypesenseClient{Client: client} +} diff --git a/internal/server/server.go b/internal/server/server.go index 28af8b3..7c18092 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -30,6 +30,7 @@ type Server struct { magazineH *handler.MagazineHandler otherDocH *handler.OtherDocumentHandler transcriptionH *handler.TranscriptionHandler + searchH *handler.SearchHandler } // New creates a new Server with all routes registered. @@ -39,6 +40,9 @@ func New(cfg *config.Config, logger *zap.Logger) *Server { logger.Fatal("failed to connect to MongoDB", zap.Error(err)) } + // Initialize Typesense client + tsClient := repository.NewTypesenseClient(cfg.TypesenseURL, cfg.TypesenseAPIKey) + // Initialize repositories bookRepo := repository.NewBookRepo(mongoDB) letterRepo := repository.NewLetterRepo(mongoDB) @@ -65,6 +69,7 @@ func New(cfg *config.Config, logger *zap.Logger) *Server { magazineH: handler.NewMagazineHandler(magazineRepo), otherDocH: handler.NewOtherDocumentHandler(otherDocRepo), transcriptionH: handler.NewTranscriptionHandler(transcriptionRepo), + searchH: handler.NewSearchHandler(tsClient), } s.setupRouter() @@ -297,10 +302,8 @@ func (s *Server) setupRouter() { r.Get("/{slug}", s.transcriptionH.GetBySlug) }) - // Search (Meilisearch) - r.Route("/search", func(r chi.Router) { - // TODO: register search handlers - }) + // Search (Typesense) + r.Get("/search", s.searchH.Search) // User routes (PostgreSQL) r.Route("/auth", func(r chi.Router) {