search added
This commit is contained in:
Krispire Admin
+102
-12
@@ -2,7 +2,7 @@
|
||||
|
||||
## What this project is
|
||||
|
||||
A Go-based Backend for Frontend (BFF) API server for [prabhupadasays.org](https://prabhupadasays.org). It is the **sole data provider** for the production Next.js frontend (and future mobile apps). The knowledgebase frontend has removed all direct Prisma, MongoDB, and S3 dependencies — everything flows through this BFF. It aggregates content from MongoDB, user data from PostgreSQL, search from Meilisearch, caching from DragonflyDB, and media from S3 into a unified API surface.
|
||||
A Go-based Backend for Frontend (BFF) API server for [prabhupadasays.org](https://prabhupadasays.org). It is the **sole data provider** for the production Next.js frontend (and future mobile apps). The knowledgebase frontend has removed all direct Prisma, MongoDB, and S3 dependencies — everything flows through this BFF. It aggregates content from MongoDB, user data from PostgreSQL, search from Typesense, caching from DragonflyDB, and media from S3 into a unified API surface.
|
||||
|
||||
---
|
||||
|
||||
@@ -13,7 +13,7 @@ A Go-based Backend for Frontend (BFF) API server for [prabhupadasays.org](https:
|
||||
- **Zap** — structured logging (uber-go/zap)
|
||||
- **MongoDB** — content database (books, verses, letters, essays, glossary, disciples, transcriptions)
|
||||
- **PostgreSQL** — relational data (users, bookmarks, collections, mentions, URL shortener)
|
||||
- **Meilisearch** — typo-tolerant full-text search across all content
|
||||
- **Typesense** — typo-tolerant full-text search engine (https://typesense.org)
|
||||
- **DragonflyDB** — Redis-compatible cache (API responses, rate limiting, sessions)
|
||||
- **AWS S3** — media storage (avatars, images, audio files)
|
||||
- **FCM + APNs** — push notifications for Android and iOS
|
||||
@@ -37,7 +37,7 @@ A Go-based Backend for Frontend (BFF) API server for [prabhupadasays.org](https:
|
||||
└──────────────┘ │ │ │
|
||||
│ ┌─────────────────────┼──────────┐ │
|
||||
│ ▼ ▼ ▼ ▼ ▼ │
|
||||
│ Mongo Postgres Meili S3 Dragonfly│
|
||||
│ Mongo Postgres Typesense S3 Dragonfly│
|
||||
│ (content)(users, (search)(media)(cache)│
|
||||
│ mentions, │
|
||||
│ URL short, │
|
||||
@@ -52,15 +52,20 @@ A Go-based Backend for Frontend (BFF) API server for [prabhupadasays.org](https:
|
||||
```
|
||||
spsays-bff/
|
||||
├── cmd/
|
||||
│ └── api/
|
||||
│ └── main.go → Entry point (config, logger, server, graceful shutdown)
|
||||
│ ├── api/
|
||||
│ │ └── main.go → Entry point (config, logger, server, graceful shutdown)
|
||||
│ └── indexer/
|
||||
│ └── main.go → CLI tool for indexing MongoDB content into Typesense
|
||||
│ • --full drop all collections and reindex
|
||||
│ • --only=verses,letters index specific collections only
|
||||
├── internal/
|
||||
│ ├── config/
|
||||
│ │ └── config.go → Environment config (all services: Mongo, Postgres, Meili, S3, etc.)
|
||||
│ │ └── config.go → Environment config (all services: Mongo, Postgres, Typesense, S3, etc.)
|
||||
│ ├── server/
|
||||
│ │ └── server.go → Chi router setup, global middleware, route registration
|
||||
│ ├── handler/ → HTTP handlers (one file per domain)
|
||||
│ │ ├── health.go → Health check endpoint
|
||||
│ │ ├── search.go → Federated search handler (multi-collection, typo-tolerant)
|
||||
│ │ ├── book.go → Book routes (SB, CC, KB, BG, generic verse & prose)
|
||||
│ │ ├── letter.go → Letter routes (by year, recipient, location)
|
||||
│ │ ├── essay.go → Essay routes
|
||||
@@ -71,7 +76,7 @@ spsays-bff/
|
||||
│ │ ├── other_document.go → Other-document routes
|
||||
│ │ └── transcription.go → Transcription routes (by year, place, type, slug)
|
||||
│ ├── service/ → Business logic layer (orchestrates repositories)
|
||||
│ ├── repository/ → Data access layer (Mongo, Postgres, Meili, S3, Dragonfly)
|
||||
│ ├── repository/ → Data access layer (Mongo, Postgres, Typesense, S3, Dragonfly)
|
||||
│ │ ├── mongo.go → MongoDB connection + database handle
|
||||
│ │ ├── book_repo.go → Book / verse queries
|
||||
│ │ ├── letter_repo.go → Letter queries
|
||||
@@ -81,7 +86,8 @@ spsays-bff/
|
||||
│ │ ├── journal_repo.go → Journal queries
|
||||
│ │ ├── magazine_repo.go → Magazine queries
|
||||
│ │ ├── other_document_repo.go → Other-document queries
|
||||
│ │ └── transcription_repo.go → Transcription queries (with contentJson fallback)
|
||||
│ │ ├── transcription_repo.go → Transcription queries (with contentJson fallback)
|
||||
│ │ └── typesense.go → Typesense client wrapper (search, indexing helpers)
|
||||
│ ├── middleware/
|
||||
│ │ ├── logger.go → Zap-based request logging (method, path, status, duration)
|
||||
│ │ ├── auth.go → JWT Bearer token authentication + context user extraction
|
||||
@@ -227,6 +233,46 @@ All routes are prefixed with `/api/v1`.
|
||||
| `GET` | `/api/v1/transcriptions/adjacent/{slug}` | Prev/next within same year (`?year=N`) |
|
||||
| `GET` | `/api/v1/transcriptions/{slug}` | Single transcription (full content + audio URL) |
|
||||
|
||||
### Search
|
||||
|
||||
| Method | Route | Description |
|
||||
|--------|-------|-------------|
|
||||
| `GET` | `/api/v1/search?q=...&type=...&page=1&limit=20` | Federated typo-tolerant search across all content |
|
||||
|
||||
**Query params:**
|
||||
- `q` — search query (required)
|
||||
- `type` — content type filter, comma-separated. Valid values:
|
||||
- `all` (default) — search all 9 collections
|
||||
- `verses`, `prose`, `letters`, `transcriptions`, `glossary`, `essays`, `journals`, `magazines`, `other_documents`
|
||||
- Multiple: `verses,letters,transcriptions`
|
||||
- `page` — page number (default 1)
|
||||
- `limit` — results per page (default 20, max 100)
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"data": {
|
||||
"query": "surrender unto",
|
||||
"type": "all",
|
||||
"results": [
|
||||
{
|
||||
"collection": "verses",
|
||||
"id": "...",
|
||||
"title": "BG 18.66",
|
||||
"snippet": "...just <mark>surrender unto</mark> Me...",
|
||||
"url": "/books/bg/18/66",
|
||||
"meta": { "book": "bg", "reference": "BG 18.66" }
|
||||
}
|
||||
],
|
||||
"total": 150,
|
||||
"page": 1,
|
||||
"limit": 20,
|
||||
"total_pages": 8
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Authenticated (JWT required) — planned
|
||||
|
||||
| Method | Route | Description |
|
||||
@@ -247,6 +293,49 @@ All routes are prefixed with `/api/v1`.
|
||||
|
||||
---
|
||||
|
||||
## Typesense Indexing
|
||||
|
||||
The BFF includes a CLI tool (`cmd/indexer/main.go`) for managing Typesense indexes.
|
||||
|
||||
### Collections (9 total, ~35,500 documents)
|
||||
|
||||
| Collection | MongoDB Source | Documents | Searchable Fields |
|
||||
|---|---|---|---|
|
||||
| `verses` | `verses` | 21,127 | translation, purport, transliteration, reference (+ ASCII variants) |
|
||||
| `prose` | `prose_chapters` | 627 | title, content (+ ASCII variants) |
|
||||
| `letters` | `letters` | 6,866 | content, recipient_display (+ ASCII variants) |
|
||||
| `transcriptions` | `transcriptions` | 3,787 | content, name, reference (+ ASCII variants) |
|
||||
| `glossary` | `glossary` | 2,749 | term, definition (+ ASCII variants) |
|
||||
| `essays` | `essays` | 23 | title, content (+ ASCII variants) |
|
||||
| `journals` | `journal_chapters` | 300 | title, content (+ ASCII variants) |
|
||||
| `magazines` | `magazine_issues` | 59 | title, content (+ ASCII variants) |
|
||||
| `other_documents` | `other_document_chapters` | 28 | title, content (+ ASCII variants) |
|
||||
|
||||
### ASCII Shadow Fields
|
||||
|
||||
For diacritic-insensitive search, every searchable text field has a `_ascii` shadow field with all IAST diacritics stripped (`ā→a`, `ṛ→r`, `ṣ→s`, `ṭ→t`, `ñ→n`, etc.). This allows queries like "Krsna" to match "Kṛṣṇa", "Yashodanandana" to match "Yaśodanandana".
|
||||
|
||||
### Search Configuration
|
||||
|
||||
- **`prefix=true`** — partial word matching (e.g., "sarv" matches "sarva")
|
||||
- **`num_typos=2`** — up to 2 character substitutions allowed per word
|
||||
- **HTML stripping** — all paragraph content has HTML tags removed before indexing
|
||||
|
||||
### CLI Usage
|
||||
|
||||
```/dev/null/shell.sh#L1-12
|
||||
# Full reindex (drops all collections and rebuilds)
|
||||
go run cmd/indexer/main.go --full
|
||||
|
||||
# Index specific collections only (drops + recreates them)
|
||||
go run cmd/indexer/main.go --only=verses,letters
|
||||
|
||||
# Index all, but skip collections that already exist
|
||||
go run cmd/indexer/main.go
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Data Models
|
||||
|
||||
### Content Models (MongoDB)
|
||||
@@ -333,9 +422,9 @@ MONGO_DB=spsays
|
||||
# PostgreSQL (users, relations)
|
||||
POSTGRES_URI=postgres://...
|
||||
|
||||
# Meilisearch
|
||||
MEILISEARCH_HOST=http://localhost:7700
|
||||
MEILISEARCH_API_KEY=
|
||||
# Typesense
|
||||
TYPESENSE_URL=https://search.prabhupadasays.org
|
||||
TYPESENSE_API_KEY=
|
||||
|
||||
# DragonflyDB / Redis
|
||||
REDIS_URI=redis://localhost:6379
|
||||
@@ -402,7 +491,7 @@ make tidy
|
||||
Migration status (Prisma → BFF):
|
||||
1. ✅ Done — BFF API routes matching all former Prisma queries
|
||||
2. ✅ Done — Frontend migrated from Prisma to BFF API calls (Prisma, MongoDB driver, and S3 SDK removed from knowledgebase)
|
||||
3. Add Meilisearch indexing + search endpoints
|
||||
3. ✅ Done — Typesense indexing + federated search endpoints (`cmd/indexer/main.go`, `/api/v1/search`)
|
||||
4. Add PostgreSQL for user features (auth, bookmarks, collections, mentions)
|
||||
5. Add mobile app support (same API, push notifications)
|
||||
|
||||
@@ -418,3 +507,4 @@ Migration status (Prisma → BFF):
|
||||
- **Handler layer** — thin HTTP handlers that parse requests, call services, return responses
|
||||
- **No ORM** — use native drivers (`mongo-go-driver`, `pgx`) for full control
|
||||
- **Explicit book routing** — SB, CC, and KB have dedicated route groups to avoid URL ambiguity with generic `{code}/{chapter}/{verse}` patterns
|
||||
- **Search gateway** — search is provided by Typesense, indexed from MongoDB via the indexer CLI; the BFF acts as the search gateway with multi-collection federated queries.
|
||||
+1113
File diff suppressed because it is too large
Load Diff
@@ -6,13 +6,18 @@ require (
|
||||
github.com/go-chi/chi/v5 v5.2.1
|
||||
github.com/go-chi/cors v1.2.1
|
||||
github.com/joho/godotenv v1.5.1
|
||||
github.com/typesense/typesense-go/v2 v2.0.0
|
||||
go.mongodb.org/mongo-driver/v2 v2.2.1
|
||||
go.uber.org/zap v1.27.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
|
||||
github.com/golang/snappy v1.0.0 // indirect
|
||||
github.com/google/uuid v1.5.0 // indirect
|
||||
github.com/klauspost/compress v1.16.7 // indirect
|
||||
github.com/oapi-codegen/runtime v1.1.1 // indirect
|
||||
github.com/sony/gobreaker v0.5.0 // indirect
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
|
||||
github.com/xdg-go/scram v1.1.2 // indirect
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk=
|
||||
github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ=
|
||||
github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk=
|
||||
github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/go-chi/chi/v5 v5.2.1 h1:KOIHODQj58PmL80G2Eak4WdvUzjSJSm0vG72crDCqb8=
|
||||
@@ -8,14 +13,28 @@ github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs=
|
||||
github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
|
||||
github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/jinzhu/copier v0.3.4 h1:mfU6jI9PtCeUjkjQ322dlff9ELjGDu975C2p/nrubVI=
|
||||
github.com/jinzhu/copier v0.3.4/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
|
||||
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
|
||||
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||
github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE=
|
||||
github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
|
||||
github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
|
||||
github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro=
|
||||
github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg=
|
||||
github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
|
||||
github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/typesense/typesense-go/v2 v2.0.0 h1:+MksOnrVioDqsGpz8RXkOUqhVN+yFxZwJlGDQHr/64I=
|
||||
github.com/typesense/typesense-go/v2 v2.0.0/go.mod h1:7V1ZBSfmdciL6yb2bPtWha+W53gV5WZhyOSpVgDJfao=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
|
||||
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
|
||||
@@ -29,6 +48,8 @@ go.mongodb.org/mongo-driver/v2 v2.2.1 h1:w5xra3yyu/sGrziMzK1D0cRRaH/b7lWCSsoN6+W
|
||||
go.mongodb.org/mongo-driver/v2 v2.2.1/go.mod h1:qQkDMhCGWl3FN509DfdPd4GRBLU/41zqF/k8eTRceps=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU=
|
||||
go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
|
||||
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
|
||||
|
||||
@@ -21,9 +21,9 @@ type Config struct {
|
||||
// PostgreSQL (users, relations, URL shortener)
|
||||
PostgresURI string
|
||||
|
||||
// Meilisearch
|
||||
MeilisearchHost string
|
||||
MeilisearchAPIKey string
|
||||
// Typesense
|
||||
TypesenseURL string
|
||||
TypesenseAPIKey string
|
||||
|
||||
// DragonflyDB / Redis
|
||||
RedisURI string
|
||||
@@ -64,8 +64,8 @@ func Load() (*Config, error) {
|
||||
|
||||
PostgresURI: getEnv("POSTGRES_URI", ""),
|
||||
|
||||
MeilisearchHost: getEnv("MEILISEARCH_HOST", "http://localhost:7700"),
|
||||
MeilisearchAPIKey: getEnv("MEILISEARCH_API_KEY", ""),
|
||||
TypesenseURL: getEnv("TYPESENSE_URL", "https://search.prabhupadasays.org"),
|
||||
TypesenseAPIKey: getEnv("TYPESENSE_API_KEY", ""),
|
||||
|
||||
RedisURI: getEnv("REDIS_URI", "redis://localhost:6379"),
|
||||
|
||||
|
||||
@@ -0,0 +1,512 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/prabhupadasays/spsays-bff/internal/repository"
|
||||
"github.com/prabhupadasays/spsays-bff/pkg/response"
|
||||
"github.com/typesense/typesense-go/v2/typesense/api"
|
||||
)
|
||||
|
||||
// SearchHandler handles search requests via Typesense.
|
||||
type SearchHandler struct {
|
||||
ts *repository.TypesenseClient
|
||||
}
|
||||
|
||||
// NewSearchHandler creates a new SearchHandler.
|
||||
func NewSearchHandler(ts *repository.TypesenseClient) *SearchHandler {
|
||||
return &SearchHandler{ts: ts}
|
||||
}
|
||||
|
||||
// validSearchTypes lists the valid values for the "type" query parameter.
|
||||
var validSearchTypes = map[string]bool{
|
||||
"all": true,
|
||||
"verses": true,
|
||||
"prose": true,
|
||||
"letters": true,
|
||||
"transcriptions": true,
|
||||
"glossary": true,
|
||||
"essays": true,
|
||||
"journals": true,
|
||||
"magazines": true,
|
||||
"other_documents": true,
|
||||
}
|
||||
|
||||
// collectionQueryByFields maps each Typesense collection to the fields to search.
|
||||
// Field order matters: most relevant fields first. Used with query_by_weights
|
||||
// to control ranking — identifiers and primary text rank higher than secondary
|
||||
// text like purport/content where references/citations may appear.
|
||||
var collectionQueryByFields = map[string]string{
|
||||
// Note: purport intentionally excluded from default verse search — matches there
|
||||
// (where verses are quoted in commentary) drown out actual verse-text matches.
|
||||
"verses": "reference,transliteration,transliteration_ascii,translation,translation_ascii",
|
||||
"prose": "title,title_ascii,content,content_ascii",
|
||||
"letters": "recipient_display,recipient_display_ascii,content,content_ascii",
|
||||
"transcriptions": "reference,name,name_ascii,content,content_ascii",
|
||||
"glossary": "term,term_ascii,definition,definition_ascii",
|
||||
"essays": "title,title_ascii,content,content_ascii",
|
||||
"journals": "title,title_ascii,content,content_ascii",
|
||||
"magazines": "title,title_ascii,content,content_ascii",
|
||||
"other_documents": "title,title_ascii,content,content_ascii",
|
||||
}
|
||||
|
||||
// collectionQueryByWeights assigns weights matching the order of fields above.
|
||||
// Higher weight = stronger ranking signal. Identifiers/primary text dominate.
|
||||
var collectionQueryByWeights = map[string]string{
|
||||
"verses": "10,8,8,4,4", // reference, transliteration, transliteration_ascii, translation, translation_ascii
|
||||
"prose": "10,10,2,2",
|
||||
"letters": "10,10,2,2",
|
||||
"transcriptions": "10,8,8,2,2",
|
||||
"glossary": "10,10,4,4",
|
||||
"essays": "10,10,2,2",
|
||||
"journals": "10,10,2,2",
|
||||
"magazines": "10,10,2,2",
|
||||
"other_documents": "10,10,2,2",
|
||||
}
|
||||
|
||||
// searchResultItem represents a single search result in the API response.
|
||||
type searchResultItem struct {
|
||||
Collection string `json:"collection"`
|
||||
Slug string `json:"slug,omitempty"`
|
||||
Title string `json:"title"`
|
||||
Snippet string `json:"snippet"`
|
||||
URL string `json:"url"`
|
||||
Meta map[string]interface{} `json:"meta,omitempty"`
|
||||
score int64 // internal, not serialized
|
||||
}
|
||||
|
||||
// searchResponse is the shape of the data field in the API response.
|
||||
type searchResponse struct {
|
||||
Query string `json:"query"`
|
||||
Type string `json:"type"`
|
||||
Results []searchResultItem `json:"results"`
|
||||
Total int `json:"total"`
|
||||
Page int `json:"page"`
|
||||
Limit int `json:"limit"`
|
||||
TotalPages int `json:"total_pages"`
|
||||
}
|
||||
|
||||
// Search handles GET /api/v1/search?q=...&type=all&page=1&limit=20
|
||||
func (h *SearchHandler) Search(w http.ResponseWriter, r *http.Request) {
|
||||
query := strings.TrimSpace(r.URL.Query().Get("q"))
|
||||
if query == "" {
|
||||
response.Error(w, http.StatusBadRequest, "query parameter 'q' is required")
|
||||
return
|
||||
}
|
||||
|
||||
searchType := r.URL.Query().Get("type")
|
||||
if searchType == "" {
|
||||
searchType = "all"
|
||||
}
|
||||
|
||||
// Parse comma-separated types (e.g., "verses,letters,transcriptions")
|
||||
var requestedTypes []string
|
||||
if searchType != "all" {
|
||||
for _, t := range strings.Split(searchType, ",") {
|
||||
t = strings.TrimSpace(t)
|
||||
if t == "" || t == "all" {
|
||||
continue
|
||||
}
|
||||
if !validSearchTypes[t] {
|
||||
response.Error(w, http.StatusBadRequest, fmt.Sprintf("invalid type '%s'", t))
|
||||
return
|
||||
}
|
||||
requestedTypes = append(requestedTypes, t)
|
||||
}
|
||||
if len(requestedTypes) == 0 {
|
||||
searchType = "all"
|
||||
}
|
||||
}
|
||||
|
||||
page := parseIntParam(r.URL.Query().Get("page"), 1)
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
|
||||
limit := parseIntParam(r.URL.Query().Get("limit"), 20)
|
||||
if limit < 1 {
|
||||
limit = 1
|
||||
}
|
||||
if limit > 100 {
|
||||
limit = 100
|
||||
}
|
||||
|
||||
ctx := r.Context()
|
||||
|
||||
var results []searchResultItem
|
||||
var totalFound int
|
||||
|
||||
if searchType == "all" {
|
||||
// Search all 9 collections
|
||||
results, totalFound = h.multiSearch(ctx, nil, query, page, limit)
|
||||
} else if len(requestedTypes) == 1 {
|
||||
// Single collection — use singleSearch for efficiency
|
||||
results, totalFound = h.singleSearch(ctx, requestedTypes[0], query, page, limit)
|
||||
} else {
|
||||
// Multiple collections — use multiSearch with a subset
|
||||
results, totalFound = h.multiSearch(ctx, requestedTypes, query, page, limit)
|
||||
}
|
||||
|
||||
totalPages := int(math.Ceil(float64(totalFound) / float64(limit)))
|
||||
|
||||
resp := searchResponse{
|
||||
Query: query,
|
||||
Type: searchType,
|
||||
Results: results,
|
||||
Total: totalFound,
|
||||
Page: page,
|
||||
Limit: limit,
|
||||
TotalPages: totalPages,
|
||||
}
|
||||
|
||||
response.JSON(w, http.StatusOK, resp)
|
||||
}
|
||||
|
||||
// multiSearch searches across all 9 collections using Typesense multi_search,
|
||||
// merges results by text_match score, and returns a single paginated list.
|
||||
// multiSearch searches across multiple collections using Typesense multi_search.
|
||||
// If collections is nil, searches all 9 collections.
|
||||
func (h *SearchHandler) multiSearch(ctx context.Context, collections []string, query string, page, limit int) ([]searchResultItem, int) {
|
||||
if collections == nil {
|
||||
collections = []string{
|
||||
"verses", "prose", "letters", "transcriptions", "glossary",
|
||||
"essays", "journals", "magazines", "other_documents",
|
||||
}
|
||||
}
|
||||
|
||||
exhaustive := true
|
||||
searches := make([]api.MultiSearchCollectionParameters, 0, len(collections))
|
||||
for _, col := range collections {
|
||||
queryBy := collectionQueryByFields[col]
|
||||
weights := collectionQueryByWeights[col]
|
||||
perPage := limit
|
||||
pg := page
|
||||
params := api.MultiSearchCollectionParameters{
|
||||
Collection: col,
|
||||
Q: stringPtr(query),
|
||||
QueryBy: stringPtr(queryBy),
|
||||
PerPage: &perPage,
|
||||
Page: &pg,
|
||||
Prefix: stringPtr("true"),
|
||||
NumTypos: stringPtr("2"),
|
||||
ExhaustiveSearch: &exhaustive,
|
||||
}
|
||||
if weights != "" {
|
||||
params.QueryByWeights = stringPtr(weights)
|
||||
}
|
||||
searches = append(searches, params)
|
||||
}
|
||||
|
||||
searchParams := api.MultiSearchSearchesParameter{
|
||||
Searches: searches,
|
||||
}
|
||||
|
||||
commonParams := &api.MultiSearchParams{}
|
||||
|
||||
msResult, err := h.ts.Client.MultiSearch.Perform(ctx, commonParams, searchParams)
|
||||
if err != nil {
|
||||
return []searchResultItem{}, 0
|
||||
}
|
||||
|
||||
allItems := []searchResultItem{}
|
||||
totalFound := 0
|
||||
|
||||
for i, sr := range msResult.Results {
|
||||
collectionName := collections[i]
|
||||
if sr.Found != nil {
|
||||
totalFound += *sr.Found
|
||||
}
|
||||
if sr.Hits == nil {
|
||||
continue
|
||||
}
|
||||
for _, hit := range *sr.Hits {
|
||||
item := hitToResultItem(collectionName, &hit)
|
||||
allItems = append(allItems, item)
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by text_match score descending
|
||||
sort.Slice(allItems, func(i, j int) bool {
|
||||
return allItems[i].score > allItems[j].score
|
||||
})
|
||||
|
||||
// Paginate the merged results: since each sub-search already returned `limit`
|
||||
// results for the given page, the merged set has at most 9*limit items.
|
||||
// We take the top `limit` after sorting.
|
||||
if len(allItems) > limit {
|
||||
allItems = allItems[:limit]
|
||||
}
|
||||
|
||||
return allItems, totalFound
|
||||
}
|
||||
|
||||
// singleSearch searches a single collection.
|
||||
func (h *SearchHandler) singleSearch(ctx context.Context, collection, query string, page, limit int) ([]searchResultItem, int) {
|
||||
queryBy, ok := collectionQueryByFields[collection]
|
||||
if !ok {
|
||||
return []searchResultItem{}, 0
|
||||
}
|
||||
|
||||
exhaustive := true
|
||||
params := &api.SearchCollectionParams{
|
||||
Q: stringPtr(query),
|
||||
QueryBy: stringPtr(queryBy),
|
||||
Page: &page,
|
||||
PerPage: &limit,
|
||||
Prefix: stringPtr("true"),
|
||||
NumTypos: stringPtr("2"),
|
||||
ExhaustiveSearch: &exhaustive,
|
||||
}
|
||||
if weights := collectionQueryByWeights[collection]; weights != "" {
|
||||
params.QueryByWeights = stringPtr(weights)
|
||||
}
|
||||
|
||||
sr, err := h.ts.Client.Collection(collection).Documents().Search(ctx, params)
|
||||
if err != nil {
|
||||
return []searchResultItem{}, 0
|
||||
}
|
||||
|
||||
totalFound := 0
|
||||
if sr.Found != nil {
|
||||
totalFound = *sr.Found
|
||||
}
|
||||
|
||||
items := []searchResultItem{}
|
||||
if sr.Hits != nil {
|
||||
for _, hit := range *sr.Hits {
|
||||
item := hitToResultItem(collection, &hit)
|
||||
items = append(items, item)
|
||||
}
|
||||
}
|
||||
|
||||
return items, totalFound
|
||||
}
|
||||
|
||||
// hitToResultItem converts a Typesense SearchResultHit into our API result shape.
|
||||
func hitToResultItem(collection string, hit *api.SearchResultHit) searchResultItem {
|
||||
doc := make(map[string]interface{})
|
||||
if hit.Document != nil {
|
||||
doc = *hit.Document
|
||||
}
|
||||
|
||||
var score int64
|
||||
if hit.TextMatch != nil {
|
||||
score = *hit.TextMatch
|
||||
}
|
||||
|
||||
item := searchResultItem{
|
||||
Collection: collection,
|
||||
URL: getStringField(doc, "url"),
|
||||
score: score,
|
||||
}
|
||||
|
||||
// Extract title based on collection type
|
||||
switch collection {
|
||||
case "verses":
|
||||
item.Title = getStringField(doc, "reference")
|
||||
item.Slug = slugFromURL(item.URL)
|
||||
item.Meta = map[string]interface{}{
|
||||
"book": getStringField(doc, "book"),
|
||||
"reference": getStringField(doc, "reference"),
|
||||
}
|
||||
case "prose":
|
||||
item.Title = getStringField(doc, "title")
|
||||
item.Slug = slugFromURL(item.URL)
|
||||
item.Meta = map[string]interface{}{
|
||||
"book": getStringField(doc, "book"),
|
||||
"type": getStringField(doc, "type"),
|
||||
}
|
||||
case "letters":
|
||||
recipient := getStringField(doc, "recipient_display")
|
||||
if recipient != "" {
|
||||
item.Title = "Letter to " + recipient
|
||||
} else {
|
||||
item.Title = "Letter"
|
||||
}
|
||||
item.Slug = slugFromURL(item.URL)
|
||||
item.Meta = map[string]interface{}{
|
||||
"recipient_display": recipient,
|
||||
}
|
||||
if date, ok := doc["date"]; ok {
|
||||
item.Meta["date"] = date
|
||||
}
|
||||
if location, ok := doc["location"]; ok {
|
||||
item.Meta["location"] = location
|
||||
}
|
||||
case "transcriptions":
|
||||
item.Title = getStringField(doc, "name")
|
||||
item.Slug = slugFromURL(item.URL)
|
||||
item.Meta = map[string]interface{}{
|
||||
"type": getStringField(doc, "type"),
|
||||
}
|
||||
if place, ok := doc["place"]; ok {
|
||||
item.Meta["place"] = place
|
||||
}
|
||||
if date, ok := doc["date"]; ok {
|
||||
item.Meta["date"] = date
|
||||
}
|
||||
if ref, ok := doc["reference"]; ok {
|
||||
item.Meta["reference"] = ref
|
||||
}
|
||||
case "glossary":
|
||||
item.Title = getStringField(doc, "term")
|
||||
item.Slug = getStringField(doc, "term")
|
||||
item.Meta = map[string]interface{}{
|
||||
"first_letter": getStringField(doc, "first_letter"),
|
||||
}
|
||||
case "essays":
|
||||
item.Title = getStringField(doc, "title")
|
||||
item.Slug = slugFromURL(item.URL)
|
||||
item.Meta = map[string]interface{}{
|
||||
"author": getStringField(doc, "author"),
|
||||
}
|
||||
if year, ok := doc["year"]; ok {
|
||||
item.Meta["year"] = year
|
||||
}
|
||||
case "journals":
|
||||
item.Title = getStringField(doc, "title")
|
||||
item.Slug = slugFromURL(item.URL)
|
||||
item.Meta = map[string]interface{}{
|
||||
"book": getStringField(doc, "book"),
|
||||
}
|
||||
case "magazines":
|
||||
item.Title = getStringField(doc, "title")
|
||||
item.Slug = slugFromURL(item.URL)
|
||||
item.Meta = map[string]interface{}{
|
||||
"book": getStringField(doc, "book"),
|
||||
}
|
||||
case "other_documents":
|
||||
item.Title = getStringField(doc, "title")
|
||||
item.Slug = slugFromURL(item.URL)
|
||||
item.Meta = map[string]interface{}{
|
||||
"book": getStringField(doc, "book"),
|
||||
}
|
||||
}
|
||||
|
||||
// Extract snippet from highlights
|
||||
item.Snippet = extractSnippet(hit, collection)
|
||||
|
||||
return item
|
||||
}
|
||||
|
||||
// extractSnippet builds a snippet string from Typesense highlight data.
|
||||
// It prefers the newer highlight map, falling back to the deprecated highlights array.
|
||||
func extractSnippet(hit *api.SearchResultHit, collection string) string {
|
||||
// Try the highlights array (deprecated but widely available)
|
||||
if hit.Highlights != nil && len(*hit.Highlights) > 0 {
|
||||
for _, hl := range *hit.Highlights {
|
||||
if hl.Snippet != nil && *hl.Snippet != "" {
|
||||
return *hl.Snippet
|
||||
}
|
||||
if hl.Snippets != nil && len(*hl.Snippets) > 0 {
|
||||
return (*hl.Snippets)[0]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Try the highlight map (v2 style)
|
||||
if hit.Highlight != nil {
|
||||
hlMap := *hit.Highlight
|
||||
// Prioritize content/text fields for snippet
|
||||
priorityFields := snippetPriorityFields(collection)
|
||||
for _, field := range priorityFields {
|
||||
if val, ok := hlMap[field]; ok {
|
||||
if s := extractHighlightValue(val); s != "" {
|
||||
return s
|
||||
}
|
||||
}
|
||||
}
|
||||
// Fallback: return any highlighted field
|
||||
for _, val := range hlMap {
|
||||
if s := extractHighlightValue(val); s != "" {
|
||||
return s
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
// snippetPriorityFields returns the fields to check first for snippets, per collection.
|
||||
func snippetPriorityFields(collection string) []string {
|
||||
switch collection {
|
||||
case "verses":
|
||||
return []string{"translation", "purport", "transliteration", "reference"}
|
||||
case "prose":
|
||||
return []string{"content", "title"}
|
||||
case "letters":
|
||||
return []string{"content", "recipient_display"}
|
||||
case "transcriptions":
|
||||
return []string{"content", "name", "reference"}
|
||||
case "glossary":
|
||||
return []string{"definition", "term"}
|
||||
case "essays":
|
||||
return []string{"content", "title"}
|
||||
case "journals", "magazines", "other_documents":
|
||||
return []string{"content", "title"}
|
||||
default:
|
||||
return []string{"content", "title"}
|
||||
}
|
||||
}
|
||||
|
||||
// extractHighlightValue tries to pull a string snippet from a highlight value,
|
||||
// which may be a string, a map with "snippet" or "matched_tokens", etc.
|
||||
func extractHighlightValue(val interface{}) string {
|
||||
switch v := val.(type) {
|
||||
case string:
|
||||
if v != "" {
|
||||
return v
|
||||
}
|
||||
case map[string]interface{}:
|
||||
if snippet, ok := v["snippet"].(string); ok && snippet != "" {
|
||||
return snippet
|
||||
}
|
||||
if value, ok := v["value"].(string); ok && value != "" {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// getStringField safely extracts a string value from a map.
|
||||
func getStringField(doc map[string]interface{}, key string) string {
|
||||
if val, ok := doc[key]; ok {
|
||||
if s, ok := val.(string); ok {
|
||||
return s
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// slugFromURL extracts the last path segment from a URL as a slug.
|
||||
func slugFromURL(u string) string {
|
||||
u = strings.TrimRight(u, "/")
|
||||
parts := strings.Split(u, "/")
|
||||
if len(parts) > 0 {
|
||||
return parts[len(parts)-1]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// parseIntParam parses an integer from a query param string with a default fallback.
|
||||
func parseIntParam(s string, defaultVal int) int {
|
||||
if s == "" {
|
||||
return defaultVal
|
||||
}
|
||||
v, err := strconv.Atoi(s)
|
||||
if err != nil {
|
||||
return defaultVal
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// stringPtr returns a pointer to a string value.
|
||||
func stringPtr(s string) *string {
|
||||
return &s
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"github.com/typesense/typesense-go/v2/typesense"
|
||||
)
|
||||
|
||||
// TypesenseClient wraps the Typesense client for search operations.
|
||||
type TypesenseClient struct {
|
||||
Client *typesense.Client
|
||||
}
|
||||
|
||||
// NewTypesenseClient creates a new Typesense client with the given URL and API key.
|
||||
func NewTypesenseClient(url, apiKey string) *TypesenseClient {
|
||||
client := typesense.NewClient(
|
||||
typesense.WithServer(url),
|
||||
typesense.WithAPIKey(apiKey),
|
||||
)
|
||||
return &TypesenseClient{Client: client}
|
||||
}
|
||||
@@ -30,6 +30,7 @@ type Server struct {
|
||||
magazineH *handler.MagazineHandler
|
||||
otherDocH *handler.OtherDocumentHandler
|
||||
transcriptionH *handler.TranscriptionHandler
|
||||
searchH *handler.SearchHandler
|
||||
}
|
||||
|
||||
// New creates a new Server with all routes registered.
|
||||
@@ -39,6 +40,9 @@ func New(cfg *config.Config, logger *zap.Logger) *Server {
|
||||
logger.Fatal("failed to connect to MongoDB", zap.Error(err))
|
||||
}
|
||||
|
||||
// Initialize Typesense client
|
||||
tsClient := repository.NewTypesenseClient(cfg.TypesenseURL, cfg.TypesenseAPIKey)
|
||||
|
||||
// Initialize repositories
|
||||
bookRepo := repository.NewBookRepo(mongoDB)
|
||||
letterRepo := repository.NewLetterRepo(mongoDB)
|
||||
@@ -65,6 +69,7 @@ func New(cfg *config.Config, logger *zap.Logger) *Server {
|
||||
magazineH: handler.NewMagazineHandler(magazineRepo),
|
||||
otherDocH: handler.NewOtherDocumentHandler(otherDocRepo),
|
||||
transcriptionH: handler.NewTranscriptionHandler(transcriptionRepo),
|
||||
searchH: handler.NewSearchHandler(tsClient),
|
||||
}
|
||||
|
||||
s.setupRouter()
|
||||
@@ -297,10 +302,8 @@ func (s *Server) setupRouter() {
|
||||
r.Get("/{slug}", s.transcriptionH.GetBySlug)
|
||||
})
|
||||
|
||||
// Search (Meilisearch)
|
||||
r.Route("/search", func(r chi.Router) {
|
||||
// TODO: register search handlers
|
||||
})
|
||||
// Search (Typesense)
|
||||
r.Get("/search", s.searchH.Search)
|
||||
|
||||
// User routes (PostgreSQL)
|
||||
r.Route("/auth", func(r chi.Router) {
|
||||
|
||||
Reference in New Issue
Block a user