package main

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"net/url"
	"os"
	"regexp"
	"strings"
	"time"

	"github.com/prabhupadasays/spsays-bff/internal/config"
	"github.com/prabhupadasays/spsays-bff/internal/repository"
	"github.com/prabhupadasays/spsays-bff/pkg/revalidate"
	"github.com/typesense/typesense-go/v2/typesense"
	"github.com/typesense/typesense-go/v2/typesense/api"
	"go.mongodb.org/mongo-driver/v2/bson"
	"go.uber.org/zap"
)

// CLI flags
var (
	fullReindex = flag.Bool("full", false, "Drop all collections and reindex everything")
	onlyFlag    = flag.String("only", "", "Comma-separated list of collections to index (e.g., verses,letters)")
	noDrop      = flag.Bool("no-drop", false, "Do not drop the collection first — upsert into the existing one (resume a partial import)")
)

// All supported collection names
var allCollections = []string{
	"verses", "prose", "letters", "transcriptions", "glossary",
	"essays", "journals", "magazines", "other_documents",
}

// hyphenSeparators is applied to every collection so that hyphenated
// Sanskrit/Vedic terms (e.g. "Dīkṣā-guru", "Hare-Kṛṣṇa", "sarva-dharman")
// are tokenised into individual words. Without this, a query like
// "Diksa guru" only matches one of the two query tokens against the
// indexed term "diksa-guru", which dramatically lowers its score.
var hyphenSeparators = []string{"-"}

// HTML stripping helper
var htmlTagRe = regexp.MustCompile(`<[^>]*>`)

func stripHTML(s string) string {
	return htmlTagRe.ReplaceAllString(s, "")
}

func joinParagraphs(paragraphs []string) string {
	var sb strings.Builder
	for i, p := range paragraphs {
		if i > 0 {
			sb.WriteString(" ")
		}
		sb.WriteString(stripHTML(p))
	}
	return sb.String()
}

// IAST diacritical → ASCII mapping for typo-tolerant search.
// Allows "Krsna" to match "Kṛṣṇa", "Prabhupada" to match "Prabhupāda", etc.
var diacriticReplacer = strings.NewReplacer(
	"ā", "a", "Ā", "A",
	"ī", "i", "Ī", "I",
	"ū", "u", "Ū", "U",
	"ṛ", "r", "Ṛ", "R",
	"ṝ", "r", "Ṝ", "R",
	"ṣ", "s", "Ṣ", "S",
	"ś", "s", "Ś", "S",
	"ṭ", "t", "Ṭ", "T",
	"ḍ", "d", "Ḍ", "D",
	"ṇ", "n", "Ṇ", "N",
	"ñ", "n", "Ñ", "N",
	"ṅ", "n", "Ṅ", "N",
	"ṁ", "m", "Ṁ", "M",
	"ḥ", "h", "Ḥ", "H",
	"ḷ", "l", "Ḷ", "L",
	"é", "e", "É", "E",
	"è", "e",
	"ö", "o",
	"ü", "u",
	"ä", "a",
	// Common ligatures
	"æ", "ae", "œ", "oe",
)

// stripDiacritics converts IAST/Unicode diacritical characters to plain ASCII.
func stripDiacritics(s string) string {
	return diacriticReplacer.Replace(s)
}

// Pointer helpers
func stringPtr(s string) *string { return &s }
func boolPtr(b bool) *bool       { return &b }

func main() {
	flag.Parse()

	// Load config
	cfg, err := config.Load()
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to load config: %v\n", err)
		os.Exit(1)
	}

	// Initialize logger
	logger, _ := zap.NewDevelopment()
	defer logger.Sync()
	sugar := logger.Sugar()

	// Connect to MongoDB
	mongoDB, err := repository.NewMongoDB(cfg.MongoURI, cfg.MongoDB, logger)
	if err != nil {
		sugar.Fatalf("failed to connect to MongoDB: %v", err)
	}
	defer mongoDB.Close()

	// Create Typesense client.
	// Default connection timeout is 5s — too short for large content imports
	// (magazines and other_documents can ship multi-MB batches).
	tsClient := typesense.NewClient(
		typesense.WithServer(cfg.TypesenseURL),
		typesense.WithAPIKey(cfg.TypesenseAPIKey),
		typesense.WithConnectionTimeout(5*time.Minute),
	)

	// Determine which collections to index
	collectionsToIndex := allCollections
	if *onlyFlag != "" {
		collectionsToIndex = strings.Split(*onlyFlag, ",")
		for i := range collectionsToIndex {
			collectionsToIndex[i] = strings.TrimSpace(collectionsToIndex[i])
		}
	}

	// Get existing collections
	existingCollections := make(map[string]bool)
	existing, err := tsClient.Collections().Retrieve(context.Background())
	if err != nil {
		sugar.Warnf("could not retrieve existing collections: %v", err)
	} else {
		for _, col := range existing {
			existingCollections[col.Name] = true
		}
	}

	for _, name := range collectionsToIndex {
		sugar.Infof("=== Processing collection: %s ===", name)

		shouldDrop := (*fullReindex || *onlyFlag != "") && !*noDrop
		// On a bare run (no targeting), skip collections that already exist.
		// An explicit --full / --only / --no-drop always (re)processes them.
		targeted := *fullReindex || *onlyFlag != "" || *noDrop
		if !targeted && existingCollections[name] {
			sugar.Infof("[%s] Collection already exists, skipping (use --full to force reindex)", name)
			continue
		}

		// Drop if exists and we need to recreate
		if existingCollections[name] && shouldDrop {
			sugar.Infof("[%s] Dropping existing collection...", name)
			_, err := tsClient.Collection(name).Delete(context.Background())
			if err != nil {
				sugar.Warnf("[%s] Failed to drop collection: %v", name, err)
			}
		}

		// Create schema and index
		switch name {
		case "verses":
			indexVerses(sugar, tsClient, mongoDB)
		case "prose":
			indexProse(sugar, tsClient, mongoDB)
		case "letters":
			indexLetters(sugar, tsClient, mongoDB)
		case "transcriptions":
			indexTranscriptions(sugar, tsClient, mongoDB)
		case "glossary":
			indexGlossary(sugar, tsClient, mongoDB)
		case "essays":
			indexEssays(sugar, tsClient, mongoDB)
		case "journals":
			indexJournals(sugar, tsClient, mongoDB)
		case "magazines":
			indexMagazines(sugar, tsClient, mongoDB)
		case "other_documents":
			indexOtherDocuments(sugar, tsClient, mongoDB)
		default:
			sugar.Warnf("Unknown collection: %s", name)
		}
	}

	sugar.Info("=== Indexing complete ===")

	// Content changed → refresh the frontend sitemap cache (synchronous: this
	// CLI exits right after, so a background goroutine wouldn't run). No-op
	// when FRONTEND_URL / REVALIDATE_SECRET are unset.
	if err := revalidate.New(cfg.FrontendURL, cfg.RevalidateSecret, logger).TriggerSync("sitemap"); err != nil {
		sugar.Warnf("sitemap revalidation ping failed: %v", err)
	} else if cfg.RevalidateSecret != "" {
		sugar.Info("sitemap revalidation pinged")
	}
}

// importBatch imports documents into Typesense in batches of batchSize.
func importBatch(sugar *zap.SugaredLogger, tsClient *typesense.Client, collectionName string, docs []interface{}, batchSize int) {
	total := len(docs)
	// Upsert (not create) so a re-run over an existing collection is
	// idempotent and additive — essential for resuming a partial import
	// when a flaky link drops batches mid-run.
	action := "upsert"
	for i := 0; i < total; i += batchSize {
		end := i + batchSize
		if end > total {
			end = total
		}
		batch := docs[i:end]
		// Retry the batch on transient transport errors (flaky networks can
		// drop large TLS writes — "broken pipe" / "bad record MAC").
		var responses []*api.ImportDocumentResponse
		var err error
		for attempt := 1; attempt <= 8; attempt++ {
			responses, err = tsClient.Collection(collectionName).Documents().Import(context.Background(), batch, &api.ImportDocumentsParams{
				Action: &action,
			})
			if err == nil {
				break
			}
			sugar.Warnf("[%s] batch %d-%d attempt %d/5 failed: %v", collectionName, i, end, attempt, err)
			time.Sleep(time.Duration(attempt) * time.Second)
		}
		if err != nil {
			sugar.Errorf("[%s] Import error at batch %d-%d (gave up): %v", collectionName, i, end, err)
			continue
		}
		// Check for per-document errors
		errCount := 0
		for _, resp := range responses {
			if !resp.Success {
				errCount++
				if errCount <= 3 {
					sugar.Warnf("[%s] Document import error: %s", collectionName, resp.Error)
				}
			}
		}
		if errCount > 3 {
			sugar.Warnf("[%s] ... and %d more document errors in this batch", collectionName, errCount-3)
		}
		sugar.Infof("[%s] %d/%d indexed...", collectionName, end, total)
	}
}

// ==================== VERSES ====================

func buildVerseURL(book string, division *string, chapter, verse int) string {
	switch book {
	case "bg":
		return fmt.Sprintf("/books/bg/%d/%d", chapter, verse)
	case "sb":
		if division != nil {
			return fmt.Sprintf("/books/sb/%s/%d/%d", *division, chapter, verse)
		}
		return fmt.Sprintf("/books/sb/%d/%d", chapter, verse)
	case "cc":
		if division != nil {
			return fmt.Sprintf("/books/cc/%s/%d/%d", *division, chapter, verse)
		}
		return fmt.Sprintf("/books/cc/%d/%d", chapter, verse)
	case "sb1962":
		return fmt.Sprintf("/books/sb1962/%d/%d", chapter, verse)
	case "iso":
		return fmt.Sprintf("/books/iso/%d", chapter)
	case "noi":
		return fmt.Sprintf("/books/noi/%d", chapter)
	case "bs":
		return fmt.Sprintf("/books/bs/5/%d", verse)
	case "mm":
		return fmt.Sprintf("/books/mm/%d/%d", chapter, verse)
	case "nbs":
		return fmt.Sprintf("/books/nbs/%d/%d", chapter, verse)
	default:
		return fmt.Sprintf("/books/%s/%d/%d", book, chapter, verse)
	}
}

func indexVerses(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *repository.MongoDB) {
	defaultSortingField := "chapter"
	schema := &api.CollectionSchema{
		Name: "verses",
		Fields: []api.Field{
			{Name: "book", Type: "string", Facet: boolPtr(true)},
			{Name: "division", Type: "string", Facet: boolPtr(true), Optional: boolPtr(true)},
			{Name: "chapter", Type: "int32", Facet: boolPtr(true)},
			{Name: "verse", Type: "int32"},
			{Name: "reference", Type: "string"},
			{Name: "book_title", Type: "string"},
			{Name: "translation", Type: "string"},
			{Name: "purport", Type: "string"},
			{Name: "transliteration", Type: "string"},
			{Name: "translation_ascii", Type: "string"},
			{Name: "purport_ascii", Type: "string"},
			{Name: "transliteration_ascii", Type: "string"},
			{Name: "url", Type: "string"},
		},
		DefaultSortingField: &defaultSortingField,
		TokenSeparators:     &hyphenSeparators,
	}

	_, err := tsClient.Collections().Create(context.Background(), schema)
	if err != nil {
		sugar.Fatalf("[verses] Failed to create collection: %v", err)
	}
	sugar.Info("[verses] Collection created")

	// Read from MongoDB
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
	defer cancel()

	coll := mongoDB.Collection("verses")
	cursor, err := coll.Find(ctx, bson.M{})
	if err != nil {
		sugar.Fatalf("[verses] Failed to query MongoDB: %v", err)
	}
	defer cursor.Close(ctx)

	var docs []interface{}
	for cursor.Next(ctx) {
		var raw bson.M
		if err := cursor.Decode(&raw); err != nil {
			sugar.Warnf("[verses] Decode error: %v", err)
			continue
		}

		id := objectIDToHex(raw["_id"])
		book, _ := raw["book"].(string)
		var division *string
		if d, ok := raw["division"].(string); ok && d != "" {
			division = &d
		}
		chapter := toInt32(raw["chapter"])
		verse := toInt32(raw["verse"])
		reference, _ := raw["reference"].(string)
		bookTitle, _ := raw["bookTitle"].(string)
		translation := stripHTML(toString(raw["translation"]))
		transliteration := toString(raw["transliteration"])

		// Join purport paragraphs
		purport := ""
		if paragraphs, ok := raw["purport"].(bson.A); ok {
			strs := bsonAToStrings(paragraphs)
			purport = joinParagraphs(strs)
		}

		verseURL := buildVerseURL(book, division, int(chapter), int(verse))

		doc := map[string]interface{}{
			"id":                    id,
			"book":                  book,
			"chapter":               chapter,
			"verse":                 verse,
			"reference":             reference,
			"book_title":            bookTitle,
			"translation":           translation,
			"purport":               purport,
			"transliteration":       transliteration,
			"translation_ascii":     stripDiacritics(translation),
			"purport_ascii":         stripDiacritics(purport),
			"transliteration_ascii": stripDiacritics(transliteration),
			"url":                   verseURL,
		}
		if division != nil {
			doc["division"] = *division
		}

		docs = append(docs, doc)
	}

	sugar.Infof("[verses] Read %d documents from MongoDB", len(docs))
	importBatch(sugar, tsClient, "verses", docs, 500)
}

// ==================== PROSE ====================

func indexProse(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *repository.MongoDB) {
	schema := &api.CollectionSchema{
		Name: "prose",
		Fields: []api.Field{
			{Name: "book", Type: "string", Facet: boolPtr(true)},
			{Name: "title", Type: "string"},
			{Name: "content", Type: "string"},
			{Name: "title_ascii", Type: "string"},
			{Name: "content_ascii", Type: "string"},
			{Name: "type", Type: "string", Facet: boolPtr(true)},
			{Name: "url", Type: "string"},
		},
		TokenSeparators: &hyphenSeparators,
	}

	_, err := tsClient.Collections().Create(context.Background(), schema)
	if err != nil {
		// An already-existing collection is fine: with --no-drop we resume an
		// upsert into it (the schema is unchanged).
		sugar.Warnf("[prose] Create collection: %v (continuing — assuming it exists)", err)
	} else {
		sugar.Info("[prose] Collection created")
	}

	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
	defer cancel()

	coll := mongoDB.Collection("prose_chapters")
	cursor, err := coll.Find(ctx, bson.M{})
	if err != nil {
		sugar.Fatalf("[prose] Failed to query MongoDB: %v", err)
	}
	defer cursor.Close(ctx)

	var docs []interface{}
	for cursor.Next(ctx) {
		var raw bson.M
		if err := cursor.Decode(&raw); err != nil {
			sugar.Warnf("[prose] Decode error: %v", err)
			continue
		}

		id := objectIDToHex(raw["_id"])
		book, _ := raw["book"].(string)
		title, _ := raw["title"].(string)
		docType, _ := raw["type"].(string)
		slug, _ := raw["slug"].(string)
		volume := toInt32(raw["volume"])

		content := ""
		if paragraphs, ok := raw["paragraphs"].(bson.A); ok {
			strs := bsonAToStrings(paragraphs)
			content = joinParagraphs(strs)
		}

		// Build URL: KB uses /books/kb/{volume}/{slug}; chapter summaries map
		// their "<lila>-<n>" slug to the chapter landing page; others use
		// /books/{book}/{slug}.
		var proseURL string
		switch {
		case book == "kb":
			proseURL = fmt.Sprintf("/books/kb/%d/%s", volume, slug)
		case docType == "chapter-summary":
			if parts := strings.SplitN(slug, "-", 2); len(parts) == 2 {
				proseURL = fmt.Sprintf("/books/%s/%s/%s", book, parts[0], parts[1])
			} else {
				proseURL = fmt.Sprintf("/books/%s/%s", book, slug)
			}
		default:
			proseURL = fmt.Sprintf("/books/%s/%s", book, slug)
		}

		doc := map[string]interface{}{
			"id":            id,
			"book":          book,
			"title":         title,
			"content":       content,
			"title_ascii":   stripDiacritics(title),
			"content_ascii": stripDiacritics(content),
			"type":          docType,
			"url":           proseURL,
		}
		docs = append(docs, doc)
	}

	sugar.Infof("[prose] Read %d documents from MongoDB", len(docs))
	// Small batches keep each import POST modest in size — large multi-MB
	// bodies (front-matter intro segments) are prone to drops on flaky links.
	importBatch(sugar, tsClient, "prose", docs, 10)
}

// ==================== LETTERS ====================

func indexLetters(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *repository.MongoDB) {
	schema := &api.CollectionSchema{
		Name: "letters",
		Fields: []api.Field{
			{Name: "recipient_display", Type: "string"},
			{Name: "recipients", Type: "string[]", Facet: boolPtr(true)},
			{Name: "date", Type: "string", Optional: boolPtr(true)},
			{Name: "year", Type: "int32", Facet: boolPtr(true), Optional: boolPtr(true)},
			{Name: "location", Type: "string", Facet: boolPtr(true), Optional: boolPtr(true)},
			{Name: "content", Type: "string"},
			{Name: "content_ascii", Type: "string"},
			{Name: "recipient_display_ascii", Type: "string"},
			{Name: "url", Type: "string"},
		},
		TokenSeparators: &hyphenSeparators,
	}

	_, err := tsClient.Collections().Create(context.Background(), schema)
	if err != nil {
		sugar.Fatalf("[letters] Failed to create collection: %v", err)
	}
	sugar.Info("[letters] Collection created")

	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
	defer cancel()

	coll := mongoDB.Collection("letters")
	cursor, err := coll.Find(ctx, bson.M{})
	if err != nil {
		sugar.Fatalf("[letters] Failed to query MongoDB: %v", err)
	}
	defer cursor.Close(ctx)

	var docs []interface{}
	for cursor.Next(ctx) {
		var raw bson.M
		if err := cursor.Decode(&raw); err != nil {
			sugar.Warnf("[letters] Decode error: %v", err)
			continue
		}

		id := objectIDToHex(raw["_id"])
		slug, _ := raw["slug"].(string)
		recipientDisplay, _ := raw["recipientDisplay"].(string)

		recipients := []string{}
		if arr, ok := raw["recipients"].(bson.A); ok {
			recipients = bsonAToStrings(arr)
		}

		content := ""
		if paragraphs, ok := raw["paragraphs"].(bson.A); ok {
			strs := bsonAToStrings(paragraphs)
			content = joinParagraphs(strs)
		}

		doc := map[string]interface{}{
			"id":                      id,
			"recipient_display":       recipientDisplay,
			"recipients":              recipients,
			"content":                 content,
			"content_ascii":           stripDiacritics(content),
			"recipient_display_ascii": stripDiacritics(recipientDisplay),
			"url":                     fmt.Sprintf("/letters/%s", slug),
		}

		if date, ok := raw["date"].(string); ok && date != "" {
			doc["date"] = date
		}
		if year, ok := toOptionalInt32(raw["year"]); ok {
			doc["year"] = year
		}
		if location, ok := raw["location"].(string); ok && location != "" {
			doc["location"] = location
		}

		docs = append(docs, doc)
	}

	sugar.Infof("[letters] Read %d documents from MongoDB", len(docs))
	importBatch(sugar, tsClient, "letters", docs, 500)
}

// ==================== TRANSCRIPTIONS ====================

func indexTranscriptions(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *repository.MongoDB) {
	schema := &api.CollectionSchema{
		Name: "transcriptions",
		Fields: []api.Field{
			{Name: "name", Type: "string"},
			{Name: "type", Type: "string", Facet: boolPtr(true)},
			{Name: "date", Type: "string", Optional: boolPtr(true)},
			{Name: "year", Type: "int32", Facet: boolPtr(true), Optional: boolPtr(true)},
			{Name: "place", Type: "string", Facet: boolPtr(true), Optional: boolPtr(true)},
			{Name: "reference", Type: "string", Optional: boolPtr(true)},
			{Name: "participants", Type: "string[]", Facet: boolPtr(true)},
			{Name: "content", Type: "string"},
			{Name: "name_ascii", Type: "string"},
			{Name: "content_ascii", Type: "string"},
			{Name: "url", Type: "string"},
		},
		TokenSeparators: &hyphenSeparators,
	}

	_, err := tsClient.Collections().Create(context.Background(), schema)
	if err != nil {
		sugar.Fatalf("[transcriptions] Failed to create collection: %v", err)
	}
	sugar.Info("[transcriptions] Collection created")

	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
	defer cancel()

	coll := mongoDB.Collection("transcriptions")
	cursor, err := coll.Find(ctx, bson.M{"isPublished": true})
	if err != nil {
		sugar.Fatalf("[transcriptions] Failed to query MongoDB: %v", err)
	}
	defer cursor.Close(ctx)

	var docs []interface{}
	for cursor.Next(ctx) {
		var raw bson.M
		if err := cursor.Decode(&raw); err != nil {
			sugar.Warnf("[transcriptions] Decode error: %v", err)
			continue
		}

		id := objectIDToHex(raw["_id"])
		slug, _ := raw["slug"].(string)
		name, _ := raw["name"].(string)
		docType, _ := raw["type"].(string)

		participants := []string{}
		if arr, ok := raw["participants"].(bson.A); ok {
			participants = bsonAToStrings(arr)
		}

		// Build content from content blocks or contentJson
		content := extractTranscriptionContent(raw)

		doc := map[string]interface{}{
			"id":            id,
			"name":          name,
			"type":          docType,
			"participants":  participants,
			"content":       content,
			"name_ascii":    stripDiacritics(name),
			"content_ascii": stripDiacritics(content),
			"url":           fmt.Sprintf("/transcriptions/%s", slug),
		}

		if date, ok := raw["date"].(string); ok && date != "" {
			doc["date"] = date
		}
		if year, ok := toOptionalInt32(raw["year"]); ok {
			doc["year"] = year
		}
		if place, ok := raw["place"].(string); ok && place != "" {
			doc["place"] = place
		}
		if ref, ok := raw["reference"].(string); ok && ref != "" {
			doc["reference"] = ref
		}

		docs = append(docs, doc)
	}

	sugar.Infof("[transcriptions] Read %d documents from MongoDB", len(docs))
	importBatch(sugar, tsClient, "transcriptions", docs, 500)
}

func extractTranscriptionContent(raw bson.M) string {
	// Try content array first
	if contentArr, ok := raw["content"].(bson.A); ok && len(contentArr) > 0 {
		var texts []string
		for _, item := range contentArr {
			if block, ok := item.(bson.M); ok {
				if text, ok := block["text"].(string); ok && text != "" {
					texts = append(texts, stripHTML(text))
				}
			}
		}
		if len(texts) > 0 {
			return strings.Join(texts, " ")
		}
	}

	// Fallback to contentJson
	if contentJson, ok := raw["contentJson"].(string); ok && contentJson != "" {
		var blocks []struct {
			Text string `json:"text"`
		}
		if err := json.Unmarshal([]byte(contentJson), &blocks); err == nil {
			var texts []string
			for _, b := range blocks {
				if b.Text != "" {
					texts = append(texts, stripHTML(b.Text))
				}
			}
			return strings.Join(texts, " ")
		}
	}

	return ""
}

// ==================== GLOSSARY ====================

func indexGlossary(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *repository.MongoDB) {
	schema := &api.CollectionSchema{
		Name: "glossary",
		Fields: []api.Field{
			{Name: "term", Type: "string"},
			{Name: "definition", Type: "string"},
			{Name: "term_ascii", Type: "string"},
			{Name: "definition_ascii", Type: "string"},
			{Name: "first_letter", Type: "string", Facet: boolPtr(true)},
			{Name: "url", Type: "string"},
		},
		TokenSeparators: &hyphenSeparators,
	}

	_, err := tsClient.Collections().Create(context.Background(), schema)
	if err != nil {
		sugar.Fatalf("[glossary] Failed to create collection: %v", err)
	}
	sugar.Info("[glossary] Collection created")

	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
	defer cancel()

	coll := mongoDB.Collection("glossary")
	cursor, err := coll.Find(ctx, bson.M{})
	if err != nil {
		sugar.Fatalf("[glossary] Failed to query MongoDB: %v", err)
	}
	defer cursor.Close(ctx)

	var docs []interface{}
	for cursor.Next(ctx) {
		var raw bson.M
		if err := cursor.Decode(&raw); err != nil {
			sugar.Warnf("[glossary] Decode error: %v", err)
			continue
		}

		id := objectIDToHex(raw["_id"])
		term, _ := raw["term"].(string)
		definition := stripHTML(toString(raw["definition"]))

		firstLetter := getFirstLetter(term)
		encodedTerm := url.QueryEscape(term)

		doc := map[string]interface{}{
			"id":               id,
			"term":             term,
			"definition":       definition,
			"term_ascii":       stripDiacritics(term),
			"definition_ascii": stripDiacritics(definition),
			"first_letter":     firstLetter,
			"url":              fmt.Sprintf("/glossary/%s?term=%s", strings.ToLower(firstLetter), encodedTerm),
		}
		docs = append(docs, doc)
	}

	sugar.Infof("[glossary] Read %d documents from MongoDB", len(docs))
	importBatch(sugar, tsClient, "glossary", docs, 500)
}

// ==================== ESSAYS ====================

func indexEssays(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *repository.MongoDB) {
	schema := &api.CollectionSchema{
		Name: "essays",
		Fields: []api.Field{
			{Name: "title", Type: "string"},
			{Name: "author", Type: "string"},
			{Name: "year", Type: "int32", Optional: boolPtr(true)},
			{Name: "content", Type: "string"},
			{Name: "title_ascii", Type: "string"},
			{Name: "content_ascii", Type: "string"},
			{Name: "url", Type: "string"},
		},
		TokenSeparators: &hyphenSeparators,
	}

	_, err := tsClient.Collections().Create(context.Background(), schema)
	if err != nil {
		sugar.Fatalf("[essays] Failed to create collection: %v", err)
	}
	sugar.Info("[essays] Collection created")

	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
	defer cancel()

	coll := mongoDB.Collection("essays")
	cursor, err := coll.Find(ctx, bson.M{})
	if err != nil {
		sugar.Fatalf("[essays] Failed to query MongoDB: %v", err)
	}
	defer cursor.Close(ctx)

	var docs []interface{}
	for cursor.Next(ctx) {
		var raw bson.M
		if err := cursor.Decode(&raw); err != nil {
			sugar.Warnf("[essays] Decode error: %v", err)
			continue
		}

		id := objectIDToHex(raw["_id"])
		slug, _ := raw["slug"].(string)
		title, _ := raw["title"].(string)
		author, _ := raw["author"].(string)

		content := ""
		if paragraphs, ok := raw["paragraphs"].(bson.A); ok {
			strs := bsonAToStrings(paragraphs)
			content = joinParagraphs(strs)
		}

		doc := map[string]interface{}{
			"id":            id,
			"title":         title,
			"author":        author,
			"content":       content,
			"title_ascii":   stripDiacritics(title),
			"content_ascii": stripDiacritics(content),
			"url":           fmt.Sprintf("/essays/%s", slug),
		}

		if year, ok := toOptionalInt32(raw["year"]); ok {
			doc["year"] = year
		}

		docs = append(docs, doc)
	}

	sugar.Infof("[essays] Read %d documents from MongoDB", len(docs))
	importBatch(sugar, tsClient, "essays", docs, 500)
}

// ==================== JOURNALS ====================

func indexJournals(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *repository.MongoDB) {
	schema := &api.CollectionSchema{
		Name: "journals",
		Fields: []api.Field{
			{Name: "book", Type: "string", Facet: boolPtr(true)},
			{Name: "title", Type: "string"},
			{Name: "content", Type: "string"},
			{Name: "title_ascii", Type: "string"},
			{Name: "content_ascii", Type: "string"},
			{Name: "url", Type: "string"},
		},
		TokenSeparators: &hyphenSeparators,
	}

	_, err := tsClient.Collections().Create(context.Background(), schema)
	if err != nil {
		sugar.Fatalf("[journals] Failed to create collection: %v", err)
	}
	sugar.Info("[journals] Collection created")

	// Build book slug mapping from journal_meta
	bookSlugs := buildBookSlugMap(sugar, mongoDB, "journal_meta")

	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
	defer cancel()

	coll := mongoDB.Collection("journal_chapters")
	cursor, err := coll.Find(ctx, bson.M{})
	if err != nil {
		sugar.Fatalf("[journals] Failed to query MongoDB: %v", err)
	}
	defer cursor.Close(ctx)

	var docs []interface{}
	for cursor.Next(ctx) {
		var raw bson.M
		if err := cursor.Decode(&raw); err != nil {
			sugar.Warnf("[journals] Decode error: %v", err)
			continue
		}

		id := objectIDToHex(raw["_id"])
		book, _ := raw["book"].(string)
		title, _ := raw["title"].(string)
		chapterSlug, _ := raw["slug"].(string)

		content := ""
		if paragraphs, ok := raw["paragraphs"].(bson.A); ok {
			strs := bsonAToStrings(paragraphs)
			content = joinParagraphs(strs)
		}

		bookSlug := bookSlugs[book]
		if bookSlug == "" {
			bookSlug = book
		}

		doc := map[string]interface{}{
			"id":            id,
			"book":          book,
			"title":         title,
			"content":       content,
			"title_ascii":   stripDiacritics(title),
			"content_ascii": stripDiacritics(content),
			"url":           fmt.Sprintf("/journals/%s/%s", bookSlug, chapterSlug),
		}
		docs = append(docs, doc)
	}

	sugar.Infof("[journals] Read %d documents from MongoDB", len(docs))
	importBatch(sugar, tsClient, "journals", docs, 500)
}

// ==================== MAGAZINES ====================

func indexMagazines(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *repository.MongoDB) {
	schema := &api.CollectionSchema{
		Name: "magazines",
		Fields: []api.Field{
			{Name: "book", Type: "string", Facet: boolPtr(true)},
			{Name: "title", Type: "string"},
			{Name: "content", Type: "string"},
			{Name: "title_ascii", Type: "string"},
			{Name: "content_ascii", Type: "string"},
			{Name: "url", Type: "string"},
		},
		TokenSeparators: &hyphenSeparators,
	}

	_, err := tsClient.Collections().Create(context.Background(), schema)
	if err != nil {
		sugar.Fatalf("[magazines] Failed to create collection: %v", err)
	}
	sugar.Info("[magazines] Collection created")

	// Build book slug mapping from magazine_meta
	bookSlugs := buildBookSlugMap(sugar, mongoDB, "magazine_meta")

	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
	defer cancel()

	coll := mongoDB.Collection("magazine_issues")
	cursor, err := coll.Find(ctx, bson.M{})
	if err != nil {
		sugar.Fatalf("[magazines] Failed to query MongoDB: %v", err)
	}
	defer cursor.Close(ctx)

	var docs []interface{}
	for cursor.Next(ctx) {
		var raw bson.M
		if err := cursor.Decode(&raw); err != nil {
			sugar.Warnf("[magazines] Decode error: %v", err)
			continue
		}

		id := objectIDToHex(raw["_id"])
		book, _ := raw["book"].(string)
		title, _ := raw["title"].(string)
		issueSlug, _ := raw["slug"].(string)

		content := ""
		if paragraphs, ok := raw["paragraphs"].(bson.A); ok {
			strs := bsonAToStrings(paragraphs)
			content = joinParagraphs(strs)
		}

		bookSlug := bookSlugs[book]
		if bookSlug == "" {
			bookSlug = book
		}

		doc := map[string]interface{}{
			"id":            id,
			"book":          book,
			"title":         title,
			"content":       content,
			"title_ascii":   stripDiacritics(title),
			"content_ascii": stripDiacritics(content),
			"url":           fmt.Sprintf("/magazines/%s/%s", bookSlug, issueSlug),
		}
		docs = append(docs, doc)
	}

	sugar.Infof("[magazines] Read %d documents from MongoDB", len(docs))
	// Smaller batch size: each magazine issue can have very large paragraphs
	importBatch(sugar, tsClient, "magazines", docs, 25)
}

// ==================== OTHER DOCUMENTS ====================

func indexOtherDocuments(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *repository.MongoDB) {
	schema := &api.CollectionSchema{
		Name: "other_documents",
		Fields: []api.Field{
			{Name: "book", Type: "string", Facet: boolPtr(true)},
			{Name: "title", Type: "string"},
			{Name: "content", Type: "string"},
			{Name: "title_ascii", Type: "string"},
			{Name: "content_ascii", Type: "string"},
			{Name: "url", Type: "string"},
		},
		TokenSeparators: &hyphenSeparators,
	}

	_, err := tsClient.Collections().Create(context.Background(), schema)
	if err != nil {
		sugar.Fatalf("[other_documents] Failed to create collection: %v", err)
	}
	sugar.Info("[other_documents] Collection created")

	// Build book slug mapping from other_document_meta
	bookSlugs := buildBookSlugMap(sugar, mongoDB, "other_document_meta")

	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
	defer cancel()

	coll := mongoDB.Collection("other_document_chapters")
	cursor, err := coll.Find(ctx, bson.M{})
	if err != nil {
		sugar.Fatalf("[other_documents] Failed to query MongoDB: %v", err)
	}
	defer cursor.Close(ctx)

	var docs []interface{}
	for cursor.Next(ctx) {
		var raw bson.M
		if err := cursor.Decode(&raw); err != nil {
			sugar.Warnf("[other_documents] Decode error: %v", err)
			continue
		}

		id := objectIDToHex(raw["_id"])
		book, _ := raw["book"].(string)
		title, _ := raw["title"].(string)
		chapterSlug, _ := raw["slug"].(string)

		content := ""
		if paragraphs, ok := raw["paragraphs"].(bson.A); ok {
			strs := bsonAToStrings(paragraphs)
			content = joinParagraphs(strs)
		}

		bookSlug := bookSlugs[book]
		if bookSlug == "" {
			bookSlug = book
		}

		doc := map[string]interface{}{
			"id":            id,
			"book":          book,
			"title":         title,
			"content":       content,
			"title_ascii":   stripDiacritics(title),
			"content_ascii": stripDiacritics(content),
			"url":           fmt.Sprintf("/other-documents/%s/%s", bookSlug, chapterSlug),
		}
		docs = append(docs, doc)
	}

	sugar.Infof("[other_documents] Read %d documents from MongoDB", len(docs))
	// Smaller batch size: each chapter can have very large paragraphs
	importBatch(sugar, tsClient, "other_documents", docs, 25)
}

// ==================== HELPERS ====================

// buildBookSlugMap reads a meta collection and returns a map of book code -> slug.
// The slug is typically the same as the book code, lowercased and hyphenated.
func buildBookSlugMap(sugar *zap.SugaredLogger, mongoDB *repository.MongoDB, metaCollection string) map[string]string {
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()

	coll := mongoDB.Collection(metaCollection)
	cursor, err := coll.Find(ctx, bson.M{})
	if err != nil {
		sugar.Warnf("Failed to read %s: %v", metaCollection, err)
		return map[string]string{}
	}
	defer cursor.Close(ctx)

	slugMap := make(map[string]string)
	for cursor.Next(ctx) {
		var raw bson.M
		if err := cursor.Decode(&raw); err != nil {
			continue
		}
		book, _ := raw["book"].(string)
		if book != "" {
			// Use the book code directly as the slug; this matches the API routing pattern
			slugMap[book] = book
		}
	}
	return slugMap
}

// objectIDToHex safely converts a MongoDB ObjectID to hex string.
func objectIDToHex(v interface{}) string {
	if oid, ok := v.(bson.ObjectID); ok {
		return oid.Hex()
	}
	return fmt.Sprintf("%v", v)
}

// toInt32 converts a BSON numeric value to int32.
func toInt32(v interface{}) int32 {
	switch n := v.(type) {
	case int32:
		return n
	case int64:
		return int32(n)
	case float64:
		return int32(n)
	case int:
		return int32(n)
	default:
		return 0
	}
}

// toOptionalInt32 tries to convert a BSON value to int32, returning ok=false if nil.
func toOptionalInt32(v interface{}) (int32, bool) {
	if v == nil {
		return 0, false
	}
	switch n := v.(type) {
	case int32:
		return n, true
	case int64:
		return int32(n), true
	case float64:
		return int32(n), true
	case int:
		return int32(n), true
	default:
		return 0, false
	}
}

// toString safely converts an interface to string.
func toString(v interface{}) string {
	if s, ok := v.(string); ok {
		return s
	}
	return ""
}

// bsonAToStrings converts a bson.A to a []string.
func bsonAToStrings(arr bson.A) []string {
	result := make([]string, 0, len(arr))
	for _, v := range arr {
		if s, ok := v.(string); ok {
			result = append(result, s)
		}
	}
	return result
}

// getFirstLetter returns the base letter for glossary grouping.
// Maps diacritical characters to their ASCII base letter.
func getFirstLetter(term string) string {
	if term == "" {
		return "?"
	}
	first := []rune(term)[0]
	diacriticToBase := map[rune]rune{
		'Ā': 'A', 'Ä': 'A',
		'Ī': 'I', 'É': 'I',
		'Ū': 'U', 'Ü': 'U',
		'Ṛ': 'R', 'Ṝ': 'R',
		'Ś': 'S', 'Ç': 'S', 'Ṣ': 'S',
		'Ñ': 'N', 'Ṇ': 'N', 'Ṅ': 'N',
		'Ṭ': 'T',
		'Ḍ': 'D',
		'Ḥ': 'H',
		'ā': 'A', 'ä': 'A',
		'ī': 'I', 'é': 'I',
		'ū': 'U', 'ü': 'U',
		'ṛ': 'R', 'ṝ': 'R',
		'ś': 'S', 'ç': 'S', 'ṣ': 'S',
		'ñ': 'N', 'ṇ': 'N', 'ṅ': 'N',
		'ṭ': 'T',
		'ḍ': 'D',
		'ḥ': 'H',
	}
	if base, ok := diacriticToBase[first]; ok {
		return string(base)
	}
	return strings.ToUpper(string(first))
}