From 4946ce106286b9889826410611437aecacd2c4b9 Mon Sep 17 00:00:00 2001
From: Krispire Admin <admin@krispire.com>
Date: Sun, 14 Jun 2026 14:49:14 +0530
Subject: [PATCH] feat: add sb front matter

---
 cmd/indexer/main.go       | 43 ++++++++++++++++++++++++++++++---------
 internal/server/server.go |  5 ++++-
 2 files changed, 37 insertions(+), 11 deletions(-)

diff --git a/cmd/indexer/main.go b/cmd/indexer/main.go
index 6c5624e..4b66838 100644
--- a/cmd/indexer/main.go
+++ b/cmd/indexer/main.go
@@ -23,6 +23,7 @@ import (
 var (
 	fullReindex = flag.Bool("full", false, "Drop all collections and reindex everything")
 	onlyFlag    = flag.String("only", "", "Comma-separated list of collections to index (e.g., verses,letters)")
+	noDrop      = flag.Bool("no-drop", false, "Do not drop the collection first — upsert into the existing one (resume a partial import)")
 )
 
 // All supported collection names
@@ -146,8 +147,11 @@ func main() {
 	for _, name := range collectionsToIndex {
 		sugar.Infof("=== Processing collection: %s ===", name)
 
-		shouldDrop := *fullReindex || *onlyFlag != ""
-		if !shouldDrop && existingCollections[name] {
+		shouldDrop := (*fullReindex || *onlyFlag != "") && !*noDrop
+		// On a bare run (no targeting), skip collections that already exist.
+		// An explicit --full / --only / --no-drop always (re)processes them.
+		targeted := *fullReindex || *onlyFlag != "" || *noDrop
+		if !targeted && existingCollections[name] {
 			sugar.Infof("[%s] Collection already exists, skipping (use --full to force reindex)", name)
 			continue
 		}
@@ -192,18 +196,32 @@ func main() {
 // importBatch imports documents into Typesense in batches of batchSize.
 func importBatch(sugar *zap.SugaredLogger, tsClient *typesense.Client, collectionName string, docs []interface{}, batchSize int) {
 	total := len(docs)
-	action := "create"
+	// Upsert (not create) so a re-run over an existing collection is
+	// idempotent and additive — essential for resuming a partial import
+	// when a flaky link drops batches mid-run.
+	action := "upsert"
 	for i := 0; i < total; i += batchSize {
 		end := i + batchSize
 		if end > total {
 			end = total
 		}
 		batch := docs[i:end]
-		responses, err := tsClient.Collection(collectionName).Documents().Import(context.Background(), batch, &api.ImportDocumentsParams{
-			Action: &action,
-		})
+		// Retry the batch on transient transport errors (flaky networks can
+		// drop large TLS writes — "broken pipe" / "bad record MAC").
+		var responses []*api.ImportDocumentResponse
+		var err error
+		for attempt := 1; attempt <= 8; attempt++ {
+			responses, err = tsClient.Collection(collectionName).Documents().Import(context.Background(), batch, &api.ImportDocumentsParams{
+				Action: &action,
+			})
+			if err == nil {
+				break
+			}
+			sugar.Warnf("[%s] batch %d-%d attempt %d/5 failed: %v", collectionName, i, end, attempt, err)
+			time.Sleep(time.Duration(attempt) * time.Second)
+		}
 		if err != nil {
-			sugar.Errorf("[%s] Import error at batch %d-%d: %v", collectionName, i, end, err)
+			sugar.Errorf("[%s] Import error at batch %d-%d (gave up): %v", collectionName, i, end, err)
 			continue
 		}
 		// Check for per-document errors
@@ -371,9 +389,12 @@ func indexProse(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *r
 
 	_, err := tsClient.Collections().Create(context.Background(), schema)
 	if err != nil {
-		sugar.Fatalf("[prose] Failed to create collection: %v", err)
+		// An already-existing collection is fine: with --no-drop we resume an
+		// upsert into it (the schema is unchanged).
+		sugar.Warnf("[prose] Create collection: %v (continuing — assuming it exists)", err)
+	} else {
+		sugar.Info("[prose] Collection created")
 	}
-	sugar.Info("[prose] Collection created")
 
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
 	defer cancel()
@@ -428,7 +449,9 @@ func indexProse(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *r
 	}
 
 	sugar.Infof("[prose] Read %d documents from MongoDB", len(docs))
-	importBatch(sugar, tsClient, "prose", docs, 500)
+	// Small batches keep each import POST modest in size — large multi-MB
+	// bodies (front-matter intro segments) are prone to drops on flaky links.
+	importBatch(sugar, tsClient, "prose", docs, 10)
 }
 
 // ==================== LETTERS ====================
diff --git a/internal/server/server.go b/internal/server/server.go
index f8018c0..44ad1df 100644
--- a/internal/server/server.go
+++ b/internal/server/server.go
@@ -411,10 +411,13 @@ func (s *Server) setupRouter() {
 
 			// --- Verse books with divisions ---
 
-			// SB — canto divisions
+			// SB — canto divisions (plus prose front-matter segments;
+			// the static /chapters routes take precedence over /{division}).
 			r.Route("/sb", func(r chi.Router) {
 				r.Use(injectBookCode("sb"))
 				r.Get("/", s.bookH.GetBook)
+				r.Get("/chapters", s.bookH.ListProseChapters)
+				r.Get("/chapters/{slug}", s.bookH.GetProseChapter)
 				r.Get("/{division}", s.bookH.GetDivisionInfo)
 				r.Get("/{division}/{chapter}", s.bookH.GetDivisionChapterVerseIndex)
 				r.Get("/{division}/{chapter}/verses", s.bookH.GetDivisionChapterVerses)