feat: enhance typesense indexer

cmd/indexer/main.go

@@ -31,6 +31,13 @@ var allCollections = []string{
 	"essays", "journals", "magazines", "other_documents",
 }
 
+// hyphenSeparators is applied to every collection so that hyphenated
+// Sanskrit/Vedic terms (e.g. "Dīkṣā-guru", "Hare-Kṛṣṇa", "sarva-dharman")
+// are tokenised into individual words. Without this, a query like
+// "Diksa guru" only matches one of the two query tokens against the
+// indexed term "diksa-guru", which dramatically lowers its score.
+var hyphenSeparators = []string{"-"}
+
 // HTML stripping helper
 var htmlTagRe = regexp.MustCompile(`<[^>]*>`)
 
@@ -251,9 +258,6 @@ func buildVerseURL(book string, division *string, chapter, verse int) string {
 
 func indexVerses(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *repository.MongoDB) {
 	defaultSortingField := "chapter"
-	// Treat hyphens as token separators so "sarva-dharman" indexes as
-	// two tokens ("sarva" and "dharman"), making queries like "sarva dharman" matchable.
-	tokenSeparators := []string{"-"}
 	schema := &api.CollectionSchema{
 		Name: "verses",
 		Fields: []api.Field{
@@ -272,7 +276,7 @@ func indexVerses(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *
 			{Name: "url", Type: "string"},
 		},
 		DefaultSortingField: &defaultSortingField,
-		TokenSeparators:     &tokenSeparators,
+		TokenSeparators:     &hyphenSeparators,
 	}
 
 	_, err := tsClient.Collections().Create(context.Background(), schema)
@@ -362,6 +366,7 @@ func indexProse(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *r
 			{Name: "type", Type: "string", Facet: boolPtr(true)},
 			{Name: "url", Type: "string"},
 		},
+		TokenSeparators: &hyphenSeparators,
 	}
 
 	_, err := tsClient.Collections().Create(context.Background(), schema)
@@ -442,6 +447,7 @@ func indexLetters(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB
 			{Name: "recipient_display_ascii", Type: "string"},
 			{Name: "url", Type: "string"},
 		},
+		TokenSeparators: &hyphenSeparators,
 	}
 
 	_, err := tsClient.Collections().Create(context.Background(), schema)
@@ -528,6 +534,7 @@ func indexTranscriptions(sugar *zap.SugaredLogger, tsClient *typesense.Client, m
 			{Name: "content_ascii", Type: "string"},
 			{Name: "url", Type: "string"},
 		},
+		TokenSeparators: &hyphenSeparators,
 	}
 
 	_, err := tsClient.Collections().Create(context.Background(), schema)
@@ -646,6 +653,7 @@ func indexGlossary(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB
 			{Name: "first_letter", Type: "string", Facet: boolPtr(true)},
 			{Name: "url", Type: "string"},
 		},
+		TokenSeparators: &hyphenSeparators,
 	}
 
 	_, err := tsClient.Collections().Create(context.Background(), schema)
@@ -709,6 +717,7 @@ func indexEssays(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *
 			{Name: "content_ascii", Type: "string"},
 			{Name: "url", Type: "string"},
 		},
+		TokenSeparators: &hyphenSeparators,
 	}
 
 	_, err := tsClient.Collections().Create(context.Background(), schema)
@@ -780,6 +789,7 @@ func indexJournals(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB
 			{Name: "content_ascii", Type: "string"},
 			{Name: "url", Type: "string"},
 		},
+		TokenSeparators: &hyphenSeparators,
 	}
 
 	_, err := tsClient.Collections().Create(context.Background(), schema)
@@ -854,6 +864,7 @@ func indexMagazines(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoD
 			{Name: "content_ascii", Type: "string"},
 			{Name: "url", Type: "string"},
 		},
+		TokenSeparators: &hyphenSeparators,
 	}
 
 	_, err := tsClient.Collections().Create(context.Background(), schema)
@@ -929,6 +940,7 @@ func indexOtherDocuments(sugar *zap.SugaredLogger, tsClient *typesense.Client, m
 			{Name: "content_ascii", Type: "string"},
 			{Name: "url", Type: "string"},
 		},
+		TokenSeparators: &hyphenSeparators,
 	}
 
 	_, err := tsClient.Collections().Create(context.Background(), schema)