diff --git a/cmd/indexer/main.go b/cmd/indexer/main.go index 0659892..6c5624e 100644 --- a/cmd/indexer/main.go +++ b/cmd/indexer/main.go @@ -31,6 +31,13 @@ var allCollections = []string{ "essays", "journals", "magazines", "other_documents", } +// hyphenSeparators is applied to every collection so that hyphenated +// Sanskrit/Vedic terms (e.g. "Dīkṣā-guru", "Hare-Kṛṣṇa", "sarva-dharman") +// are tokenised into individual words. Without this, a query like +// "Diksa guru" only matches one of the two query tokens against the +// indexed term "diksa-guru", which dramatically lowers its score. +var hyphenSeparators = []string{"-"} + // HTML stripping helper var htmlTagRe = regexp.MustCompile(`<[^>]*>`) @@ -251,9 +258,6 @@ func buildVerseURL(book string, division *string, chapter, verse int) string { func indexVerses(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *repository.MongoDB) { defaultSortingField := "chapter" - // Treat hyphens as token separators so "sarva-dharman" indexes as - // two tokens ("sarva" and "dharman"), making queries like "sarva dharman" matchable. - tokenSeparators := []string{"-"} schema := &api.CollectionSchema{ Name: "verses", Fields: []api.Field{ @@ -272,7 +276,7 @@ func indexVerses(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB * {Name: "url", Type: "string"}, }, DefaultSortingField: &defaultSortingField, - TokenSeparators: &tokenSeparators, + TokenSeparators: &hyphenSeparators, } _, err := tsClient.Collections().Create(context.Background(), schema) @@ -362,6 +366,7 @@ func indexProse(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB *r {Name: "type", Type: "string", Facet: boolPtr(true)}, {Name: "url", Type: "string"}, }, + TokenSeparators: &hyphenSeparators, } _, err := tsClient.Collections().Create(context.Background(), schema) @@ -442,6 +447,7 @@ func indexLetters(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB {Name: "recipient_display_ascii", Type: "string"}, {Name: "url", Type: "string"}, }, + TokenSeparators: &hyphenSeparators, } _, err := tsClient.Collections().Create(context.Background(), schema) @@ -528,6 +534,7 @@ func indexTranscriptions(sugar *zap.SugaredLogger, tsClient *typesense.Client, m {Name: "content_ascii", Type: "string"}, {Name: "url", Type: "string"}, }, + TokenSeparators: &hyphenSeparators, } _, err := tsClient.Collections().Create(context.Background(), schema) @@ -646,6 +653,7 @@ func indexGlossary(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB {Name: "first_letter", Type: "string", Facet: boolPtr(true)}, {Name: "url", Type: "string"}, }, + TokenSeparators: &hyphenSeparators, } _, err := tsClient.Collections().Create(context.Background(), schema) @@ -709,6 +717,7 @@ func indexEssays(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB * {Name: "content_ascii", Type: "string"}, {Name: "url", Type: "string"}, }, + TokenSeparators: &hyphenSeparators, } _, err := tsClient.Collections().Create(context.Background(), schema) @@ -780,6 +789,7 @@ func indexJournals(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoDB {Name: "content_ascii", Type: "string"}, {Name: "url", Type: "string"}, }, + TokenSeparators: &hyphenSeparators, } _, err := tsClient.Collections().Create(context.Background(), schema) @@ -854,6 +864,7 @@ func indexMagazines(sugar *zap.SugaredLogger, tsClient *typesense.Client, mongoD {Name: "content_ascii", Type: "string"}, {Name: "url", Type: "string"}, }, + TokenSeparators: &hyphenSeparators, } _, err := tsClient.Collections().Create(context.Background(), schema) @@ -929,6 +940,7 @@ func indexOtherDocuments(sugar *zap.SugaredLogger, tsClient *typesense.Client, m {Name: "content_ascii", Type: "string"}, {Name: "url", Type: "string"}, }, + TokenSeparators: &hyphenSeparators, } _, err := tsClient.Collections().Create(context.Background(), schema)