From a22fcf360da9302b645398cc5641d47561976f34 Mon Sep 17 00:00:00 2001 From: gosangam Date: Sun, 24 May 2026 18:07:52 +0530 Subject: [PATCH] search highlight quote fix --- lib/text-highlight.ts | 85 ++++++++++++++++++++++++++++++------------- 1 file changed, 59 insertions(+), 26 deletions(-) diff --git a/lib/text-highlight.ts b/lib/text-highlight.ts index c159a602..cacc44ad 100644 --- a/lib/text-highlight.ts +++ b/lib/text-highlight.ts @@ -37,6 +37,18 @@ export function encodeTextFragment(text: string): string { return `${prefixWords}§${suffixWords}§${len}`; } +// Map typographical (curly) quotes to ASCII so matching is resilient to +// publisher-style differences. Typesense often indexes a straight-quote form +// of the text while the rendered DOM uses curly quotes (or vice versa). Applied +// to both sides of every match so prefix/suffix and DOM text agree regardless +// of which form either one carries. Length-preserving (1 char → 1 char), so +// it does not disturb the `fragment.length` window used for disambiguation. +function normalizeQuotes(s: string): string { + return s + .replace(/[“”„‟″«»]/g, '"') + .replace(/[‘’‚‛′]/g, "'"); +} + /** * Decode a text fragment from URL params. * Supports `§` (current), `|` (v2 legacy), and `,` (v1 legacy) delimiters. @@ -91,9 +103,11 @@ export function extractMatchedText( // Build a single string with paragraph breaks const fullText = plainParagraphs.join(" "); const normalized = fullText.replace(/\s+/g, " "); - const normalizedLower = normalized.toLowerCase(); + const normalizedLower = normalizeQuotes(normalized.toLowerCase()); - const normalizedPrefix = fragment.prefix.replace(/\s+/g, " ").toLowerCase(); + const normalizedPrefix = normalizeQuotes( + fragment.prefix.replace(/\s+/g, " ").toLowerCase(), + ); let searchFrom = 0; let matchStart = -1; @@ -102,9 +116,9 @@ export function extractMatchedText( if (idx === -1) break; if (fragment.suffix) { - const normalizedSuffix = fragment.suffix - .replace(/\s+/g, " ") - .toLowerCase(); + const normalizedSuffix = normalizeQuotes( + fragment.suffix.replace(/\s+/g, " ").toLowerCase(), + ); const expectedEnd = idx + fragment.length; const suffixSearchStart = Math.max( idx, @@ -173,17 +187,11 @@ export function extractMatchedTextFromSnippet( const lastEnd = (last.index ?? 0) + last[0].length; const span = snippetHtml.slice(firstStart, lastEnd); - const plain = span - .replace(/<[^>]*>/g, "") - .replace(/ /gi, " ") - .replace(/&/gi, "&") - .replace(/</gi, "<") - .replace(/>/gi, ">") - .replace(/"/gi, '"') - .replace(/'/gi, "'") - .replace(/&[a-z]+;/gi, " ") - .replace(/\s+/g, " ") - .trim(); + const plain = normalizeQuotes( + decodeSnippetEntities(span.replace(/<[^>]*>/g, "")) + .replace(/\s+/g, " ") + .trim(), + ); if (!plain) return null; @@ -191,17 +199,36 @@ export function extractMatchedTextFromSnippet( // (e.g. snippet has marks at far ends of a long passage), fall back to // just the first marked term — that's still a real match in the doc. if (plain.length > 200) { - const firstPlain = first[1] - .replace(/<[^>]*>/g, "") - .replace(/&[a-z]+;/gi, " ") - .replace(/\s+/g, " ") - .trim(); + const firstPlain = normalizeQuotes( + decodeSnippetEntities(first[1].replace(/<[^>]*>/g, "")) + .replace(/\s+/g, " ") + .trim(), + ); return firstPlain || null; } return plain; } +// Decode the HTML entities Typesense commonly emits in match snippets. +// Curly-quote entities (“ etc.) must be decoded explicitly — if they +// fall through to the catch-all that maps unknown entities to a space, the +// resulting `?hl=` fragment loses characters that are present in the DOM, +// and findAndHighlightText fails to locate the highlight. +function decodeSnippetEntities(s: string): string { + return s + .replace(/ /gi, " ") + .replace(/&/gi, "&") + .replace(/</gi, "<") + .replace(/>/gi, ">") + .replace(/&(?:quot|#34|#x22);/gi, '"') + .replace(/&(?:apos|#39|#x27);/gi, "'") + .replace(/&(?:ldquo|rdquo|bdquo|#8220|#8221|#8222);/gi, '"') + .replace(/&(?:lsquo|rsquo|sbquo|#8216|#8217|#8218);/gi, "'") + .replace(/&[a-z]+\d*;/gi, " ") + .replace(/&#x?[0-9a-f]+;/gi, " "); +} + /** * Build a result URL with `?hl=` appended so the destination page highlights * the matched phrase. Preserves any other query params already on the URL. @@ -370,10 +397,16 @@ export function findAndHighlightText(fragment: { } } - const normalizedFullLower = collapsedNorm.toLowerCase(); + // Quote-normalize for matching so the prefix/suffix and the DOM text agree + // even when one side uses straight ASCII quotes and the other uses curly + // typographical quotes. Length-preserving, so collapsedPosMap indices stay + // valid against rawText. + const normalizedFullLower = normalizeQuotes(collapsedNorm.toLowerCase()); // Search for the prefix in the full text (case-insensitive) - const normalizedPrefix = fragment.prefix.replace(/\s+/g, " ").toLowerCase(); + const normalizedPrefix = normalizeQuotes( + fragment.prefix.replace(/\s+/g, " ").toLowerCase(), + ); let searchFrom = 0; let matchStartNorm = -1; @@ -383,9 +416,9 @@ export function findAndHighlightText(fragment: { // If we have a suffix, verify it appears near the expected length if (fragment.suffix) { - const normalizedSuffix = fragment.suffix - .replace(/\s+/g, " ") - .toLowerCase(); + const normalizedSuffix = normalizeQuotes( + fragment.suffix.replace(/\s+/g, " ").toLowerCase(), + ); // Search for suffix near the expected end position in the normalized string. // Use a wider tolerance window (±50) since normalization can shift positions. const expectedEndNorm = idx + fragment.length;