search highlight quote fix

lib/text-highlight.ts

@@ -37,6 +37,18 @@ export function encodeTextFragment(text: string): string {
   return `${prefixWords}§${suffixWords}§${len}`;
 }
 
+// Map typographical (curly) quotes to ASCII so matching is resilient to
+// publisher-style differences. Typesense often indexes a straight-quote form
+// of the text while the rendered DOM uses curly quotes (or vice versa). Applied
+// to both sides of every match so prefix/suffix and DOM text agree regardless
+// of which form either one carries. Length-preserving (1 char → 1 char), so
+// it does not disturb the `fragment.length` window used for disambiguation.
+function normalizeQuotes(s: string): string {
+  return s
+    .replace(/[“”„‟″«»]/g, '"')
+    .replace(/[‘’‚‛′]/g, "'");
+}
+
 /**
  * Decode a text fragment from URL params.
  * Supports `§` (current), `|` (v2 legacy), and `,` (v1 legacy) delimiters.
@@ -91,9 +103,11 @@ export function extractMatchedText(
   // Build a single string with paragraph breaks
   const fullText = plainParagraphs.join(" ");
   const normalized = fullText.replace(/\s+/g, " ");
-  const normalizedLower = normalized.toLowerCase();
+  const normalizedLower = normalizeQuotes(normalized.toLowerCase());
 
-  const normalizedPrefix = fragment.prefix.replace(/\s+/g, " ").toLowerCase();
+  const normalizedPrefix = normalizeQuotes(
+    fragment.prefix.replace(/\s+/g, " ").toLowerCase(),
+  );
   let searchFrom = 0;
   let matchStart = -1;
 
@@ -102,9 +116,9 @@ export function extractMatchedText(
     if (idx === -1) break;
 
     if (fragment.suffix) {
-      const normalizedSuffix = fragment.suffix
-        .replace(/\s+/g, " ")
-        .toLowerCase();
+      const normalizedSuffix = normalizeQuotes(
+        fragment.suffix.replace(/\s+/g, " ").toLowerCase(),
+      );
       const expectedEnd = idx + fragment.length;
       const suffixSearchStart = Math.max(
         idx,
@@ -173,17 +187,11 @@ export function extractMatchedTextFromSnippet(
   const lastEnd = (last.index ?? 0) + last[0].length;
 
   const span = snippetHtml.slice(firstStart, lastEnd);
-  const plain = span
-    .replace(/<[^>]*>/g, "")
-    .replace(/&nbsp;/gi, " ")
-    .replace(/&amp;/gi, "&")
-    .replace(/&lt;/gi, "<")
-    .replace(/&gt;/gi, ">")
-    .replace(/&quot;/gi, '"')
-    .replace(/&#39;/gi, "'")
-    .replace(/&[a-z]+;/gi, " ")
-    .replace(/\s+/g, " ")
-    .trim();
+  const plain = normalizeQuotes(
+    decodeSnippetEntities(span.replace(/<[^>]*>/g, ""))
+      .replace(/\s+/g, " ")
+      .trim(),
+  );
 
   if (!plain) return null;
 
@@ -191,17 +199,36 @@ export function extractMatchedTextFromSnippet(
   // (e.g. snippet has marks at far ends of a long passage), fall back to
   // just the first marked term — that's still a real match in the doc.
   if (plain.length > 200) {
-    const firstPlain = first[1]
-      .replace(/<[^>]*>/g, "")
-      .replace(/&[a-z]+;/gi, " ")
-      .replace(/\s+/g, " ")
-      .trim();
+    const firstPlain = normalizeQuotes(
+      decodeSnippetEntities(first[1].replace(/<[^>]*>/g, ""))
+        .replace(/\s+/g, " ")
+        .trim(),
+    );
     return firstPlain || null;
   }
 
   return plain;
 }
 
+// Decode the HTML entities Typesense commonly emits in match snippets.
+// Curly-quote entities (&ldquo; etc.) must be decoded explicitly — if they
+// fall through to the catch-all that maps unknown entities to a space, the
+// resulting `?hl=` fragment loses characters that are present in the DOM,
+// and findAndHighlightText fails to locate the highlight.
+function decodeSnippetEntities(s: string): string {
+  return s
+    .replace(/&nbsp;/gi, " ")
+    .replace(/&amp;/gi, "&")
+    .replace(/&lt;/gi, "<")
+    .replace(/&gt;/gi, ">")
+    .replace(/&(?:quot|#34|#x22);/gi, '"')
+    .replace(/&(?:apos|#39|#x27);/gi, "'")
+    .replace(/&(?:ldquo|rdquo|bdquo|#8220|#8221|#8222);/gi, '"')
+    .replace(/&(?:lsquo|rsquo|sbquo|#8216|#8217|#8218);/gi, "'")
+    .replace(/&[a-z]+\d*;/gi, " ")
+    .replace(/&#x?[0-9a-f]+;/gi, " ");
+}
+
 /**
  * Build a result URL with `?hl=` appended so the destination page highlights
  * the matched phrase. Preserves any other query params already on the URL.
@@ -370,10 +397,16 @@ export function findAndHighlightText(fragment: {
     }
   }
 
-  const normalizedFullLower = collapsedNorm.toLowerCase();
+  // Quote-normalize for matching so the prefix/suffix and the DOM text agree
+  // even when one side uses straight ASCII quotes and the other uses curly
+  // typographical quotes. Length-preserving, so collapsedPosMap indices stay
+  // valid against rawText.
+  const normalizedFullLower = normalizeQuotes(collapsedNorm.toLowerCase());
 
   // Search for the prefix in the full text (case-insensitive)
-  const normalizedPrefix = fragment.prefix.replace(/\s+/g, " ").toLowerCase();
+  const normalizedPrefix = normalizeQuotes(
+    fragment.prefix.replace(/\s+/g, " ").toLowerCase(),
+  );
   let searchFrom = 0;
   let matchStartNorm = -1;
 
@@ -383,9 +416,9 @@ export function findAndHighlightText(fragment: {
 
     // If we have a suffix, verify it appears near the expected length
     if (fragment.suffix) {
-      const normalizedSuffix = fragment.suffix
-        .replace(/\s+/g, " ")
-        .toLowerCase();
+      const normalizedSuffix = normalizeQuotes(
+        fragment.suffix.replace(/\s+/g, " ").toLowerCase(),
+      );
       // Search for suffix near the expected end position in the normalized string.
       // Use a wider tolerance window (±50) since normalization can shift positions.
       const expectedEndNorm = idx + fragment.length;