feat(seo): add sitemap generator

2026-06-15 13:52:12 +05:30
parent 6cc9285bad
commit be4c2f17fe
83 changed files with 633 additions and 8 deletions
@@ -0,0 +1,97 @@
+import type { MetadataRoute } from "next";
+import { bff } from "@/lib/bff";
+
+/**
+ * Sitemap (index + sharded children) for the whole public corpus.
+ *
+ * How this stays cheap on the BFF:
+ *   • One projection-only BFF call (`/sitemap/content`) returns every content
+ *     leaf path in a single sweep — no per-document fan-out.
+ *   • That fetch is cached for 24h (revalidate below) AND the rendered XML is
+ *     ISR-cached, so crawlers hit Next's cache, not the BFF. The BFF sees
+ *     ~one query/day regardless of crawl volume (tag "sitemap" allows
+ *     on-demand revalidation after a re-seed).
+ *   • generateSitemaps() shards into ≤40k-URL files (under the 50k/50MB
+ *     limits) and Next serves a sitemap index at /sitemap.xml automatically.
+ */
+
+export const revalidate = 86400; // 24h
+
+const SITE_URL =
+  process.env.NEXT_PUBLIC_SITE_URL || "https://prabhupadasays.org";
+const PER_SITEMAP = 40000;
+
+// Top-level hubs that aren't content leaves but should be indexed.
+const STATIC_PATHS = [
+  "/",
+  "/about",
+  "/books",
+  "/letters",
+  "/transcriptions",
+  "/disciples",
+  "/glossary",
+  "/essays",
+  "/magazines",
+  "/journals",
+  "/other-documents",
+];
+
+/**
+ * All indexable paths: static hubs + book/division landing pages (derived
+ * from the content leaves) + every content leaf from the BFF. Resilient — if
+ * the BFF call fails we still emit the static hubs so /sitemap.xml never 500s.
+ */
+async function fetchUrls(path: string): Promise<string[]> {
+  try {
+    const res = await bff<{ urls: string[] }>(path, {
+      revalidate,
+      tags: ["sitemap"],
+    });
+    return res?.urls ?? [];
+  } catch {
+    return [];
+  }
+}
+
+async function allPaths(): Promise<string[]> {
+  // Content (Mongo) and public profiles (Postgres) come from separate
+  // projection-only endpoints. New signups appear automatically on the next
+  // daily revalidation.
+  const [content, users] = await Promise.all([
+    fetchUrls("/sitemap/content"),
+    fetchUrls("/sitemap/users"),
+  ]);
+
+  // Derive non-redirecting hub pages from the content leaves: every book's
+  // landing page, and the SB canto / CC līlā division pages.
+  const hubs = new Set<string>();
+  for (const p of content) {
+    const book = p.match(/^\/books\/([^/]+)/);
+    if (book) hubs.add(`/books/${book[1]}`);
+    const division = p.match(/^\/books\/(sb|cc)\/([^/]+)\/\d+\/\d+$/);
+    if (division) hubs.add(`/books/${division[1]}/${division[2]}`);
+  }
+
+  // Dedupe across static + hubs + content + profiles.
+  return [...new Set([...STATIC_PATHS, ...hubs, ...content, ...users])];
+}
+
+export async function generateSitemaps(): Promise<{ id: number }[]> {
+  const paths = await allPaths();
+  const count = Math.max(1, Math.ceil(paths.length / PER_SITEMAP));
+  return Array.from({ length: count }, (_, id) => ({ id }));
+}
+
+export default async function sitemap({
+  id,
+}: {
+  id: number;
+}): Promise<MetadataRoute.Sitemap> {
+  const paths = await allPaths();
+  const slice = paths.slice(id * PER_SITEMAP, (id + 1) * PER_SITEMAP);
+  return slice.map((path) => ({
+    url: `${SITE_URL}${path}`,
+    changeFrequency: "weekly",
+    priority: path === "/" ? 1 : STATIC_PATHS.includes(path) ? 0.8 : 0.7,
+  }));
+}