diff --git a/app/sitemap.ts b/app/sitemap.ts index 21a03945..9b63fd40 100644 --- a/app/sitemap.ts +++ b/app/sitemap.ts @@ -2,24 +2,28 @@ import type { MetadataRoute } from "next"; import { bff } from "@/lib/bff"; /** - * Sitemap (index + sharded children) for the whole public corpus. + * /sitemap.xml — one file for the whole public corpus (~33k URLs, well under + * the 50,000-URL / 50 MB sitemap limits; the content is finite, so a single + * file is correct for the foreseeable future). + * + * Why a single file (not generateSitemaps): Next's generateSitemaps() serves + * shards at /sitemap/.xml and does NOT expose an index at /sitemap.xml — + * but /sitemap.xml is exactly where robots.txt + search engines look. A plain + * default export is served at /sitemap.xml. (If the URL count ever nears 50k, + * switch to a sitemap index referencing sharded children.) * * How this stays cheap on the BFF: - * • One projection-only BFF call (`/sitemap/content`) returns every content - * leaf path in a single sweep — no per-document fan-out. - * • That fetch is cached for 24h (revalidate below) AND the rendered XML is - * ISR-cached, so crawlers hit Next's cache, not the BFF. The BFF sees - * ~one query/day regardless of crawl volume (tag "sitemap" allows - * on-demand revalidation after a re-seed). - * • generateSitemaps() shards into ≤40k-URL files (under the 50k/50MB - * limits) and Next serves a sitemap index at /sitemap.xml automatically. + * • Two projection-only BFF calls (`/sitemap/content`, `/sitemap/users`) + * return every URL in a single sweep each — no per-document fan-out. + * • The fetches are cached 24h (revalidate) AND the rendered XML is + * ISR-cached, so crawlers hit Next's cache, not the BFF (~one query/day). + * Tag "sitemap" lets the BFF force-refresh on new signups / re-seeds. */ export const revalidate = 86400; // 24h const SITE_URL = process.env.NEXT_PUBLIC_SITE_URL || "https://prabhupadasays.org"; -const PER_SITEMAP = 40000; // Top-level hubs that aren't content leaves but should be indexed. const STATIC_PATHS = [ @@ -36,11 +40,6 @@ const STATIC_PATHS = [ "/other-documents", ]; -/** - * All indexable paths: static hubs + book/division landing pages (derived - * from the content leaves) + every content leaf from the BFF. Resilient — if - * the BFF call fails we still emit the static hubs so /sitemap.xml never 500s. - */ async function fetchUrls(path: string): Promise { try { const res = await bff<{ urls: string[] }>(path, { @@ -53,17 +52,17 @@ async function fetchUrls(path: string): Promise { } } +/** + * All indexable paths: static hubs + book/division landing pages (derived + * from the content leaves) + every content leaf + public profiles. Resilient — + * a failed BFF call still yields the static hubs, so /sitemap.xml never 500s. + */ async function allPaths(): Promise { - // Content (Mongo) and public profiles (Postgres) come from separate - // projection-only endpoints. New signups appear automatically on the next - // daily revalidation. const [content, users] = await Promise.all([ fetchUrls("/sitemap/content"), fetchUrls("/sitemap/users"), ]); - // Derive non-redirecting hub pages from the content leaves: every book's - // landing page, and the SB canto / CC līlā division pages. const hubs = new Set(); for (const p of content) { const book = p.match(/^\/books\/([^/]+)/); @@ -72,24 +71,12 @@ async function allPaths(): Promise { if (division) hubs.add(`/books/${division[1]}/${division[2]}`); } - // Dedupe across static + hubs + content + profiles. return [...new Set([...STATIC_PATHS, ...hubs, ...content, ...users])]; } -export async function generateSitemaps(): Promise<{ id: number }[]> { +export default async function sitemap(): Promise { const paths = await allPaths(); - const count = Math.max(1, Math.ceil(paths.length / PER_SITEMAP)); - return Array.from({ length: count }, (_, id) => ({ id })); -} - -export default async function sitemap({ - id, -}: { - id: number; -}): Promise { - const paths = await allPaths(); - const slice = paths.slice(id * PER_SITEMAP, (id + 1) * PER_SITEMAP); - return slice.map((path) => ({ + return paths.map((path) => ({ url: `${SITE_URL}${path}`, changeFrequency: "weekly", priority: path === "/" ? 1 : STATIC_PATHS.includes(path) ? 0.8 : 0.7,