feat(seo): add sitemap generator

app/robots.ts

@@ -0,0 +1,45 @@
+import type { MetadataRoute } from "next";
+
+const SITE_URL =
+  process.env.NEXT_PUBLIC_SITE_URL || "https://prabhupadasays.org";
+
+/**
+ * robots.txt (generated).
+ *
+ * Open the whole site to crawlers — search engines AND AI/LLM crawlers
+ * (GPTBot, ClaudeBot, PerplexityBot, Google-Extended, etc. all honour the
+ * `*` rule, and we deliberately don't block them so the corpus is
+ * discoverable). Only private / non-content surfaces are disallowed:
+ *
+ *   • /api/           — JSON endpoints, never a destination page
+ *   • /settings       — auth-only account settings
+ *   • /notifications  — auth-only
+ *   • /verify-email   — transactional
+ *   • /chat           — interactive tool, not indexable content
+ *   • /u/me, /u/me/*  — the signed-in user's own private library
+ *                       ($ anchors the bare path so public handles like
+ *                       /u/meera aren't caught by the prefix)
+ *
+ * Public content (books, verses, prose, letters, transcriptions, essays,
+ * disciples, magazines, journals, glossary, public profiles, collections)
+ * stays crawlable and is enumerated in the sitemap index.
+ */
+export default function robots(): MetadataRoute.Robots {
+  return {
+    rules: {
+      userAgent: "*",
+      allow: "/",
+      disallow: [
+        "/api/",
+        "/settings",
+        "/notifications",
+        "/verify-email",
+        "/chat",
+        "/u/me$",
+        "/u/me/",
+      ],
+    },
+    sitemap: `${SITE_URL}/sitemap.xml`,
+    host: SITE_URL,
+  };
+}