46 lines
1.5 KiB
TypeScript
46 lines
1.5 KiB
TypeScript
import type { MetadataRoute } from "next";
|
|
|
|
const SITE_URL =
|
|
process.env.NEXT_PUBLIC_SITE_URL || "https://prabhupadasays.org";
|
|
|
|
/**
|
|
* robots.txt (generated).
|
|
*
|
|
* Open the whole site to crawlers — search engines AND AI/LLM crawlers
|
|
* (GPTBot, ClaudeBot, PerplexityBot, Google-Extended, etc. all honour the
|
|
* `*` rule, and we deliberately don't block them so the corpus is
|
|
* discoverable). Only private / non-content surfaces are disallowed:
|
|
*
|
|
* • /api/ — JSON endpoints, never a destination page
|
|
* • /settings — auth-only account settings
|
|
* • /notifications — auth-only
|
|
* • /verify-email — transactional
|
|
* • /ask — interactive tool, not indexable content
|
|
* • /u/me, /u/me/* — the signed-in user's own private library
|
|
* ($ anchors the bare path so public handles like
|
|
* /u/meera aren't caught by the prefix)
|
|
*
|
|
* Public content (books, verses, prose, letters, transcriptions, essays,
|
|
* disciples, magazines, journals, glossary, public profiles, collections)
|
|
* stays crawlable and is enumerated in the sitemap index.
|
|
*/
|
|
export default function robots(): MetadataRoute.Robots {
|
|
return {
|
|
rules: {
|
|
userAgent: "*",
|
|
allow: "/",
|
|
disallow: [
|
|
"/api/",
|
|
"/settings",
|
|
"/notifications",
|
|
"/verify-email",
|
|
"/ask",
|
|
"/u/me$",
|
|
"/u/me/",
|
|
],
|
|
},
|
|
sitemap: `${SITE_URL}/sitemap.xml`,
|
|
host: SITE_URL,
|
|
};
|
|
}
|