feat(seo): add sitemap generator

internal/handler/sitemap.go

@@ -0,0 +1,34 @@
+package handler
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/prabhupadasays/spsays-bff/internal/repository"
+	"github.com/prabhupadasays/spsays-bff/pkg/response"
+)
+
+// SitemapHandler serves the flat list of public content URLs the frontend
+// turns into sitemap XML. Public + heavily cached on the frontend (daily
+// ISR), so this expensive-ish sweep runs ~once/day, never per crawler request.
+type SitemapHandler struct{ repo *repository.SitemapRepo }
+
+func NewSitemapHandler(repo *repository.SitemapRepo) *SitemapHandler {
+	return &SitemapHandler{repo: repo}
+}
+
+// GetContent — GET /sitemap/content → { "urls": ["/books/bg/1/1", …] }.
+func (h *SitemapHandler) GetContent(w http.ResponseWriter, r *http.Request) {
+	// The sweep touches tens of thousands of docs across several collections;
+	// give it room but bound it so a stuck query can't hang the handler.
+	ctx, cancel := context.WithTimeout(r.Context(), 90*time.Second)
+	defer cancel()
+
+	urls, err := h.repo.ContentURLs(ctx)
+	if err != nil {
+		response.Error(w, http.StatusInternalServerError, "could not build sitemap")
+		return
+	}
+	response.JSON(w, http.StatusOK, map[string]any{"urls": urls})
+}

internal/repository/sitemap_repo.go

@@ -0,0 +1,210 @@
+package repository
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strings"
+
+	"go.mongodb.org/mongo-driver/v2/bson"
+	"go.mongodb.org/mongo-driver/v2/mongo/options"
+)
+
+// SitemapRepo enumerates every public content leaf path for the sitemap.
+// All queries are projection-only (no content hydration), so a single sweep
+// over the ~tens-of-thousands of content docs stays cheap. URL construction
+// mirrors cmd/indexer (the canonical URLs already indexed into Typesense), so
+// sitemap entries match the real, crawlable pages exactly.
+type SitemapRepo struct{ db *MongoDB }
+
+func NewSitemapRepo(db *MongoDB) *SitemapRepo { return &SitemapRepo{db: db} }
+
+// ContentURLs returns the sorted, de-duplicated set of content leaf paths
+// (relative — the caller prefixes the origin).
+func (r *SitemapRepo) ContentURLs(ctx context.Context) ([]string, error) {
+	set := make(map[string]struct{}, 40000)
+	add := func(u string) {
+		if u != "" {
+			set[u] = struct{}{}
+		}
+	}
+
+	// Verses (bg/sb/cc/sb1962/iso/noi/bs/mm/nbs/…).
+	if err := r.each(ctx, "verses",
+		bson.M{"book": 1, "division": 1, "chapter": 1, "verse": 1},
+		func(m bson.M) {
+			add(verseURL(asString(m["book"]), asString(m["division"]),
+				asInt(m["chapter"]), asInt(m["verse"])))
+		}); err != nil {
+		return nil, err
+	}
+
+	// Prose chapters (KB volumes, CC chapter summaries, NoD/BBD/… front matter).
+	if err := r.each(ctx, "prose_chapters",
+		bson.M{"book": 1, "volume": 1, "slug": 1, "type": 1},
+		func(m bson.M) {
+			add(proseURL(asString(m["book"]), asInt(m["volume"]),
+				asString(m["slug"]), asString(m["type"])))
+		}); err != nil {
+		return nil, err
+	}
+
+	// Slug-keyed collections.
+	for _, c := range []struct{ coll, prefix string }{
+		{"letters", "/letters/"},
+		{"transcriptions", "/transcriptions/"},
+		{"essays", "/essays/"},
+	} {
+		if err := r.each(ctx, c.coll, bson.M{"slug": 1}, func(m bson.M) {
+			if s := asString(m["slug"]); s != "" {
+				add(c.prefix + s)
+			}
+		}); err != nil {
+			return nil, err
+		}
+	}
+
+	// Book-scoped chapter collections (book code is the URL slug, per indexer).
+	for _, c := range []struct{ coll, prefix string }{
+		{"journal_chapters", "/journals/"},
+		{"magazine_issues", "/magazines/"},
+		{"other_document_chapters", "/other-documents/"},
+	} {
+		if err := r.each(ctx, c.coll, bson.M{"book": 1, "slug": 1}, func(m bson.M) {
+			b, s := asString(m["book"]), asString(m["slug"])
+			if b != "" && s != "" {
+				add(fmt.Sprintf("%s%s/%s", c.prefix, b, s))
+			}
+		}); err != nil {
+			return nil, err
+		}
+	}
+
+	// Glossary letter pages — one per distinct first letter that has entries.
+	letters := make(map[string]struct{})
+	if err := r.each(ctx, "glossary", bson.M{"term": 1}, func(m bson.M) {
+		if l := firstAlphaLower(asString(m["term"])); l != "" {
+			letters[l] = struct{}{}
+		}
+	}); err != nil {
+		return nil, err
+	}
+	for l := range letters {
+		add("/glossary/" + l)
+	}
+
+	out := make([]string, 0, len(set))
+	for u := range set {
+		out = append(out, u)
+	}
+	sort.Strings(out)
+	return out, nil
+}
+
+// each streams a collection with the given projection, invoking fn per doc.
+func (r *SitemapRepo) each(ctx context.Context, coll string, projection bson.M, fn func(bson.M)) error {
+	cur, err := r.db.Collection(coll).Find(ctx, bson.M{}, options.Find().SetProjection(projection))
+	if err != nil {
+		return fmt.Errorf("sitemap query %s: %w", coll, err)
+	}
+	defer cur.Close(ctx)
+	for cur.Next(ctx) {
+		var m bson.M
+		if err := cur.Decode(&m); err != nil {
+			continue
+		}
+		fn(m)
+	}
+	return cur.Err()
+}
+
+// --- URL builders (mirror cmd/indexer/main.go) ---
+
+func verseURL(book, division string, chapter, verse int) string {
+	switch book {
+	case "bg":
+		return fmt.Sprintf("/books/bg/%d/%d", chapter, verse)
+	case "sb":
+		if division != "" {
+			return fmt.Sprintf("/books/sb/%s/%d/%d", division, chapter, verse)
+		}
+		return fmt.Sprintf("/books/sb/%d/%d", chapter, verse)
+	case "cc":
+		if division != "" {
+			return fmt.Sprintf("/books/cc/%s/%d/%d", division, chapter, verse)
+		}
+		return fmt.Sprintf("/books/cc/%d/%d", chapter, verse)
+	case "sb1962":
+		return fmt.Sprintf("/books/sb1962/%d/%d", chapter, verse)
+	case "iso":
+		return fmt.Sprintf("/books/iso/%d", chapter)
+	case "noi":
+		return fmt.Sprintf("/books/noi/%d", chapter)
+	case "bs":
+		return fmt.Sprintf("/books/bs/5/%d", verse)
+	case "mm":
+		return fmt.Sprintf("/books/mm/%d/%d", chapter, verse)
+	case "nbs":
+		return fmt.Sprintf("/books/nbs/%d/%d", chapter, verse)
+	case "":
+		return ""
+	default:
+		return fmt.Sprintf("/books/%s/%d/%d", book, chapter, verse)
+	}
+}
+
+func proseURL(book string, volume int, slug, typ string) string {
+	if slug == "" {
+		return ""
+	}
+	switch {
+	case book == "kb":
+		return fmt.Sprintf("/books/kb/%d/%s", volume, slug)
+	case typ == "chapter-summary":
+		if parts := strings.SplitN(slug, "-", 2); len(parts) == 2 {
+			return fmt.Sprintf("/books/%s/%s/%s", book, parts[0], parts[1])
+		}
+		return fmt.Sprintf("/books/%s/%s", book, slug)
+	default:
+		return fmt.Sprintf("/books/%s/%s", book, slug)
+	}
+}
+
+// firstAlphaLower returns the lowercased first ASCII letter of a term, or ""
+// when it doesn't start with a-z (matches the a–z glossary letter routes).
+func firstAlphaLower(term string) string {
+	term = strings.TrimSpace(term)
+	if term == "" {
+		return ""
+	}
+	c := term[0]
+	if c >= 'A' && c <= 'Z' {
+		c += 'a' - 'A'
+	}
+	if c >= 'a' && c <= 'z' {
+		return string(c)
+	}
+	return ""
+}
+
+func asString(v any) string {
+	if s, ok := v.(string); ok {
+		return s
+	}
+	return ""
+}
+
+func asInt(v any) int {
+	switch n := v.(type) {
+	case int32:
+		return int(n)
+	case int64:
+		return int(n)
+	case int:
+		return n
+	case float64:
+		return int(n)
+	default:
+		return 0
+	}
+}

internal/repository/user_repo.go

@@ -379,6 +379,27 @@ func (r *UserRepo) Search(ctx context.Context, q string, limit, offset int) ([]m
 	return out, total, rows.Err()
 }
 
+// AllHandles returns every user's handle, for the public-profile sitemap.
+// Projection-only (handle is unique + indexed), cheap to sweep. Excludes the
+// reserved "me" handle (the signed-in user's own private library route).
+func (r *UserRepo) AllHandles(ctx context.Context) ([]string, error) {
+	rows, err := r.pg.Pool.Query(ctx,
+		`SELECT handle FROM users WHERE handle <> '' AND handle::text <> 'me' ORDER BY handle`)
+	if err != nil {
+		return nil, fmt.Errorf("list handles: %w", err)
+	}
+	defer rows.Close()
+	handles := []string{}
+	for rows.Next() {
+		var h string
+		if err := rows.Scan(&h); err != nil {
+			return nil, fmt.Errorf("scan handle: %w", err)
+		}
+		handles = append(handles, h)
+	}
+	return handles, rows.Err()
+}
+
 // RecentlyJoined returns up to `limit` users who registered on/after `since`,
 // newest first, as a discovery list for the feed's "new on the platform"
 // strip. Excludes the viewer, anyone the viewer already has a follow edge with

internal/server/server.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"net/http"
 	"path/filepath"
+	"time"
 
 	"github.com/go-chi/chi/v5"
 	chiMiddleware "github.com/go-chi/chi/v5/middleware"
@@ -18,6 +19,7 @@ import (
 	"github.com/prabhupadasays/spsays-bff/pkg/mailer"
 	"github.com/prabhupadasays/spsays-bff/pkg/oauth"
 	"github.com/prabhupadasays/spsays-bff/pkg/realtime"
+	"github.com/prabhupadasays/spsays-bff/pkg/response"
 	"github.com/prabhupadasays/spsays-bff/pkg/shorturl"
 	"go.uber.org/zap"
 )
@@ -46,6 +48,7 @@ type Server struct {
 	otherDocH      *handler.OtherDocumentHandler
 	transcriptionH *handler.TranscriptionHandler
 	searchH        *handler.SearchHandler
+	sitemapH       *handler.SitemapHandler
 	shortURLH      *handler.ShortURLHandler
 	authH          *handler.AuthHandler
 	oauthH         *handler.OAuthHandler
@@ -89,6 +92,7 @@ func New(cfg *config.Config, logger *zap.Logger) *Server {
 	magazineRepo := repository.NewMagazineRepo(mongoDB)
 	otherDocRepo := repository.NewOtherDocumentRepo(mongoDB)
 	transcriptionRepo := repository.NewTranscriptionRepo(mongoDB)
+	sitemapRepo := repository.NewSitemapRepo(mongoDB)
 
 	s := &Server{
 		cfg:    cfg,
@@ -106,6 +110,7 @@ func New(cfg *config.Config, logger *zap.Logger) *Server {
 		otherDocH:      handler.NewOtherDocumentHandler(otherDocRepo),
 		transcriptionH: handler.NewTranscriptionHandler(transcriptionRepo),
 		searchH:        handler.NewSearchHandler(tsClient),
+		sitemapH:       handler.NewSitemapHandler(sitemapRepo),
 	}
 
 	// URL shortener: Postgres (canonical) + Redis (queue + hot cache) + background workers.
@@ -342,6 +347,29 @@ func (s *Server) Close() {
 	s.logger.Info("all connections closed")
 }
 
+// sitemapUsers — GET /api/v1/sitemap/users → { "urls": ["/u/handle", …] }.
+// Public-profile sitemap. Reads userRepo at request time (it's wired only
+// after Postgres init); degrades to an empty list when Postgres is down.
+func (s *Server) sitemapUsers(w http.ResponseWriter, r *http.Request) {
+	if s.userRepo == nil {
+		response.JSON(w, http.StatusOK, map[string]any{"urls": []string{}})
+		return
+	}
+	ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
+	defer cancel()
+
+	handles, err := s.userRepo.AllHandles(ctx)
+	if err != nil {
+		response.Error(w, http.StatusInternalServerError, "could not build user sitemap")
+		return
+	}
+	urls := make([]string, 0, len(handles))
+	for _, h := range handles {
+		urls = append(urls, "/u/"+h)
+	}
+	response.JSON(w, http.StatusOK, map[string]any{"urls": urls})
+}
+
 // authUnavailable is the 503 fallback used when Postgres is down and the
 // auth handlers couldn't be wired up. Keeps every /auth/* and /users/* URL
 // returning a structured error rather than 404.
@@ -405,6 +433,12 @@ func (s *Server) setupRouter() {
 
 	// API v1
 	r.Route("/api/v1", func(r chi.Router) {
+		// Sitemap — flat list of public content URLs (consumed by the
+		// frontend's daily-cached sitemap routes). Public, read-only.
+		r.Get("/sitemap/content", s.sitemapH.GetContent)
+		// Public user profiles (Postgres) — separate from Mongo content.
+		r.Get("/sitemap/users", s.sitemapUsers)
+
 		// Books
 		r.Route("/books", func(r chi.Router) {
 			r.Get("/", s.bookH.ListBooks)