From bbe491ae139b3d932eb3f482321cee1134c59420 Mon Sep 17 00:00:00 2001 From: Krispire Admin Date: Mon, 15 Jun 2026 13:28:15 +0530 Subject: [PATCH] feat(seo): add sitemap generator --- internal/handler/sitemap.go | 34 +++++ internal/repository/sitemap_repo.go | 210 ++++++++++++++++++++++++++++ internal/repository/user_repo.go | 21 +++ internal/server/server.go | 34 +++++ 4 files changed, 299 insertions(+) create mode 100644 internal/handler/sitemap.go create mode 100644 internal/repository/sitemap_repo.go diff --git a/internal/handler/sitemap.go b/internal/handler/sitemap.go new file mode 100644 index 0000000..fd6740c --- /dev/null +++ b/internal/handler/sitemap.go @@ -0,0 +1,34 @@ +package handler + +import ( + "context" + "net/http" + "time" + + "github.com/prabhupadasays/spsays-bff/internal/repository" + "github.com/prabhupadasays/spsays-bff/pkg/response" +) + +// SitemapHandler serves the flat list of public content URLs the frontend +// turns into sitemap XML. Public + heavily cached on the frontend (daily +// ISR), so this expensive-ish sweep runs ~once/day, never per crawler request. +type SitemapHandler struct{ repo *repository.SitemapRepo } + +func NewSitemapHandler(repo *repository.SitemapRepo) *SitemapHandler { + return &SitemapHandler{repo: repo} +} + +// GetContent — GET /sitemap/content → { "urls": ["/books/bg/1/1", …] }. +func (h *SitemapHandler) GetContent(w http.ResponseWriter, r *http.Request) { + // The sweep touches tens of thousands of docs across several collections; + // give it room but bound it so a stuck query can't hang the handler. + ctx, cancel := context.WithTimeout(r.Context(), 90*time.Second) + defer cancel() + + urls, err := h.repo.ContentURLs(ctx) + if err != nil { + response.Error(w, http.StatusInternalServerError, "could not build sitemap") + return + } + response.JSON(w, http.StatusOK, map[string]any{"urls": urls}) +} diff --git a/internal/repository/sitemap_repo.go b/internal/repository/sitemap_repo.go new file mode 100644 index 0000000..abfd5d3 --- /dev/null +++ b/internal/repository/sitemap_repo.go @@ -0,0 +1,210 @@ +package repository + +import ( + "context" + "fmt" + "sort" + "strings" + + "go.mongodb.org/mongo-driver/v2/bson" + "go.mongodb.org/mongo-driver/v2/mongo/options" +) + +// SitemapRepo enumerates every public content leaf path for the sitemap. +// All queries are projection-only (no content hydration), so a single sweep +// over the ~tens-of-thousands of content docs stays cheap. URL construction +// mirrors cmd/indexer (the canonical URLs already indexed into Typesense), so +// sitemap entries match the real, crawlable pages exactly. +type SitemapRepo struct{ db *MongoDB } + +func NewSitemapRepo(db *MongoDB) *SitemapRepo { return &SitemapRepo{db: db} } + +// ContentURLs returns the sorted, de-duplicated set of content leaf paths +// (relative — the caller prefixes the origin). +func (r *SitemapRepo) ContentURLs(ctx context.Context) ([]string, error) { + set := make(map[string]struct{}, 40000) + add := func(u string) { + if u != "" { + set[u] = struct{}{} + } + } + + // Verses (bg/sb/cc/sb1962/iso/noi/bs/mm/nbs/…). + if err := r.each(ctx, "verses", + bson.M{"book": 1, "division": 1, "chapter": 1, "verse": 1}, + func(m bson.M) { + add(verseURL(asString(m["book"]), asString(m["division"]), + asInt(m["chapter"]), asInt(m["verse"]))) + }); err != nil { + return nil, err + } + + // Prose chapters (KB volumes, CC chapter summaries, NoD/BBD/… front matter). + if err := r.each(ctx, "prose_chapters", + bson.M{"book": 1, "volume": 1, "slug": 1, "type": 1}, + func(m bson.M) { + add(proseURL(asString(m["book"]), asInt(m["volume"]), + asString(m["slug"]), asString(m["type"]))) + }); err != nil { + return nil, err + } + + // Slug-keyed collections. + for _, c := range []struct{ coll, prefix string }{ + {"letters", "/letters/"}, + {"transcriptions", "/transcriptions/"}, + {"essays", "/essays/"}, + } { + if err := r.each(ctx, c.coll, bson.M{"slug": 1}, func(m bson.M) { + if s := asString(m["slug"]); s != "" { + add(c.prefix + s) + } + }); err != nil { + return nil, err + } + } + + // Book-scoped chapter collections (book code is the URL slug, per indexer). + for _, c := range []struct{ coll, prefix string }{ + {"journal_chapters", "/journals/"}, + {"magazine_issues", "/magazines/"}, + {"other_document_chapters", "/other-documents/"}, + } { + if err := r.each(ctx, c.coll, bson.M{"book": 1, "slug": 1}, func(m bson.M) { + b, s := asString(m["book"]), asString(m["slug"]) + if b != "" && s != "" { + add(fmt.Sprintf("%s%s/%s", c.prefix, b, s)) + } + }); err != nil { + return nil, err + } + } + + // Glossary letter pages — one per distinct first letter that has entries. + letters := make(map[string]struct{}) + if err := r.each(ctx, "glossary", bson.M{"term": 1}, func(m bson.M) { + if l := firstAlphaLower(asString(m["term"])); l != "" { + letters[l] = struct{}{} + } + }); err != nil { + return nil, err + } + for l := range letters { + add("/glossary/" + l) + } + + out := make([]string, 0, len(set)) + for u := range set { + out = append(out, u) + } + sort.Strings(out) + return out, nil +} + +// each streams a collection with the given projection, invoking fn per doc. +func (r *SitemapRepo) each(ctx context.Context, coll string, projection bson.M, fn func(bson.M)) error { + cur, err := r.db.Collection(coll).Find(ctx, bson.M{}, options.Find().SetProjection(projection)) + if err != nil { + return fmt.Errorf("sitemap query %s: %w", coll, err) + } + defer cur.Close(ctx) + for cur.Next(ctx) { + var m bson.M + if err := cur.Decode(&m); err != nil { + continue + } + fn(m) + } + return cur.Err() +} + +// --- URL builders (mirror cmd/indexer/main.go) --- + +func verseURL(book, division string, chapter, verse int) string { + switch book { + case "bg": + return fmt.Sprintf("/books/bg/%d/%d", chapter, verse) + case "sb": + if division != "" { + return fmt.Sprintf("/books/sb/%s/%d/%d", division, chapter, verse) + } + return fmt.Sprintf("/books/sb/%d/%d", chapter, verse) + case "cc": + if division != "" { + return fmt.Sprintf("/books/cc/%s/%d/%d", division, chapter, verse) + } + return fmt.Sprintf("/books/cc/%d/%d", chapter, verse) + case "sb1962": + return fmt.Sprintf("/books/sb1962/%d/%d", chapter, verse) + case "iso": + return fmt.Sprintf("/books/iso/%d", chapter) + case "noi": + return fmt.Sprintf("/books/noi/%d", chapter) + case "bs": + return fmt.Sprintf("/books/bs/5/%d", verse) + case "mm": + return fmt.Sprintf("/books/mm/%d/%d", chapter, verse) + case "nbs": + return fmt.Sprintf("/books/nbs/%d/%d", chapter, verse) + case "": + return "" + default: + return fmt.Sprintf("/books/%s/%d/%d", book, chapter, verse) + } +} + +func proseURL(book string, volume int, slug, typ string) string { + if slug == "" { + return "" + } + switch { + case book == "kb": + return fmt.Sprintf("/books/kb/%d/%s", volume, slug) + case typ == "chapter-summary": + if parts := strings.SplitN(slug, "-", 2); len(parts) == 2 { + return fmt.Sprintf("/books/%s/%s/%s", book, parts[0], parts[1]) + } + return fmt.Sprintf("/books/%s/%s", book, slug) + default: + return fmt.Sprintf("/books/%s/%s", book, slug) + } +} + +// firstAlphaLower returns the lowercased first ASCII letter of a term, or "" +// when it doesn't start with a-z (matches the a–z glossary letter routes). +func firstAlphaLower(term string) string { + term = strings.TrimSpace(term) + if term == "" { + return "" + } + c := term[0] + if c >= 'A' && c <= 'Z' { + c += 'a' - 'A' + } + if c >= 'a' && c <= 'z' { + return string(c) + } + return "" +} + +func asString(v any) string { + if s, ok := v.(string); ok { + return s + } + return "" +} + +func asInt(v any) int { + switch n := v.(type) { + case int32: + return int(n) + case int64: + return int(n) + case int: + return n + case float64: + return int(n) + default: + return 0 + } +} diff --git a/internal/repository/user_repo.go b/internal/repository/user_repo.go index 21bf9ac..72d1989 100644 --- a/internal/repository/user_repo.go +++ b/internal/repository/user_repo.go @@ -379,6 +379,27 @@ func (r *UserRepo) Search(ctx context.Context, q string, limit, offset int) ([]m return out, total, rows.Err() } +// AllHandles returns every user's handle, for the public-profile sitemap. +// Projection-only (handle is unique + indexed), cheap to sweep. Excludes the +// reserved "me" handle (the signed-in user's own private library route). +func (r *UserRepo) AllHandles(ctx context.Context) ([]string, error) { + rows, err := r.pg.Pool.Query(ctx, + `SELECT handle FROM users WHERE handle <> '' AND handle::text <> 'me' ORDER BY handle`) + if err != nil { + return nil, fmt.Errorf("list handles: %w", err) + } + defer rows.Close() + handles := []string{} + for rows.Next() { + var h string + if err := rows.Scan(&h); err != nil { + return nil, fmt.Errorf("scan handle: %w", err) + } + handles = append(handles, h) + } + return handles, rows.Err() +} + // RecentlyJoined returns up to `limit` users who registered on/after `since`, // newest first, as a discovery list for the feed's "new on the platform" // strip. Excludes the viewer, anyone the viewer already has a follow edge with diff --git a/internal/server/server.go b/internal/server/server.go index a0ac409..182bea4 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -4,6 +4,7 @@ import ( "context" "net/http" "path/filepath" + "time" "github.com/go-chi/chi/v5" chiMiddleware "github.com/go-chi/chi/v5/middleware" @@ -18,6 +19,7 @@ import ( "github.com/prabhupadasays/spsays-bff/pkg/mailer" "github.com/prabhupadasays/spsays-bff/pkg/oauth" "github.com/prabhupadasays/spsays-bff/pkg/realtime" + "github.com/prabhupadasays/spsays-bff/pkg/response" "github.com/prabhupadasays/spsays-bff/pkg/shorturl" "go.uber.org/zap" ) @@ -46,6 +48,7 @@ type Server struct { otherDocH *handler.OtherDocumentHandler transcriptionH *handler.TranscriptionHandler searchH *handler.SearchHandler + sitemapH *handler.SitemapHandler shortURLH *handler.ShortURLHandler authH *handler.AuthHandler oauthH *handler.OAuthHandler @@ -89,6 +92,7 @@ func New(cfg *config.Config, logger *zap.Logger) *Server { magazineRepo := repository.NewMagazineRepo(mongoDB) otherDocRepo := repository.NewOtherDocumentRepo(mongoDB) transcriptionRepo := repository.NewTranscriptionRepo(mongoDB) + sitemapRepo := repository.NewSitemapRepo(mongoDB) s := &Server{ cfg: cfg, @@ -106,6 +110,7 @@ func New(cfg *config.Config, logger *zap.Logger) *Server { otherDocH: handler.NewOtherDocumentHandler(otherDocRepo), transcriptionH: handler.NewTranscriptionHandler(transcriptionRepo), searchH: handler.NewSearchHandler(tsClient), + sitemapH: handler.NewSitemapHandler(sitemapRepo), } // URL shortener: Postgres (canonical) + Redis (queue + hot cache) + background workers. @@ -342,6 +347,29 @@ func (s *Server) Close() { s.logger.Info("all connections closed") } +// sitemapUsers — GET /api/v1/sitemap/users → { "urls": ["/u/handle", …] }. +// Public-profile sitemap. Reads userRepo at request time (it's wired only +// after Postgres init); degrades to an empty list when Postgres is down. +func (s *Server) sitemapUsers(w http.ResponseWriter, r *http.Request) { + if s.userRepo == nil { + response.JSON(w, http.StatusOK, map[string]any{"urls": []string{}}) + return + } + ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second) + defer cancel() + + handles, err := s.userRepo.AllHandles(ctx) + if err != nil { + response.Error(w, http.StatusInternalServerError, "could not build user sitemap") + return + } + urls := make([]string, 0, len(handles)) + for _, h := range handles { + urls = append(urls, "/u/"+h) + } + response.JSON(w, http.StatusOK, map[string]any{"urls": urls}) +} + // authUnavailable is the 503 fallback used when Postgres is down and the // auth handlers couldn't be wired up. Keeps every /auth/* and /users/* URL // returning a structured error rather than 404. @@ -405,6 +433,12 @@ func (s *Server) setupRouter() { // API v1 r.Route("/api/v1", func(r chi.Router) { + // Sitemap — flat list of public content URLs (consumed by the + // frontend's daily-cached sitemap routes). Public, read-only. + r.Get("/sitemap/content", s.sitemapH.GetContent) + // Public user profiles (Postgres) — separate from Mongo content. + r.Get("/sitemap/users", s.sitemapUsers) + // Books r.Route("/books", func(r chi.Router) { r.Get("/", s.bookH.ListBooks)