diff --git a/go.mod b/go.mod index 9bd579d..196b58c 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,13 @@ module github.com/prabhupadasays/spsays-bff -go 1.23 +go 1.25.0 require ( github.com/go-chi/chi/v5 v5.2.1 github.com/go-chi/cors v1.2.1 + github.com/jackc/pgx/v5 v5.9.2 github.com/joho/godotenv v1.5.1 + github.com/redis/go-redis/v9 v9.19.0 github.com/typesense/typesense-go/v2 v2.0.0 go.mongodb.org/mongo-driver/v2 v2.2.1 go.uber.org/zap v1.27.0 @@ -13,8 +15,12 @@ require ( require ( github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/golang/snappy v1.0.0 // indirect github.com/google/uuid v1.5.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/klauspost/compress v1.16.7 // indirect github.com/oapi-codegen/runtime v1.1.1 // indirect github.com/sony/gobreaker v0.5.0 // indirect @@ -22,8 +28,9 @@ require ( github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect + go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.33.0 // indirect - golang.org/x/sync v0.11.0 // indirect - golang.org/x/text v0.22.0 // indirect + golang.org/x/sync v0.17.0 // indirect + golang.org/x/text v0.29.0 // indirect ) diff --git a/go.sum b/go.sum index f47a02f..8dc8d98 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,12 @@ github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMz github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -15,6 +21,14 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw= +github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jinzhu/copier v0.3.4 h1:mfU6jI9PtCeUjkjQ322dlff9ELjGDu975C2p/nrubVI= github.com/jinzhu/copier v0.3.4/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= @@ -22,17 +36,22 @@ github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwA github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro= github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.19.0 h1:XPVaaPSnG6RhYf7p+rmSa9zZfeVAnWsH5h3lxthOm/k= +github.com/redis/go-redis/v9 v9.19.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA= github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg= github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/typesense/typesense-go/v2 v2.0.0 h1:+MksOnrVioDqsGpz8RXkOUqhVN+yFxZwJlGDQHr/64I= github.com/typesense/typesense-go/v2 v2.0.0/go.mod h1:7V1ZBSfmdciL6yb2bPtWha+W53gV5WZhyOSpVgDJfao= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= @@ -44,8 +63,12 @@ github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gi github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= go.mongodb.org/mongo-driver/v2 v2.2.1 h1:w5xra3yyu/sGrziMzK1D0cRRaH/b7lWCSsoN6+WV6AM= go.mongodb.org/mongo-driver/v2 v2.2.1/go.mod h1:qQkDMhCGWl3FN509DfdPd4GRBLU/41zqF/k8eTRceps= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= @@ -64,24 +87,28 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= -golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= -golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/handler/shorturl.go b/internal/handler/shorturl.go new file mode 100644 index 0000000..193f34b --- /dev/null +++ b/internal/handler/shorturl.go @@ -0,0 +1,185 @@ +package handler + +import ( + "context" + "encoding/json" + "errors" + "net" + "net/http" + "strings" + "time" + + "github.com/go-chi/chi/v5" + "github.com/jackc/pgx/v5" + "github.com/prabhupadasays/spsays-bff/internal/repository" + "github.com/prabhupadasays/spsays-bff/pkg/response" + "github.com/prabhupadasays/spsays-bff/pkg/shorturl" + "go.uber.org/zap" +) + +type ShortURLHandler struct { + repo *repository.ShortURLRepo + queue *shorturl.Queue + logger *zap.Logger +} + +func NewShortURLHandler(repo *repository.ShortURLRepo, queue *shorturl.Queue, logger *zap.Logger) *ShortURLHandler { + return &ShortURLHandler{repo: repo, queue: queue, logger: logger} +} + +type createRequest struct { + URL string `json:"url"` +} + +type createResponse struct { + Code string `json:"code"` +} + +type resolveResponse struct { + URL string `json:"url"` +} + +// Create handles POST /api/v1/share. Validates the path, allocates a code in +// Redis, enqueues the row for background insertion into Postgres, and returns +// the code immediately. The path persisted is exactly what the client sent, +// preserving the `?hl=` fragment. +func (h *ShortURLHandler) Create(w http.ResponseWriter, r *http.Request) { + var req createRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + response.Error(w, http.StatusBadRequest, "invalid request body") + return + } + if err := shorturl.ValidatePath(req.URL); err != nil { + response.Error(w, http.StatusBadRequest, err.Error()) + return + } + + if h.queue == nil { + response.Error(w, http.StatusServiceUnavailable, "url shortener unavailable") + return + } + + ctx := r.Context() + + code, err := h.queue.AllocateCode(ctx) + if err != nil { + h.logger.Warn("allocate code", zap.Error(err)) + response.Error(w, http.StatusServiceUnavailable, "failed to allocate short code") + return + } + + payload := shorturl.PendingPayload{ + URL: req.URL, + CreatorIP: clientIP(r), + CreatorUA: truncateTo(r.UserAgent(), 1024), + CreatorReferer: truncateTo(r.Referer(), 1024), + CreatedAt: time.Now().UTC(), + } + if err := h.queue.MarkPending(ctx, code, payload); err != nil { + h.logger.Warn("mark pending", zap.String("code", code), zap.Error(err)) + response.Error(w, http.StatusServiceUnavailable, "failed to queue request") + return + } + if err := h.queue.EnqueueCreate(ctx, code); err != nil { + h.logger.Warn("enqueue create", zap.String("code", code), zap.Error(err)) + response.Error(w, http.StatusServiceUnavailable, "failed to queue request") + return + } + + response.JSON(w, http.StatusOK, createResponse{Code: code}) +} + +// Resolve handles GET /api/v1/share/{code}. Resolves through the cache chain +// (hot cache → pending → Postgres) and enqueues a click event. Click events +// are flushed asynchronously by the click-worker. +func (h *ShortURLHandler) Resolve(w http.ResponseWriter, r *http.Request) { + code := strings.ToLower(chi.URLParam(r, "code")) + if !shorturl.ValidCode(code) { + response.Error(w, http.StatusBadRequest, "invalid code") + return + } + + if h.queue == nil { + response.Error(w, http.StatusServiceUnavailable, "url shortener unavailable") + return + } + + ctx := r.Context() + now := time.Now().UTC() + + // 1. Hot cache. + if longURL, ok := h.queue.LookupCached(ctx, code); ok { + h.enqueueClick(ctx, code, r, now) + response.JSON(w, http.StatusOK, resolveResponse{URL: longURL}) + return + } + + // 2. Pending (still in flight to Postgres). + if p, ok := h.queue.LookupPending(ctx, code); ok { + h.enqueueClick(ctx, code, r, now) + response.JSON(w, http.StatusOK, resolveResponse{URL: p.URL}) + return + } + + // 3. Postgres. + row, err := h.repo.GetByCode(ctx, code) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + response.Error(w, http.StatusNotFound, "short link not found") + return + } + h.logger.Error("resolve postgres", zap.String("code", code), zap.Error(err)) + response.Error(w, http.StatusInternalServerError, "failed to resolve short link") + return + } + if row.Disabled { + response.Error(w, http.StatusGone, "short link disabled") + return + } + if row.ExpiresAt != nil && row.ExpiresAt.Before(now) { + response.Error(w, http.StatusGone, "short link expired") + return + } + + // Repopulate hot cache and enqueue the click. + _ = h.queue.MarkCached(ctx, code, row.LongURL) + h.enqueueClick(ctx, code, r, now) + response.JSON(w, http.StatusOK, resolveResponse{URL: row.LongURL}) +} + +func (h *ShortURLHandler) enqueueClick(ctx context.Context, code string, r *http.Request, ts time.Time) { + ua := r.UserAgent() + ev := shorturl.ClickEvent{ + Code: code, + IP: clientIP(r), + UA: truncateTo(ua, 1024), + Referer: truncateTo(r.Referer(), 1024), + IsBot: shorturl.IsBot(ua), + ClickedAt: ts, + } + if err := h.queue.EnqueueClick(ctx, ev); err != nil { + h.logger.Warn("enqueue click", zap.String("code", code), zap.Error(err)) + } +} + +// clientIP extracts the best-effort client IP. RemoteAddr is "host:port"; we +// strip the port. The chi RealIP middleware already overwrites RemoteAddr +// from X-Forwarded-For / X-Real-IP when those are present. +func clientIP(r *http.Request) string { + addr := r.RemoteAddr + if addr == "" { + return "" + } + host, _, err := net.SplitHostPort(addr) + if err != nil { + return addr + } + return host +} + +func truncateTo(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] +} diff --git a/internal/repository/postgres.go b/internal/repository/postgres.go new file mode 100644 index 0000000..def5ce8 --- /dev/null +++ b/internal/repository/postgres.go @@ -0,0 +1,131 @@ +package repository + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "go.uber.org/zap" +) + +type Postgres struct { + Pool *pgxpool.Pool + logger *zap.Logger +} + +func NewPostgres(ctx context.Context, uri string, logger *zap.Logger) (*Postgres, error) { + if uri == "" { + return nil, fmt.Errorf("postgres URI is empty") + } + + cfg, err := pgxpool.ParseConfig(uri) + if err != nil { + return nil, fmt.Errorf("postgres parse config: %w", err) + } + cfg.MaxConns = 20 + cfg.MinConns = 2 + cfg.MaxConnLifetime = 30 * time.Minute + cfg.MaxConnIdleTime = 5 * time.Minute + + pingCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + pool, err := pgxpool.NewWithConfig(pingCtx, cfg) + if err != nil { + return nil, fmt.Errorf("postgres connect: %w", err) + } + if err := pool.Ping(pingCtx); err != nil { + pool.Close() + return nil, fmt.Errorf("postgres ping: %w", err) + } + + logger.Info("connected to Postgres") + return &Postgres{Pool: pool, logger: logger}, nil +} + +func (p *Postgres) Close() { + if p == nil || p.Pool == nil { + return + } + p.Pool.Close() +} + +// RunMigrations applies any *.up.sql files in dir that haven't been recorded +// in schema_migrations yet, in lexical order. Tiny homegrown runner — no +// external dependency. Down-migrations are not executed automatically. +func (p *Postgres) RunMigrations(ctx context.Context, dir string) error { + if _, err := p.Pool.Exec(ctx, ` + CREATE TABLE IF NOT EXISTS schema_migrations ( + version BIGINT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() + )`); err != nil { + return fmt.Errorf("create schema_migrations: %w", err) + } + + files, err := filepath.Glob(filepath.Join(dir, "*.up.sql")) + if err != nil { + return fmt.Errorf("glob migrations: %w", err) + } + sort.Strings(files) + + for _, f := range files { + version, err := parseMigrationVersion(f) + if err != nil { + p.logger.Warn("skipping unparseable migration filename", zap.String("file", f), zap.Error(err)) + continue + } + + var exists bool + if err := p.Pool.QueryRow(ctx, + `SELECT EXISTS(SELECT 1 FROM schema_migrations WHERE version = $1)`, version, + ).Scan(&exists); err != nil { + return fmt.Errorf("check migration %d: %w", version, err) + } + if exists { + continue + } + + sqlBytes, err := os.ReadFile(f) + if err != nil { + return fmt.Errorf("read %s: %w", f, err) + } + + tx, err := p.Pool.Begin(ctx) + if err != nil { + return fmt.Errorf("begin tx for %s: %w", f, err) + } + if _, err := tx.Exec(ctx, string(sqlBytes)); err != nil { + _ = tx.Rollback(ctx) + return fmt.Errorf("apply %s: %w", f, err) + } + if _, err := tx.Exec(ctx, `INSERT INTO schema_migrations(version) VALUES ($1)`, version); err != nil { + _ = tx.Rollback(ctx) + return fmt.Errorf("record %s: %w", f, err) + } + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("commit %s: %w", f, err) + } + + p.logger.Info("applied migration", zap.String("file", filepath.Base(f)), zap.Int64("version", version)) + } + return nil +} + +func parseMigrationVersion(path string) (int64, error) { + base := filepath.Base(path) + idx := strings.Index(base, "_") + if idx <= 0 { + return 0, fmt.Errorf("filename %q must start with _", base) + } + v, err := strconv.ParseInt(base[:idx], 10, 64) + if err != nil { + return 0, fmt.Errorf("version prefix not an integer: %w", err) + } + return v, nil +} diff --git a/internal/repository/redis.go b/internal/repository/redis.go new file mode 100644 index 0000000..72e19cb --- /dev/null +++ b/internal/repository/redis.go @@ -0,0 +1,45 @@ +package repository + +import ( + "context" + "fmt" + "time" + + "github.com/redis/go-redis/v9" + "go.uber.org/zap" +) + +type Redis struct { + Client *redis.Client + logger *zap.Logger +} + +func NewRedis(ctx context.Context, uri string, logger *zap.Logger) (*Redis, error) { + if uri == "" { + return nil, fmt.Errorf("redis URI is empty") + } + + opts, err := redis.ParseURL(uri) + if err != nil { + return nil, fmt.Errorf("redis parse URL: %w", err) + } + + client := redis.NewClient(opts) + + pingCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if err := client.Ping(pingCtx).Err(); err != nil { + _ = client.Close() + return nil, fmt.Errorf("redis ping: %w", err) + } + + logger.Info("connected to Redis") + return &Redis{Client: client, logger: logger}, nil +} + +func (r *Redis) Close() error { + if r == nil || r.Client == nil { + return nil + } + return r.Client.Close() +} diff --git a/internal/repository/shorturl_repo.go b/internal/repository/shorturl_repo.go new file mode 100644 index 0000000..a58853b --- /dev/null +++ b/internal/repository/shorturl_repo.go @@ -0,0 +1,157 @@ +package repository + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/prabhupadasays/spsays-bff/pkg/shorturl" +) + +// ShortenedURLRow mirrors the analytics-relevant columns of shortened_urls. +type ShortenedURLRow struct { + ID int64 + Code string + LongURL string + Disabled bool + ExpiresAt *time.Time + ClickCount int64 +} + +// ShortURLRepo wraps Postgres CRUD for the shortener. +type ShortURLRepo struct { + pg *Postgres +} + +func NewShortURLRepo(pg *Postgres) *ShortURLRepo { + return &ShortURLRepo{pg: pg} +} + +// Insert persists a new short URL. Returns the new row's id. If a unique +// violation occurs (race: another worker already inserted this code) the +// caller can detect via IsUniqueViolation(err). +func (r *ShortURLRepo) Insert( + ctx context.Context, + code string, + payload shorturl.PendingPayload, + parsed shorturl.Parsed, +) (int64, error) { + const q = ` + INSERT INTO shortened_urls ( + code, long_url, + path, content_type, book_code, division, chapter, verse, slug, + has_highlight, highlight_param, + creator_ip, creator_ua, creator_referer, creator_country, + created_at + ) VALUES ( + $1, $2, + $3, $4, $5, $6, $7, $8, $9, + $10, $11, + NULLIF($12,'')::inet, NULLIF($13,''), NULLIF($14,''), NULLIF($15,''), + $16 + ) + RETURNING id + ` + var id int64 + err := r.pg.Pool.QueryRow(ctx, q, + code, payload.URL, + parsed.Path, parsed.ContentType, + nullableStr(parsed.BookCode), + nullableStr(parsed.Division), + nullableStr(parsed.Chapter), + nullableStr(parsed.Verse), + nullableStr(parsed.Slug), + parsed.HasHighlight, nullableStr(parsed.HighlightParam), + payload.CreatorIP, payload.CreatorUA, payload.CreatorReferer, payload.CreatorCountry, + payload.CreatedAt, + ).Scan(&id) + if err != nil { + return 0, fmt.Errorf("insert shortened_url: %w", err) + } + return id, nil +} + +// GetByCode returns the row for code, or pgx.ErrNoRows if missing. +func (r *ShortURLRepo) GetByCode(ctx context.Context, code string) (*ShortenedURLRow, error) { + const q = ` + SELECT id, code, long_url, disabled, expires_at, click_count + FROM shortened_urls + WHERE code = $1 + LIMIT 1 + ` + var row ShortenedURLRow + err := r.pg.Pool.QueryRow(ctx, q, code).Scan( + &row.ID, &row.Code, &row.LongURL, &row.Disabled, &row.ExpiresAt, &row.ClickCount, + ) + if err != nil { + return nil, err + } + return &row, nil +} + +// RecordClick inserts a row in shortened_url_clicks and bumps the aggregate +// counters on the parent row in a single transaction. The parent lookup uses +// the code (not id) so this works even if the click worker is processing an +// event for a URL inserted by a different process. +func (r *ShortURLRepo) RecordClick(ctx context.Context, ev shorturl.ClickEvent) error { + tx, err := r.pg.Pool.Begin(ctx) + if err != nil { + return err + } + defer func() { _ = tx.Rollback(ctx) }() + + var id int64 + err = tx.QueryRow(ctx, `SELECT id FROM shortened_urls WHERE code = $1`, ev.Code).Scan(&id) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + // Parent row hasn't landed in PG yet (still pending). Skip silently; + // the click is lost. (Rare: only when a resolution and the parent + // insert race within the worker drain interval.) + return nil + } + return err + } + + if _, err := tx.Exec(ctx, ` + INSERT INTO shortened_url_clicks ( + short_url_id, code, clicked_at, client_ip, client_ua, referer, country, is_bot + ) VALUES ( + $1, $2, $3, NULLIF($4,'')::inet, NULLIF($5,''), NULLIF($6,''), NULLIF($7,''), $8 + )`, + id, ev.Code, ev.ClickedAt, ev.IP, ev.UA, ev.Referer, ev.Country, ev.IsBot, + ); err != nil { + return err + } + + if _, err := tx.Exec(ctx, ` + UPDATE shortened_urls + SET click_count = click_count + 1, + human_click_count = human_click_count + CASE WHEN $2 THEN 0 ELSE 1 END, + last_clicked_at = $1, + first_clicked_at = COALESCE(first_clicked_at, $1) + WHERE id = $3 + `, ev.ClickedAt, ev.IsBot, id); err != nil { + return err + } + + return tx.Commit(ctx) +} + +// IsUniqueViolation reports whether err is a Postgres unique constraint error. +func IsUniqueViolation(err error) bool { + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) { + return pgErr.Code == "23505" + } + return false +} + +func nullableStr(s string) any { + if s == "" { + return nil + } + return s +} diff --git a/internal/server/server.go b/internal/server/server.go index 7c18092..69bcd11 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1,7 +1,9 @@ package server import ( + "context" "net/http" + "path/filepath" "github.com/go-chi/chi/v5" chiMiddleware "github.com/go-chi/chi/v5/middleware" @@ -10,6 +12,8 @@ import ( "github.com/prabhupadasays/spsays-bff/internal/handler" "github.com/prabhupadasays/spsays-bff/internal/middleware" "github.com/prabhupadasays/spsays-bff/internal/repository" + "github.com/prabhupadasays/spsays-bff/internal/worker" + "github.com/prabhupadasays/spsays-bff/pkg/shorturl" "go.uber.org/zap" ) @@ -19,6 +23,11 @@ type Server struct { logger *zap.Logger router chi.Router mongo *repository.MongoDB + pg *repository.Postgres + redis *repository.Redis + + // Worker lifecycle + workerCancel context.CancelFunc // Handlers bookH *handler.BookHandler @@ -31,6 +40,7 @@ type Server struct { otherDocH *handler.OtherDocumentHandler transcriptionH *handler.TranscriptionHandler searchH *handler.SearchHandler + shortURLH *handler.ShortURLHandler } // New creates a new Server with all routes registered. @@ -72,10 +82,52 @@ func New(cfg *config.Config, logger *zap.Logger) *Server { searchH: handler.NewSearchHandler(tsClient), } + // URL shortener: Postgres (canonical) + Redis (queue + hot cache) + background workers. + // Connection failures degrade gracefully — the BFF stays up; only /share returns 503. + s.initShortener(cfg, logger) + s.setupRouter() return s } +// initShortener wires Postgres + Redis + the shorturl handler/workers. +// On connection failure it logs and leaves the components nil — the handler +// short-circuits to 503 so the rest of the BFF keeps working. +func (s *Server) initShortener(cfg *config.Config, logger *zap.Logger) { + ctx, cancel := context.WithCancel(context.Background()) + s.workerCancel = cancel + + pg, err := repository.NewPostgres(ctx, cfg.PostgresURI, logger) + if err != nil { + logger.Warn("URL shortener: Postgres unavailable — /share endpoints will return 503", + zap.Error(err)) + return + } + s.pg = pg + + migrationsDir, err := filepath.Abs("migrations") + if err == nil { + if err := pg.RunMigrations(ctx, migrationsDir); err != nil { + logger.Warn("URL shortener: migrations failed", zap.Error(err)) + } + } + + rd, err := repository.NewRedis(ctx, cfg.RedisURI, logger) + if err != nil { + logger.Warn("URL shortener: Redis unavailable — /share endpoints will return 503", + zap.Error(err)) + return + } + s.redis = rd + + shortRepo := repository.NewShortURLRepo(pg) + queue := shorturl.NewQueue(rd.Client) + s.shortURLH = handler.NewShortURLHandler(shortRepo, queue, logger) + + w := worker.NewShortURLWorker(shortRepo, queue, logger) + w.Run(ctx) +} + // Router returns the chi router. func (s *Server) Router() http.Handler { return s.router @@ -83,10 +135,20 @@ func (s *Server) Router() http.Handler { // Close shuts down all connections gracefully. func (s *Server) Close() { + if s.workerCancel != nil { + s.workerCancel() + } if err := s.mongo.Close(); err != nil { s.logger.Error("failed to close MongoDB connection", zap.Error(err)) } - // TODO: close PostgreSQL, Redis, etc. connections + if s.pg != nil { + s.pg.Close() + } + if s.redis != nil { + if err := s.redis.Close(); err != nil { + s.logger.Error("failed to close Redis connection", zap.Error(err)) + } + } s.logger.Info("all connections closed") } @@ -319,9 +381,21 @@ func (s *Server) setupRouter() { // TODO: register mention handlers }) - // URL Shortener - r.Route("/s", func(r chi.Router) { - // TODO: register URL shortener handlers + // URL Shortener — Postgres-backed, Redis-queued. + // The handler short-circuits to 503 if Postgres or Redis are unavailable. + r.Route("/share", func(r chi.Router) { + if s.shortURLH != nil { + r.Post("/", s.shortURLH.Create) + r.Get("/{code}", s.shortURLH.Resolve) + } else { + stub := func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusServiceUnavailable) + w.Write([]byte(`{"success":false,"error":{"code":503,"message":"url shortener unavailable"}}`)) + } + r.Post("/", stub) + r.Get("/{code}", stub) + } }) // Push Notifications diff --git a/internal/worker/shorturl_worker.go b/internal/worker/shorturl_worker.go new file mode 100644 index 0000000..a4b2abc --- /dev/null +++ b/internal/worker/shorturl_worker.go @@ -0,0 +1,106 @@ +package worker + +import ( + "context" + "errors" + "time" + + "github.com/prabhupadasays/spsays-bff/internal/repository" + "github.com/prabhupadasays/spsays-bff/pkg/shorturl" + "go.uber.org/zap" +) + +// ShortURLWorker drains the Redis create-queue + click-queue into Postgres. +type ShortURLWorker struct { + repo *repository.ShortURLRepo + queue *shorturl.Queue + logger *zap.Logger +} + +func NewShortURLWorker(repo *repository.ShortURLRepo, queue *shorturl.Queue, logger *zap.Logger) *ShortURLWorker { + return &ShortURLWorker{repo: repo, queue: queue, logger: logger} +} + +// Run starts the create-worker and click-worker goroutines. Both exit cleanly +// when ctx is cancelled. Call from server startup. +func (w *ShortURLWorker) Run(ctx context.Context) { + go w.runCreateLoop(ctx) + go w.runClickLoop(ctx) +} + +func (w *ShortURLWorker) runCreateLoop(ctx context.Context) { + w.logger.Info("shorturl create-worker started") + defer w.logger.Info("shorturl create-worker stopped") + + for { + if ctx.Err() != nil { + return + } + code, err := w.queue.DequeueCreate(ctx, 5*time.Second) + if err != nil { + if errors.Is(err, context.Canceled) { + return + } + w.logger.Warn("create-worker dequeue", zap.Error(err)) + time.Sleep(time.Second) + continue + } + if code == "" { + continue + } + + payload, ok := w.queue.LookupPending(ctx, code) + if !ok { + // Placeholder without a real payload — likely the producer crashed + // between MarkPending and EnqueueCreate. Skip. + w.logger.Warn("create-worker: missing pending payload", zap.String("code", code)) + continue + } + + parsed := shorturl.ParsePath(payload.URL) + if _, err := w.repo.Insert(ctx, code, payload, parsed); err != nil { + if repository.IsUniqueViolation(err) { + // Already persisted by a previous attempt; clean up the queue entry. + _ = w.queue.UnmarkPending(ctx, code) + continue + } + w.logger.Error("create-worker insert", zap.String("code", code), zap.Error(err)) + _ = w.queue.PushDeadLetter(ctx, code, err.Error()) + continue + } + + if err := w.queue.MarkCached(ctx, code, payload.URL); err != nil { + w.logger.Warn("create-worker mark cached", zap.String("code", code), zap.Error(err)) + } + if err := w.queue.UnmarkPending(ctx, code); err != nil { + w.logger.Warn("create-worker unmark pending", zap.String("code", code), zap.Error(err)) + } + } +} + +func (w *ShortURLWorker) runClickLoop(ctx context.Context) { + w.logger.Info("shorturl click-worker started") + defer w.logger.Info("shorturl click-worker stopped") + + for { + if ctx.Err() != nil { + return + } + ev, ok, err := w.queue.DequeueClick(ctx, 5*time.Second) + if err != nil { + if errors.Is(err, context.Canceled) { + return + } + w.logger.Warn("click-worker dequeue", zap.Error(err)) + time.Sleep(time.Second) + continue + } + if !ok { + continue + } + + if err := w.repo.RecordClick(ctx, ev); err != nil { + w.logger.Warn("click-worker record", zap.String("code", ev.Code), zap.Error(err)) + } + } +} diff --git a/migrations/0001_create_shortened_urls.down.sql b/migrations/0001_create_shortened_urls.down.sql new file mode 100644 index 0000000..a54d39b --- /dev/null +++ b/migrations/0001_create_shortened_urls.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS shortened_urls; diff --git a/migrations/0001_create_shortened_urls.up.sql b/migrations/0001_create_shortened_urls.up.sql new file mode 100644 index 0000000..273a371 --- /dev/null +++ b/migrations/0001_create_shortened_urls.up.sql @@ -0,0 +1,39 @@ +CREATE TABLE IF NOT EXISTS shortened_urls ( + id BIGSERIAL PRIMARY KEY, + code VARCHAR(16) NOT NULL UNIQUE, + long_url TEXT NOT NULL, + + path TEXT NOT NULL, + content_type VARCHAR(32) NOT NULL, + book_code VARCHAR(16), + division VARCHAR(16), + chapter VARCHAR(64), + verse VARCHAR(32), + slug VARCHAR(255), + + has_highlight BOOLEAN NOT NULL DEFAULT FALSE, + highlight_param TEXT, + + creator_ip INET, + creator_ua TEXT, + creator_referer TEXT, + creator_country CHAR(2), + + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + expires_at TIMESTAMPTZ, + disabled BOOLEAN NOT NULL DEFAULT FALSE, + disabled_reason TEXT, + disabled_at TIMESTAMPTZ, + + click_count BIGINT NOT NULL DEFAULT 0, + human_click_count BIGINT NOT NULL DEFAULT 0, + first_clicked_at TIMESTAMPTZ, + last_clicked_at TIMESTAMPTZ +); + +CREATE INDEX IF NOT EXISTS shortened_urls_created_at_idx ON shortened_urls (created_at DESC); +CREATE INDEX IF NOT EXISTS shortened_urls_content_type_idx ON shortened_urls (content_type); +CREATE INDEX IF NOT EXISTS shortened_urls_book_code_idx ON shortened_urls (book_code) WHERE book_code IS NOT NULL; +CREATE INDEX IF NOT EXISTS shortened_urls_creator_ip_idx ON shortened_urls (creator_ip) WHERE creator_ip IS NOT NULL; +CREATE INDEX IF NOT EXISTS shortened_urls_expires_at_idx ON shortened_urls (expires_at) WHERE expires_at IS NOT NULL; +CREATE INDEX IF NOT EXISTS shortened_urls_click_count_idx ON shortened_urls (click_count DESC) WHERE NOT disabled; diff --git a/migrations/0002_create_shortened_url_clicks.down.sql b/migrations/0002_create_shortened_url_clicks.down.sql new file mode 100644 index 0000000..c8f67e3 --- /dev/null +++ b/migrations/0002_create_shortened_url_clicks.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS shortened_url_clicks; diff --git a/migrations/0002_create_shortened_url_clicks.up.sql b/migrations/0002_create_shortened_url_clicks.up.sql new file mode 100644 index 0000000..4658430 --- /dev/null +++ b/migrations/0002_create_shortened_url_clicks.up.sql @@ -0,0 +1,17 @@ +CREATE TABLE IF NOT EXISTS shortened_url_clicks ( + id BIGSERIAL PRIMARY KEY, + short_url_id BIGINT NOT NULL REFERENCES shortened_urls(id) ON DELETE CASCADE, + code VARCHAR(16) NOT NULL, + clicked_at TIMESTAMPTZ NOT NULL DEFAULT now(), + client_ip INET, + client_ua TEXT, + referer TEXT, + country CHAR(2), + is_bot BOOLEAN NOT NULL DEFAULT FALSE +); + +CREATE INDEX IF NOT EXISTS shortened_url_clicks_short_url_id_idx ON shortened_url_clicks (short_url_id); +CREATE INDEX IF NOT EXISTS shortened_url_clicks_clicked_at_idx ON shortened_url_clicks (clicked_at DESC); +CREATE INDEX IF NOT EXISTS shortened_url_clicks_code_idx ON shortened_url_clicks (code); +CREATE INDEX IF NOT EXISTS shortened_url_clicks_country_idx ON shortened_url_clicks (country) WHERE country IS NOT NULL; +CREATE INDEX IF NOT EXISTS shortened_url_clicks_is_bot_idx ON shortened_url_clicks (is_bot); diff --git a/pkg/shorturl/bot.go b/pkg/shorturl/bot.go new file mode 100644 index 0000000..f96da71 --- /dev/null +++ b/pkg/shorturl/bot.go @@ -0,0 +1,52 @@ +package shorturl + +import ( + "regexp" + "strings" +) + +// botSubstrings catches the named social-platform link previewers explicitly, +// so we don't over-count human clicks from OG scraping. +var botSubstrings = []string{ + "facebookexternalhit", + "facebot", + "twitterbot", + "linkedinbot", + "slackbot", + "telegrambot", + "discordbot", + "pinterest", + "whatsapp", + "applebot", + "googlebot", + "bingbot", + "yahoo! slurp", + "duckduckbot", + "baiduspider", + "yandex", + "redditbot", + "embedly", + "vkshare", + "viber", + "signal", + "skypeuripreview", +} + +var botFallbackRe = regexp.MustCompile(`(?i)\b(bot|crawler|spider|preview|fetch)\b`) + +// IsBot returns true if the user-agent looks like an OG scraper / search +// crawler / link previewer, rather than a real human browser. False positives +// here only affect analytics aggregation (the is_bot flag); they never change +// redirect behavior. +func IsBot(userAgent string) bool { + if userAgent == "" { + return true // missing UA is almost always a bot or curl + } + ua := strings.ToLower(userAgent) + for _, needle := range botSubstrings { + if strings.Contains(ua, needle) { + return true + } + } + return botFallbackRe.MatchString(ua) +} diff --git a/pkg/shorturl/code.go b/pkg/shorturl/code.go new file mode 100644 index 0000000..caa979b --- /dev/null +++ b/pkg/shorturl/code.go @@ -0,0 +1,38 @@ +package shorturl + +import ( + "crypto/rand" + "fmt" + "math/big" + "regexp" +) + +// CodeLength is the length of generated short codes. 10 base36 characters +// give 36^10 ≈ 3.6 × 10^15 combinations. +const CodeLength = 10 + +// codeAlphabet is the base36 alphabet (lowercase letters + digits). +const codeAlphabet = "abcdefghijklmnopqrstuvwxyz0123456789" + +var codeAlphabetBig = big.NewInt(int64(len(codeAlphabet))) + +// RandomCode returns a cryptographically random base36 code of CodeLength chars. +func RandomCode() (string, error) { + buf := make([]byte, CodeLength) + for i := range buf { + n, err := rand.Int(rand.Reader, codeAlphabetBig) + if err != nil { + return "", fmt.Errorf("random code: %w", err) + } + buf[i] = codeAlphabet[n.Int64()] + } + return string(buf), nil +} + +var validCodeRe = regexp.MustCompile(`^[a-z0-9]{4,16}$`) + +// ValidCode returns true if s could plausibly be a short code we issued. +// Lenient on length (4-16) to allow future tuning without breaking old links. +func ValidCode(s string) bool { + return validCodeRe.MatchString(s) +} diff --git a/pkg/shorturl/parse.go b/pkg/shorturl/parse.go new file mode 100644 index 0000000..7197e35 --- /dev/null +++ b/pkg/shorturl/parse.go @@ -0,0 +1,189 @@ +package shorturl + +import ( + "net/url" + "strings" +) + +// Parsed contains analytics-friendly components extracted from a long URL. +// Unrecognized paths produce ContentType = "unknown" with the other fields +// left empty. +type Parsed struct { + Path string + ContentType string + BookCode string + Division string + Chapter string + Verse string + Slug string + HasHighlight bool + HighlightParam string +} + +// ParsePath decomposes a long URL (path + query) into analytics columns. +// It expects a same-site relative path; callers should run ValidatePath first. +func ParsePath(longURL string) Parsed { + u, err := url.Parse("http://x" + longURL) + if err != nil { + return Parsed{Path: longURL, ContentType: "unknown"} + } + + hl := u.Query().Get("hl") + out := Parsed{ + Path: u.Path, + HasHighlight: hl != "", + HighlightParam: truncate(hl, 1024), + ContentType: "unknown", + } + + parts := splitPath(u.Path) + if len(parts) == 0 { + return out + } + + switch parts[0] { + case "books": + out = parseBooks(parts, out) + case "letters": + if len(parts) >= 2 { + out.ContentType = "letter" + out.Slug = parts[1] + } + case "transcriptions": + if len(parts) >= 2 { + out.ContentType = "transcription" + out.Slug = parts[1] + } + case "essays": + if len(parts) >= 2 { + out.ContentType = "essay" + out.Slug = parts[1] + } + case "journals": + if len(parts) >= 2 { + out.ContentType = "journal" + out.Slug = parts[1] + } + if len(parts) >= 3 { + out.Chapter = parts[2] + } + case "magazines": + if len(parts) >= 2 { + out.ContentType = "magazine" + out.Slug = parts[1] + } + if len(parts) >= 3 { + out.Chapter = parts[2] + } + case "other-documents": + if len(parts) >= 2 { + out.ContentType = "other_document" + out.Slug = parts[1] + } + if len(parts) >= 3 { + out.Chapter = parts[2] + } + case "glossary": + out.ContentType = "glossary" + if len(parts) >= 2 { + out.Slug = parts[1] + } + } + return out +} + +// verseBookCodes are the books whose URL pattern has a per-verse leaf. +var verseBookCodes = map[string]bool{ + "bg": true, + "sb1962": true, + "iso": true, + "noi": true, + "bs": true, + "mm": true, + "nbs": true, +} + +// divisionVerseBookCodes have an extra division segment (canto for SB, līlā for CC). +var divisionVerseBookCodes = map[string]bool{ + "sb": true, + "cc": true, +} + +func parseBooks(parts []string, out Parsed) Parsed { + if len(parts) < 2 { + out.ContentType = "book" + return out + } + code := parts[1] + out.BookCode = code + + if divisionVerseBookCodes[code] { + // /books/sb/{canto}/{chapter}/{verse} + // /books/cc/{lila}/{chapter}/{verse} + if len(parts) >= 5 { + out.ContentType = "verse" + out.Division = parts[2] + out.Chapter = parts[3] + out.Verse = parts[4] + } else if len(parts) >= 4 { + out.ContentType = "verse_index" + out.Division = parts[2] + out.Chapter = parts[3] + } else { + out.ContentType = "book" + } + return out + } + + if verseBookCodes[code] { + // /books/{code}/{chapter}/{verse} (BG, SB1962, BS, MM, NBS) + // /books/{code}/{slug} (ISO, NOI — single-segment leaf) + if len(parts) >= 4 { + out.ContentType = "verse" + out.Chapter = parts[2] + out.Verse = parts[3] + } else if len(parts) >= 3 { + out.ContentType = "verse" + out.Slug = parts[2] + } else { + out.ContentType = "book" + } + return out + } + + // All other book codes are prose. KB has /kb/{volume}/{slug}; others /code/{slug}. + if code == "kb" && len(parts) >= 4 { + out.ContentType = "prose" + out.Division = parts[2] // volume number + out.Slug = parts[3] + return out + } + if len(parts) >= 3 { + out.ContentType = "prose" + out.Slug = parts[2] + return out + } + out.ContentType = "book" + return out +} + +func splitPath(p string) []string { + p = strings.Trim(p, "/") + if p == "" { + return nil + } + parts := strings.Split(p, "/") + for i, s := range parts { + if decoded, err := url.PathUnescape(s); err == nil { + parts[i] = decoded + } + } + return parts +} + +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] +} diff --git a/pkg/shorturl/queue.go b/pkg/shorturl/queue.go new file mode 100644 index 0000000..2b65c5b --- /dev/null +++ b/pkg/shorturl/queue.go @@ -0,0 +1,181 @@ +package shorturl + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/redis/go-redis/v9" +) + +// Redis key prefixes. +const ( + pendingKeyPrefix = "share:pending:" + cacheKeyPrefix = "share:cache:" + createQueueKey = "share:queue" + clickQueueKey = "share:click-queue" + deadLetterKey = "share:dead-letter" +) + +// Lifetimes. +const ( + PendingTTL = 1 * time.Hour + CacheTTL = 7 * 24 * time.Hour +) + +// PendingPayload is what the POST handler stores in Redis before the worker +// drains it into Postgres. +type PendingPayload struct { + URL string `json:"url"` + CreatorIP string `json:"creator_ip,omitempty"` + CreatorUA string `json:"creator_ua,omitempty"` + CreatorReferer string `json:"creator_referer,omitempty"` + CreatorCountry string `json:"creator_country,omitempty"` + CreatedAt time.Time `json:"created_at"` +} + +// ClickEvent is what the resolve handler enqueues per click. +type ClickEvent struct { + Code string `json:"code"` + IP string `json:"ip,omitempty"` + UA string `json:"ua,omitempty"` + Referer string `json:"referer,omitempty"` + Country string `json:"country,omitempty"` + IsBot bool `json:"is_bot"` + ClickedAt time.Time `json:"clicked_at"` +} + +// Queue provides write-behind primitives backed by Redis. The HTTP handlers +// only ever talk to Queue; the workers drain its lists into Postgres. +type Queue struct { + rdb *redis.Client +} + +func NewQueue(rdb *redis.Client) *Queue { + return &Queue{rdb: rdb} +} + +// AllocateCode generates a random unused code via SETNX on the pending key. +// The pending key is created as a placeholder ('_') so concurrent allocators +// won't collide; MarkPending then overwrites it with the real payload. +func (q *Queue) AllocateCode(ctx context.Context) (string, error) { + for attempt := 0; attempt < 5; attempt++ { + code, err := RandomCode() + if err != nil { + return "", err + } + ok, err := q.rdb.SetNX(ctx, pendingKeyPrefix+code, "_", PendingTTL).Result() + if err != nil { + return "", fmt.Errorf("setnx pending: %w", err) + } + if ok { + return code, nil + } + } + return "", errors.New("could not allocate unique code after 5 attempts") +} + +// MarkPending overwrites the placeholder with the real payload. +func (q *Queue) MarkPending(ctx context.Context, code string, p PendingPayload) error { + b, err := json.Marshal(p) + if err != nil { + return fmt.Errorf("marshal pending: %w", err) + } + return q.rdb.Set(ctx, pendingKeyPrefix+code, b, PendingTTL).Err() +} + +// LookupPending returns the in-flight payload, or (zero, false) if not present. +// Used by the resolve handler when a code was created very recently and the +// create-worker hasn't yet flushed it to Postgres. +func (q *Queue) LookupPending(ctx context.Context, code string) (PendingPayload, bool) { + val, err := q.rdb.Get(ctx, pendingKeyPrefix+code).Result() + if err != nil { + return PendingPayload{}, false + } + if val == "_" || val == "" { + return PendingPayload{}, false // placeholder, not yet marked + } + var p PendingPayload + if err := json.Unmarshal([]byte(val), &p); err != nil { + return PendingPayload{}, false + } + return p, true +} + +// UnmarkPending deletes the pending entry once the worker has persisted to PG. +func (q *Queue) UnmarkPending(ctx context.Context, code string) error { + return q.rdb.Del(ctx, pendingKeyPrefix+code).Err() +} + +// EnqueueCreate pushes a code onto the create-worker queue. +func (q *Queue) EnqueueCreate(ctx context.Context, code string) error { + return q.rdb.LPush(ctx, createQueueKey, code).Err() +} + +// DequeueCreate blocks until a code is available (or timeout). Returns +// ("", nil) on timeout, ("", err) on real error, (code, nil) on hit. +func (q *Queue) DequeueCreate(ctx context.Context, timeout time.Duration) (string, error) { + res, err := q.rdb.BRPop(ctx, timeout, createQueueKey).Result() + if err == redis.Nil { + return "", nil + } + if err != nil { + return "", err + } + if len(res) < 2 { + return "", nil + } + return res[1], nil +} + +// PushDeadLetter records an unprocessable create-job for manual inspection. +func (q *Queue) PushDeadLetter(ctx context.Context, code, reason string) error { + payload := map[string]any{"code": code, "reason": reason, "at": time.Now().UTC()} + b, _ := json.Marshal(payload) + return q.rdb.LPush(ctx, deadLetterKey, b).Err() +} + +// MarkCached stores a resolved long_url for fast future reads. Refreshed +// (TTL reset) on every successful resolve so popular links stay hot. +func (q *Queue) MarkCached(ctx context.Context, code, longURL string) error { + return q.rdb.Set(ctx, cacheKeyPrefix+code, longURL, CacheTTL).Err() +} + +// LookupCached returns the cached long_url, or (empty, false) on miss. +func (q *Queue) LookupCached(ctx context.Context, code string) (string, bool) { + val, err := q.rdb.Get(ctx, cacheKeyPrefix+code).Result() + if err != nil || val == "" { + return "", false + } + return val, true +} + +// EnqueueClick pushes a click event onto the click-worker queue. +func (q *Queue) EnqueueClick(ctx context.Context, ev ClickEvent) error { + b, err := json.Marshal(ev) + if err != nil { + return err + } + return q.rdb.LPush(ctx, clickQueueKey, b).Err() +} + +// DequeueClick blocks until a click event is available. +func (q *Queue) DequeueClick(ctx context.Context, timeout time.Duration) (ClickEvent, bool, error) { + res, err := q.rdb.BRPop(ctx, timeout, clickQueueKey).Result() + if err == redis.Nil { + return ClickEvent{}, false, nil + } + if err != nil { + return ClickEvent{}, false, err + } + if len(res) < 2 { + return ClickEvent{}, false, nil + } + var ev ClickEvent + if err := json.Unmarshal([]byte(res[1]), &ev); err != nil { + return ClickEvent{}, false, err + } + return ev, true, nil +} diff --git a/pkg/shorturl/validate.go b/pkg/shorturl/validate.go new file mode 100644 index 0000000..f7f64c0 --- /dev/null +++ b/pkg/shorturl/validate.go @@ -0,0 +1,42 @@ +package shorturl + +import ( + "fmt" + "strings" +) + +// MaxPathLength caps the long URL length we'll accept. 2KB is well beyond any +// real `?hl=` URL we generate while preventing pathological storage abuse. +const MaxPathLength = 2048 + +// ValidatePath verifies that s is a same-site relative URL — starts with "/", +// no scheme/host, no NUL bytes, within length. Prevents using the shortener +// as an open redirect to arbitrary external domains. +func ValidatePath(s string) error { + if s == "" { + return fmt.Errorf("url is empty") + } + if len(s) > MaxPathLength { + return fmt.Errorf("url exceeds %d bytes", MaxPathLength) + } + if !strings.HasPrefix(s, "/") { + return fmt.Errorf("url must start with '/'") + } + // "//host/…" is protocol-relative — disallow. + if strings.HasPrefix(s, "//") { + return fmt.Errorf("url must not be protocol-relative") + } + if strings.ContainsRune(s, 0) { + return fmt.Errorf("url contains NUL byte") + } + // No scheme allowed: scan up to the first '/' or '?' for ':'. + for i, c := range s { + if c == '?' || c == '#' { + break + } + if c == ':' && i > 0 { + return fmt.Errorf("url must not contain a scheme") + } + } + return nil +}