feat: add url shortner
This commit is contained in:
Krispire Admin
@@ -0,0 +1,52 @@
|
||||
package shorturl
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// botSubstrings catches the named social-platform link previewers explicitly,
|
||||
// so we don't over-count human clicks from OG scraping.
|
||||
var botSubstrings = []string{
|
||||
"facebookexternalhit",
|
||||
"facebot",
|
||||
"twitterbot",
|
||||
"linkedinbot",
|
||||
"slackbot",
|
||||
"telegrambot",
|
||||
"discordbot",
|
||||
"pinterest",
|
||||
"whatsapp",
|
||||
"applebot",
|
||||
"googlebot",
|
||||
"bingbot",
|
||||
"yahoo! slurp",
|
||||
"duckduckbot",
|
||||
"baiduspider",
|
||||
"yandex",
|
||||
"redditbot",
|
||||
"embedly",
|
||||
"vkshare",
|
||||
"viber",
|
||||
"signal",
|
||||
"skypeuripreview",
|
||||
}
|
||||
|
||||
var botFallbackRe = regexp.MustCompile(`(?i)\b(bot|crawler|spider|preview|fetch)\b`)
|
||||
|
||||
// IsBot returns true if the user-agent looks like an OG scraper / search
|
||||
// crawler / link previewer, rather than a real human browser. False positives
|
||||
// here only affect analytics aggregation (the is_bot flag); they never change
|
||||
// redirect behavior.
|
||||
func IsBot(userAgent string) bool {
|
||||
if userAgent == "" {
|
||||
return true // missing UA is almost always a bot or curl
|
||||
}
|
||||
ua := strings.ToLower(userAgent)
|
||||
for _, needle := range botSubstrings {
|
||||
if strings.Contains(ua, needle) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return botFallbackRe.MatchString(ua)
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package shorturl
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"regexp"
|
||||
)
|
||||
|
||||
// CodeLength is the length of generated short codes. 10 base36 characters
|
||||
// give 36^10 ≈ 3.6 × 10^15 combinations.
|
||||
const CodeLength = 10
|
||||
|
||||
// codeAlphabet is the base36 alphabet (lowercase letters + digits).
|
||||
const codeAlphabet = "abcdefghijklmnopqrstuvwxyz0123456789"
|
||||
|
||||
var codeAlphabetBig = big.NewInt(int64(len(codeAlphabet)))
|
||||
|
||||
// RandomCode returns a cryptographically random base36 code of CodeLength chars.
|
||||
func RandomCode() (string, error) {
|
||||
buf := make([]byte, CodeLength)
|
||||
for i := range buf {
|
||||
n, err := rand.Int(rand.Reader, codeAlphabetBig)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("random code: %w", err)
|
||||
}
|
||||
buf[i] = codeAlphabet[n.Int64()]
|
||||
}
|
||||
return string(buf), nil
|
||||
}
|
||||
|
||||
var validCodeRe = regexp.MustCompile(`^[a-z0-9]{4,16}$`)
|
||||
|
||||
// ValidCode returns true if s could plausibly be a short code we issued.
|
||||
// Lenient on length (4-16) to allow future tuning without breaking old links.
|
||||
func ValidCode(s string) bool {
|
||||
return validCodeRe.MatchString(s)
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
package shorturl
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Parsed contains analytics-friendly components extracted from a long URL.
|
||||
// Unrecognized paths produce ContentType = "unknown" with the other fields
|
||||
// left empty.
|
||||
type Parsed struct {
|
||||
Path string
|
||||
ContentType string
|
||||
BookCode string
|
||||
Division string
|
||||
Chapter string
|
||||
Verse string
|
||||
Slug string
|
||||
HasHighlight bool
|
||||
HighlightParam string
|
||||
}
|
||||
|
||||
// ParsePath decomposes a long URL (path + query) into analytics columns.
|
||||
// It expects a same-site relative path; callers should run ValidatePath first.
|
||||
func ParsePath(longURL string) Parsed {
|
||||
u, err := url.Parse("http://x" + longURL)
|
||||
if err != nil {
|
||||
return Parsed{Path: longURL, ContentType: "unknown"}
|
||||
}
|
||||
|
||||
hl := u.Query().Get("hl")
|
||||
out := Parsed{
|
||||
Path: u.Path,
|
||||
HasHighlight: hl != "",
|
||||
HighlightParam: truncate(hl, 1024),
|
||||
ContentType: "unknown",
|
||||
}
|
||||
|
||||
parts := splitPath(u.Path)
|
||||
if len(parts) == 0 {
|
||||
return out
|
||||
}
|
||||
|
||||
switch parts[0] {
|
||||
case "books":
|
||||
out = parseBooks(parts, out)
|
||||
case "letters":
|
||||
if len(parts) >= 2 {
|
||||
out.ContentType = "letter"
|
||||
out.Slug = parts[1]
|
||||
}
|
||||
case "transcriptions":
|
||||
if len(parts) >= 2 {
|
||||
out.ContentType = "transcription"
|
||||
out.Slug = parts[1]
|
||||
}
|
||||
case "essays":
|
||||
if len(parts) >= 2 {
|
||||
out.ContentType = "essay"
|
||||
out.Slug = parts[1]
|
||||
}
|
||||
case "journals":
|
||||
if len(parts) >= 2 {
|
||||
out.ContentType = "journal"
|
||||
out.Slug = parts[1]
|
||||
}
|
||||
if len(parts) >= 3 {
|
||||
out.Chapter = parts[2]
|
||||
}
|
||||
case "magazines":
|
||||
if len(parts) >= 2 {
|
||||
out.ContentType = "magazine"
|
||||
out.Slug = parts[1]
|
||||
}
|
||||
if len(parts) >= 3 {
|
||||
out.Chapter = parts[2]
|
||||
}
|
||||
case "other-documents":
|
||||
if len(parts) >= 2 {
|
||||
out.ContentType = "other_document"
|
||||
out.Slug = parts[1]
|
||||
}
|
||||
if len(parts) >= 3 {
|
||||
out.Chapter = parts[2]
|
||||
}
|
||||
case "glossary":
|
||||
out.ContentType = "glossary"
|
||||
if len(parts) >= 2 {
|
||||
out.Slug = parts[1]
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// verseBookCodes are the books whose URL pattern has a per-verse leaf.
|
||||
var verseBookCodes = map[string]bool{
|
||||
"bg": true,
|
||||
"sb1962": true,
|
||||
"iso": true,
|
||||
"noi": true,
|
||||
"bs": true,
|
||||
"mm": true,
|
||||
"nbs": true,
|
||||
}
|
||||
|
||||
// divisionVerseBookCodes have an extra division segment (canto for SB, līlā for CC).
|
||||
var divisionVerseBookCodes = map[string]bool{
|
||||
"sb": true,
|
||||
"cc": true,
|
||||
}
|
||||
|
||||
func parseBooks(parts []string, out Parsed) Parsed {
|
||||
if len(parts) < 2 {
|
||||
out.ContentType = "book"
|
||||
return out
|
||||
}
|
||||
code := parts[1]
|
||||
out.BookCode = code
|
||||
|
||||
if divisionVerseBookCodes[code] {
|
||||
// /books/sb/{canto}/{chapter}/{verse}
|
||||
// /books/cc/{lila}/{chapter}/{verse}
|
||||
if len(parts) >= 5 {
|
||||
out.ContentType = "verse"
|
||||
out.Division = parts[2]
|
||||
out.Chapter = parts[3]
|
||||
out.Verse = parts[4]
|
||||
} else if len(parts) >= 4 {
|
||||
out.ContentType = "verse_index"
|
||||
out.Division = parts[2]
|
||||
out.Chapter = parts[3]
|
||||
} else {
|
||||
out.ContentType = "book"
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
if verseBookCodes[code] {
|
||||
// /books/{code}/{chapter}/{verse} (BG, SB1962, BS, MM, NBS)
|
||||
// /books/{code}/{slug} (ISO, NOI — single-segment leaf)
|
||||
if len(parts) >= 4 {
|
||||
out.ContentType = "verse"
|
||||
out.Chapter = parts[2]
|
||||
out.Verse = parts[3]
|
||||
} else if len(parts) >= 3 {
|
||||
out.ContentType = "verse"
|
||||
out.Slug = parts[2]
|
||||
} else {
|
||||
out.ContentType = "book"
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// All other book codes are prose. KB has /kb/{volume}/{slug}; others /code/{slug}.
|
||||
if code == "kb" && len(parts) >= 4 {
|
||||
out.ContentType = "prose"
|
||||
out.Division = parts[2] // volume number
|
||||
out.Slug = parts[3]
|
||||
return out
|
||||
}
|
||||
if len(parts) >= 3 {
|
||||
out.ContentType = "prose"
|
||||
out.Slug = parts[2]
|
||||
return out
|
||||
}
|
||||
out.ContentType = "book"
|
||||
return out
|
||||
}
|
||||
|
||||
func splitPath(p string) []string {
|
||||
p = strings.Trim(p, "/")
|
||||
if p == "" {
|
||||
return nil
|
||||
}
|
||||
parts := strings.Split(p, "/")
|
||||
for i, s := range parts {
|
||||
if decoded, err := url.PathUnescape(s); err == nil {
|
||||
parts[i] = decoded
|
||||
}
|
||||
}
|
||||
return parts
|
||||
}
|
||||
|
||||
func truncate(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
return s[:n]
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
package shorturl
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
// Redis key prefixes.
|
||||
const (
|
||||
pendingKeyPrefix = "share:pending:"
|
||||
cacheKeyPrefix = "share:cache:"
|
||||
createQueueKey = "share:queue"
|
||||
clickQueueKey = "share:click-queue"
|
||||
deadLetterKey = "share:dead-letter"
|
||||
)
|
||||
|
||||
// Lifetimes.
|
||||
const (
|
||||
PendingTTL = 1 * time.Hour
|
||||
CacheTTL = 7 * 24 * time.Hour
|
||||
)
|
||||
|
||||
// PendingPayload is what the POST handler stores in Redis before the worker
|
||||
// drains it into Postgres.
|
||||
type PendingPayload struct {
|
||||
URL string `json:"url"`
|
||||
CreatorIP string `json:"creator_ip,omitempty"`
|
||||
CreatorUA string `json:"creator_ua,omitempty"`
|
||||
CreatorReferer string `json:"creator_referer,omitempty"`
|
||||
CreatorCountry string `json:"creator_country,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
// ClickEvent is what the resolve handler enqueues per click.
|
||||
type ClickEvent struct {
|
||||
Code string `json:"code"`
|
||||
IP string `json:"ip,omitempty"`
|
||||
UA string `json:"ua,omitempty"`
|
||||
Referer string `json:"referer,omitempty"`
|
||||
Country string `json:"country,omitempty"`
|
||||
IsBot bool `json:"is_bot"`
|
||||
ClickedAt time.Time `json:"clicked_at"`
|
||||
}
|
||||
|
||||
// Queue provides write-behind primitives backed by Redis. The HTTP handlers
|
||||
// only ever talk to Queue; the workers drain its lists into Postgres.
|
||||
type Queue struct {
|
||||
rdb *redis.Client
|
||||
}
|
||||
|
||||
func NewQueue(rdb *redis.Client) *Queue {
|
||||
return &Queue{rdb: rdb}
|
||||
}
|
||||
|
||||
// AllocateCode generates a random unused code via SETNX on the pending key.
|
||||
// The pending key is created as a placeholder ('_') so concurrent allocators
|
||||
// won't collide; MarkPending then overwrites it with the real payload.
|
||||
func (q *Queue) AllocateCode(ctx context.Context) (string, error) {
|
||||
for attempt := 0; attempt < 5; attempt++ {
|
||||
code, err := RandomCode()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
ok, err := q.rdb.SetNX(ctx, pendingKeyPrefix+code, "_", PendingTTL).Result()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("setnx pending: %w", err)
|
||||
}
|
||||
if ok {
|
||||
return code, nil
|
||||
}
|
||||
}
|
||||
return "", errors.New("could not allocate unique code after 5 attempts")
|
||||
}
|
||||
|
||||
// MarkPending overwrites the placeholder with the real payload.
|
||||
func (q *Queue) MarkPending(ctx context.Context, code string, p PendingPayload) error {
|
||||
b, err := json.Marshal(p)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal pending: %w", err)
|
||||
}
|
||||
return q.rdb.Set(ctx, pendingKeyPrefix+code, b, PendingTTL).Err()
|
||||
}
|
||||
|
||||
// LookupPending returns the in-flight payload, or (zero, false) if not present.
|
||||
// Used by the resolve handler when a code was created very recently and the
|
||||
// create-worker hasn't yet flushed it to Postgres.
|
||||
func (q *Queue) LookupPending(ctx context.Context, code string) (PendingPayload, bool) {
|
||||
val, err := q.rdb.Get(ctx, pendingKeyPrefix+code).Result()
|
||||
if err != nil {
|
||||
return PendingPayload{}, false
|
||||
}
|
||||
if val == "_" || val == "" {
|
||||
return PendingPayload{}, false // placeholder, not yet marked
|
||||
}
|
||||
var p PendingPayload
|
||||
if err := json.Unmarshal([]byte(val), &p); err != nil {
|
||||
return PendingPayload{}, false
|
||||
}
|
||||
return p, true
|
||||
}
|
||||
|
||||
// UnmarkPending deletes the pending entry once the worker has persisted to PG.
|
||||
func (q *Queue) UnmarkPending(ctx context.Context, code string) error {
|
||||
return q.rdb.Del(ctx, pendingKeyPrefix+code).Err()
|
||||
}
|
||||
|
||||
// EnqueueCreate pushes a code onto the create-worker queue.
|
||||
func (q *Queue) EnqueueCreate(ctx context.Context, code string) error {
|
||||
return q.rdb.LPush(ctx, createQueueKey, code).Err()
|
||||
}
|
||||
|
||||
// DequeueCreate blocks until a code is available (or timeout). Returns
|
||||
// ("", nil) on timeout, ("", err) on real error, (code, nil) on hit.
|
||||
func (q *Queue) DequeueCreate(ctx context.Context, timeout time.Duration) (string, error) {
|
||||
res, err := q.rdb.BRPop(ctx, timeout, createQueueKey).Result()
|
||||
if err == redis.Nil {
|
||||
return "", nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(res) < 2 {
|
||||
return "", nil
|
||||
}
|
||||
return res[1], nil
|
||||
}
|
||||
|
||||
// PushDeadLetter records an unprocessable create-job for manual inspection.
|
||||
func (q *Queue) PushDeadLetter(ctx context.Context, code, reason string) error {
|
||||
payload := map[string]any{"code": code, "reason": reason, "at": time.Now().UTC()}
|
||||
b, _ := json.Marshal(payload)
|
||||
return q.rdb.LPush(ctx, deadLetterKey, b).Err()
|
||||
}
|
||||
|
||||
// MarkCached stores a resolved long_url for fast future reads. Refreshed
|
||||
// (TTL reset) on every successful resolve so popular links stay hot.
|
||||
func (q *Queue) MarkCached(ctx context.Context, code, longURL string) error {
|
||||
return q.rdb.Set(ctx, cacheKeyPrefix+code, longURL, CacheTTL).Err()
|
||||
}
|
||||
|
||||
// LookupCached returns the cached long_url, or (empty, false) on miss.
|
||||
func (q *Queue) LookupCached(ctx context.Context, code string) (string, bool) {
|
||||
val, err := q.rdb.Get(ctx, cacheKeyPrefix+code).Result()
|
||||
if err != nil || val == "" {
|
||||
return "", false
|
||||
}
|
||||
return val, true
|
||||
}
|
||||
|
||||
// EnqueueClick pushes a click event onto the click-worker queue.
|
||||
func (q *Queue) EnqueueClick(ctx context.Context, ev ClickEvent) error {
|
||||
b, err := json.Marshal(ev)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return q.rdb.LPush(ctx, clickQueueKey, b).Err()
|
||||
}
|
||||
|
||||
// DequeueClick blocks until a click event is available.
|
||||
func (q *Queue) DequeueClick(ctx context.Context, timeout time.Duration) (ClickEvent, bool, error) {
|
||||
res, err := q.rdb.BRPop(ctx, timeout, clickQueueKey).Result()
|
||||
if err == redis.Nil {
|
||||
return ClickEvent{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return ClickEvent{}, false, err
|
||||
}
|
||||
if len(res) < 2 {
|
||||
return ClickEvent{}, false, nil
|
||||
}
|
||||
var ev ClickEvent
|
||||
if err := json.Unmarshal([]byte(res[1]), &ev); err != nil {
|
||||
return ClickEvent{}, false, err
|
||||
}
|
||||
return ev, true, nil
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package shorturl
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// MaxPathLength caps the long URL length we'll accept. 2KB is well beyond any
|
||||
// real `?hl=` URL we generate while preventing pathological storage abuse.
|
||||
const MaxPathLength = 2048
|
||||
|
||||
// ValidatePath verifies that s is a same-site relative URL — starts with "/",
|
||||
// no scheme/host, no NUL bytes, within length. Prevents using the shortener
|
||||
// as an open redirect to arbitrary external domains.
|
||||
func ValidatePath(s string) error {
|
||||
if s == "" {
|
||||
return fmt.Errorf("url is empty")
|
||||
}
|
||||
if len(s) > MaxPathLength {
|
||||
return fmt.Errorf("url exceeds %d bytes", MaxPathLength)
|
||||
}
|
||||
if !strings.HasPrefix(s, "/") {
|
||||
return fmt.Errorf("url must start with '/'")
|
||||
}
|
||||
// "//host/…" is protocol-relative — disallow.
|
||||
if strings.HasPrefix(s, "//") {
|
||||
return fmt.Errorf("url must not be protocol-relative")
|
||||
}
|
||||
if strings.ContainsRune(s, 0) {
|
||||
return fmt.Errorf("url contains NUL byte")
|
||||
}
|
||||
// No scheme allowed: scan up to the first '/' or '?' for ':'.
|
||||
for i, c := range s {
|
||||
if c == '?' || c == '#' {
|
||||
break
|
||||
}
|
||||
if c == ':' && i > 0 {
|
||||
return fmt.Errorf("url must not contain a scheme")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user