feat: add chat

2026-08-29 03:49:30 +05:30
parent 3b10e726b7
commit d50031cdb3
21 changed files with 6960 additions and 11 deletions
@@ -0,0 +1,313 @@
+// Package unfurl fetches link preview metadata for URLs users paste into chat.
+//
+// Fetching a user-supplied URL from the server is a server-side request forgery
+// primitive: without care, "https://x" can be pointed at 169.254.169.254 (cloud
+// metadata), at a database on 10.x, or at localhost. Everything below exists to
+// make that impossible:
+//
+//   - only http/https,
+//   - every connection is vetted at the RESOLVED IP, so a hostname that
+//     resolves to a private address is refused (and re-checked on each redirect
+//     hop, which is what defeats DNS rebinding),
+//   - redirects capped, response size capped, timeouts on everything,
+//   - only text/html is parsed.
+//
+// The metadata is deliberately fetched by the server rather than accepted from
+// the client. A client that could supply the title and image could make any
+// link claim to be anything.
+package unfurl
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"html"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+	"syscall"
+	"time"
+)
+
+// ErrBlocked is returned for a URL that resolves somewhere we refuse to fetch.
+var ErrBlocked = errors.New("refusing to fetch that address")
+
+// ErrNotHTML is returned when the response isn't a page we can read metadata
+// from — a PDF or an image has nothing to unfurl.
+var ErrNotHTML = errors.New("not an HTML page")
+
+const (
+	maxRedirects  = 3
+	maxBodyBytes  = 512 << 10 // 512 KiB is far more than any <head>
+	fetchTimeout  = 6 * time.Second
+	dialTimeout   = 3 * time.Second
+	maxFieldRunes = 300
+)
+
+// Preview is the metadata rendered as a link card.
+type Preview struct {
+	URL         string `json:"url"`
+	Title       string `json:"title,omitempty"`
+	Description string `json:"description,omitempty"`
+	SiteName    string `json:"site_name,omitempty"`
+	ImageURL    string `json:"image_url,omitempty"`
+}
+
+// Fetcher retrieves previews. Safe for concurrent use.
+type Fetcher struct {
+	client *http.Client
+	// allowLoopback exists only so the package's own tests can point at an
+	// httptest server. It is unexported and never set outside this package.
+	allowLoopback bool
+}
+
+// New returns a Fetcher with every guard enabled.
+func New() *Fetcher {
+	f := &Fetcher{}
+	f.client = &http.Client{
+		Timeout: fetchTimeout,
+		Transport: &http.Transport{
+			DialContext: (&net.Dialer{
+				Timeout: dialTimeout,
+				// Control runs after DNS resolution with the address actually
+				// being dialled, so it sees through a hostname that points at
+				// a private range — including one that changes between the
+				// lookup and the connection.
+				Control: func(network, address string, _ syscall.RawConn) error {
+					return f.checkAddr(network, address)
+				},
+			}).DialContext,
+			TLSHandshakeTimeout:   dialTimeout,
+			ResponseHeaderTimeout: fetchTimeout,
+			DisableKeepAlives:     true,
+		},
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			if len(via) >= maxRedirects {
+				return fmt.Errorf("too many redirects")
+			}
+			if !allowedScheme(req.URL) {
+				return ErrBlocked
+			}
+			return nil
+		},
+	}
+	return f
+}
+
+func (f *Fetcher) checkAddr(network, address string) error {
+	if network != "tcp4" && network != "tcp6" && network != "tcp" {
+		return ErrBlocked
+	}
+	host, _, err := net.SplitHostPort(address)
+	if err != nil {
+		return ErrBlocked
+	}
+	ip := net.ParseIP(host)
+	if ip == nil {
+		return ErrBlocked
+	}
+	if f.allowLoopback && ip.IsLoopback() {
+		return nil
+	}
+	if blockedIP(ip) {
+		return ErrBlocked
+	}
+	return nil
+}
+
+// blockedIP reports whether an address is one we refuse to talk to.
+func blockedIP(ip net.IP) bool {
+	// Normalise IPv4-mapped IPv6 (::ffff:127.0.0.1) so the v4 rules apply.
+	if v4 := ip.To4(); v4 != nil {
+		ip = v4
+	}
+	if ip.IsLoopback() || ip.IsPrivate() || ip.IsUnspecified() ||
+		ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() ||
+		ip.IsMulticast() || ip.IsInterfaceLocalMulticast() {
+		return true
+	}
+	// Ranges Go's helpers don't cover but which still reach infrastructure.
+	for _, cidr := range extraBlocked {
+		if cidr.Contains(ip) {
+			return true
+		}
+	}
+	return false
+}
+
+var extraBlocked = func() []*net.IPNet {
+	nets := []string{
+		"100.64.0.0/10",   // CGNAT
+		"192.0.0.0/24",    // IETF protocol assignments
+		"192.0.2.0/24",    // TEST-NET-1
+		"198.18.0.0/15",   // benchmarking
+		"198.51.100.0/24", // TEST-NET-2
+		"203.0.113.0/24",  // TEST-NET-3
+		"240.0.0.0/4",     // reserved
+		"::/128",          // unspecified
+		"64:ff9b::/96",    // NAT64, can map to private v4
+	}
+	out := make([]*net.IPNet, 0, len(nets))
+	for _, n := range nets {
+		if _, cidr, err := net.ParseCIDR(n); err == nil {
+			out = append(out, cidr)
+		}
+	}
+	return out
+}()
+
+func allowedScheme(u *url.URL) bool {
+	return u.Scheme == "http" || u.Scheme == "https"
+}
+
+// Fetch retrieves a preview for rawURL.
+func (f *Fetcher) Fetch(ctx context.Context, rawURL string) (*Preview, error) {
+	u, err := url.Parse(strings.TrimSpace(rawURL))
+	if err != nil {
+		return nil, fmt.Errorf("parse url: %w", err)
+	}
+	if !allowedScheme(u) || u.Host == "" {
+		return nil, ErrBlocked
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
+	if err != nil {
+		return nil, fmt.Errorf("build request: %w", err)
+	}
+	// Identify honestly; some sites serve richer metadata to a known crawler
+	// and it gives operators someone to contact.
+	req.Header.Set("User-Agent", "PrabhupadaSaysBot/1.0 (+https://prabhupadasays.org)")
+	req.Header.Set("Accept", "text/html,application/xhtml+xml")
+
+	resp, err := f.client.Do(req)
+	if err != nil {
+		// A blocked dial surfaces wrapped in a url.Error; unwrap so callers can
+		// tell "refused on purpose" from "the site is down".
+		if errors.Is(err, ErrBlocked) {
+			return nil, ErrBlocked
+		}
+		return nil, fmt.Errorf("fetch: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return nil, fmt.Errorf("unfurl status %d", resp.StatusCode)
+	}
+	if ct := resp.Header.Get("Content-Type"); ct != "" &&
+		!strings.HasPrefix(strings.ToLower(ct), "text/html") &&
+		!strings.HasPrefix(strings.ToLower(ct), "application/xhtml") {
+		return nil, ErrNotHTML
+	}
+
+	body, err := io.ReadAll(io.LimitReader(resp.Body, maxBodyBytes))
+	if err != nil {
+		return nil, fmt.Errorf("read body: %w", err)
+	}
+
+	// resp.Request.URL is the FINAL url after redirects — relative images must
+	// resolve against where we actually landed.
+	final := resp.Request.URL
+	p := extract(string(body))
+	p.URL = final.String()
+	if p.ImageURL != "" {
+		if abs, err := final.Parse(p.ImageURL); err == nil && allowedScheme(abs) {
+			p.ImageURL = abs.String()
+		} else {
+			p.ImageURL = ""
+		}
+	}
+	if p.Title == "" && p.Description == "" {
+		return nil, ErrNotHTML
+	}
+	return p, nil
+}
+
+var (
+	metaRe  = regexp.MustCompile(`(?is)<meta\s+([^>]*?)/?>`)
+	attrRe  = regexp.MustCompile(`(?is)([a-z0-9:_.-]+)\s*=\s*("([^"]*)"|'([^']*)'|([^\s"'>]+))`)
+	titleRe = regexp.MustCompile(`(?is)<title[^>]*>(.*?)</title>`)
+	headRe  = regexp.MustCompile(`(?is)</head\s*>`)
+	tagRe   = regexp.MustCompile(`(?s)<[^>]*>`)
+)
+
+// extract pulls Open Graph metadata out of a document's head.
+//
+// Written against the meta tags themselves rather than a full HTML parse: the
+// fields wanted here are a handful of well-formed <meta> elements, and this
+// keeps the BFF free of another dependency.
+func extract(doc string) *Preview {
+	// Metadata lives in <head>; stopping there avoids scanning a whole page.
+	if loc := headRe.FindStringIndex(doc); loc != nil {
+		doc = doc[:loc[0]]
+	}
+
+	p := &Preview{}
+	for _, m := range metaRe.FindAllStringSubmatch(doc, -1) {
+		attrs := parseAttrs(m[1])
+		key := attrs["property"]
+		if key == "" {
+			key = attrs["name"]
+		}
+		content := attrs["content"]
+		if content == "" {
+			continue
+		}
+		switch strings.ToLower(key) {
+		case "og:title", "twitter:title":
+			setIfEmpty(&p.Title, content)
+		case "og:description", "twitter:description", "description":
+			setIfEmpty(&p.Description, content)
+		case "og:site_name":
+			setIfEmpty(&p.SiteName, content)
+		case "og:image", "og:image:url", "og:image:secure_url", "twitter:image":
+			setIfEmpty(&p.ImageURL, content)
+		}
+	}
+
+	if p.Title == "" {
+		if m := titleRe.FindStringSubmatch(doc); m != nil {
+			p.Title = clean(tagRe.ReplaceAllString(m[1], ""))
+		}
+	}
+
+	p.Title = truncate(p.Title, maxFieldRunes)
+	p.Description = truncate(p.Description, maxFieldRunes)
+	p.SiteName = truncate(p.SiteName, maxFieldRunes)
+	return p
+}
+
+func parseAttrs(s string) map[string]string {
+	out := map[string]string{}
+	for _, m := range attrRe.FindAllStringSubmatch(s, -1) {
+		value := m[3]
+		if value == "" {
+			value = m[4]
+		}
+		if value == "" {
+			value = m[5]
+		}
+		out[strings.ToLower(m[1])] = clean(value)
+	}
+	return out
+}
+
+func setIfEmpty(dst *string, v string) {
+	if *dst == "" {
+		*dst = v
+	}
+}
+
+func clean(s string) string {
+	return strings.TrimSpace(html.UnescapeString(strings.Join(strings.Fields(s), " ")))
+}
+
+func truncate(s string, n int) string {
+	r := []rune(s)
+	if len(r) <= n {
+		return s
+	}
+	return strings.TrimSpace(string(r[:n])) + "…"
+}