feat: add chat

2026-08-29 03:49:30 +05:30
parent 3b10e726b7
commit d50031cdb3
21 changed files with 6960 additions and 11 deletions
@@ -0,0 +1,313 @@
+// Package unfurl fetches link preview metadata for URLs users paste into chat.
+//
+// Fetching a user-supplied URL from the server is a server-side request forgery
+// primitive: without care, "https://x" can be pointed at 169.254.169.254 (cloud
+// metadata), at a database on 10.x, or at localhost. Everything below exists to
+// make that impossible:
+//
+//   - only http/https,
+//   - every connection is vetted at the RESOLVED IP, so a hostname that
+//     resolves to a private address is refused (and re-checked on each redirect
+//     hop, which is what defeats DNS rebinding),
+//   - redirects capped, response size capped, timeouts on everything,
+//   - only text/html is parsed.
+//
+// The metadata is deliberately fetched by the server rather than accepted from
+// the client. A client that could supply the title and image could make any
+// link claim to be anything.
+package unfurl
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"html"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+	"syscall"
+	"time"
+)
+
+// ErrBlocked is returned for a URL that resolves somewhere we refuse to fetch.
+var ErrBlocked = errors.New("refusing to fetch that address")
+
+// ErrNotHTML is returned when the response isn't a page we can read metadata
+// from — a PDF or an image has nothing to unfurl.
+var ErrNotHTML = errors.New("not an HTML page")
+
+const (
+	maxRedirects  = 3
+	maxBodyBytes  = 512 << 10 // 512 KiB is far more than any <head>
+	fetchTimeout  = 6 * time.Second
+	dialTimeout   = 3 * time.Second
+	maxFieldRunes = 300
+)
+
+// Preview is the metadata rendered as a link card.
+type Preview struct {
+	URL         string `json:"url"`
+	Title       string `json:"title,omitempty"`
+	Description string `json:"description,omitempty"`
+	SiteName    string `json:"site_name,omitempty"`
+	ImageURL    string `json:"image_url,omitempty"`
+}
+
+// Fetcher retrieves previews. Safe for concurrent use.
+type Fetcher struct {
+	client *http.Client
+	// allowLoopback exists only so the package's own tests can point at an
+	// httptest server. It is unexported and never set outside this package.
+	allowLoopback bool
+}
+
+// New returns a Fetcher with every guard enabled.
+func New() *Fetcher {
+	f := &Fetcher{}
+	f.client = &http.Client{
+		Timeout: fetchTimeout,
+		Transport: &http.Transport{
+			DialContext: (&net.Dialer{
+				Timeout: dialTimeout,
+				// Control runs after DNS resolution with the address actually
+				// being dialled, so it sees through a hostname that points at
+				// a private range — including one that changes between the
+				// lookup and the connection.
+				Control: func(network, address string, _ syscall.RawConn) error {
+					return f.checkAddr(network, address)
+				},
+			}).DialContext,
+			TLSHandshakeTimeout:   dialTimeout,
+			ResponseHeaderTimeout: fetchTimeout,
+			DisableKeepAlives:     true,
+		},
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			if len(via) >= maxRedirects {
+				return fmt.Errorf("too many redirects")
+			}
+			if !allowedScheme(req.URL) {
+				return ErrBlocked
+			}
+			return nil
+		},
+	}
+	return f
+}
+
+func (f *Fetcher) checkAddr(network, address string) error {
+	if network != "tcp4" && network != "tcp6" && network != "tcp" {
+		return ErrBlocked
+	}
+	host, _, err := net.SplitHostPort(address)
+	if err != nil {
+		return ErrBlocked
+	}
+	ip := net.ParseIP(host)
+	if ip == nil {
+		return ErrBlocked
+	}
+	if f.allowLoopback && ip.IsLoopback() {
+		return nil
+	}
+	if blockedIP(ip) {
+		return ErrBlocked
+	}
+	return nil
+}
+
+// blockedIP reports whether an address is one we refuse to talk to.
+func blockedIP(ip net.IP) bool {
+	// Normalise IPv4-mapped IPv6 (::ffff:127.0.0.1) so the v4 rules apply.
+	if v4 := ip.To4(); v4 != nil {
+		ip = v4
+	}
+	if ip.IsLoopback() || ip.IsPrivate() || ip.IsUnspecified() ||
+		ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() ||
+		ip.IsMulticast() || ip.IsInterfaceLocalMulticast() {
+		return true
+	}
+	// Ranges Go's helpers don't cover but which still reach infrastructure.
+	for _, cidr := range extraBlocked {
+		if cidr.Contains(ip) {
+			return true
+		}
+	}
+	return false
+}
+
+var extraBlocked = func() []*net.IPNet {
+	nets := []string{
+		"100.64.0.0/10",   // CGNAT
+		"192.0.0.0/24",    // IETF protocol assignments
+		"192.0.2.0/24",    // TEST-NET-1
+		"198.18.0.0/15",   // benchmarking
+		"198.51.100.0/24", // TEST-NET-2
+		"203.0.113.0/24",  // TEST-NET-3
+		"240.0.0.0/4",     // reserved
+		"::/128",          // unspecified
+		"64:ff9b::/96",    // NAT64, can map to private v4
+	}
+	out := make([]*net.IPNet, 0, len(nets))
+	for _, n := range nets {
+		if _, cidr, err := net.ParseCIDR(n); err == nil {
+			out = append(out, cidr)
+		}
+	}
+	return out
+}()
+
+func allowedScheme(u *url.URL) bool {
+	return u.Scheme == "http" || u.Scheme == "https"
+}
+
+// Fetch retrieves a preview for rawURL.
+func (f *Fetcher) Fetch(ctx context.Context, rawURL string) (*Preview, error) {
+	u, err := url.Parse(strings.TrimSpace(rawURL))
+	if err != nil {
+		return nil, fmt.Errorf("parse url: %w", err)
+	}
+	if !allowedScheme(u) || u.Host == "" {
+		return nil, ErrBlocked
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
+	if err != nil {
+		return nil, fmt.Errorf("build request: %w", err)
+	}
+	// Identify honestly; some sites serve richer metadata to a known crawler
+	// and it gives operators someone to contact.
+	req.Header.Set("User-Agent", "PrabhupadaSaysBot/1.0 (+https://prabhupadasays.org)")
+	req.Header.Set("Accept", "text/html,application/xhtml+xml")
+
+	resp, err := f.client.Do(req)
+	if err != nil {
+		// A blocked dial surfaces wrapped in a url.Error; unwrap so callers can
+		// tell "refused on purpose" from "the site is down".
+		if errors.Is(err, ErrBlocked) {
+			return nil, ErrBlocked
+		}
+		return nil, fmt.Errorf("fetch: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return nil, fmt.Errorf("unfurl status %d", resp.StatusCode)
+	}
+	if ct := resp.Header.Get("Content-Type"); ct != "" &&
+		!strings.HasPrefix(strings.ToLower(ct), "text/html") &&
+		!strings.HasPrefix(strings.ToLower(ct), "application/xhtml") {
+		return nil, ErrNotHTML
+	}
+
+	body, err := io.ReadAll(io.LimitReader(resp.Body, maxBodyBytes))
+	if err != nil {
+		return nil, fmt.Errorf("read body: %w", err)
+	}
+
+	// resp.Request.URL is the FINAL url after redirects — relative images must
+	// resolve against where we actually landed.
+	final := resp.Request.URL
+	p := extract(string(body))
+	p.URL = final.String()
+	if p.ImageURL != "" {
+		if abs, err := final.Parse(p.ImageURL); err == nil && allowedScheme(abs) {
+			p.ImageURL = abs.String()
+		} else {
+			p.ImageURL = ""
+		}
+	}
+	if p.Title == "" && p.Description == "" {
+		return nil, ErrNotHTML
+	}
+	return p, nil
+}
+
+var (
+	metaRe  = regexp.MustCompile(`(?is)<meta\s+([^>]*?)/?>`)
+	attrRe  = regexp.MustCompile(`(?is)([a-z0-9:_.-]+)\s*=\s*("([^"]*)"|'([^']*)'|([^\s"'>]+))`)
+	titleRe = regexp.MustCompile(`(?is)<title[^>]*>(.*?)</title>`)
+	headRe  = regexp.MustCompile(`(?is)</head\s*>`)
+	tagRe   = regexp.MustCompile(`(?s)<[^>]*>`)
+)
+
+// extract pulls Open Graph metadata out of a document's head.
+//
+// Written against the meta tags themselves rather than a full HTML parse: the
+// fields wanted here are a handful of well-formed <meta> elements, and this
+// keeps the BFF free of another dependency.
+func extract(doc string) *Preview {
+	// Metadata lives in <head>; stopping there avoids scanning a whole page.
+	if loc := headRe.FindStringIndex(doc); loc != nil {
+		doc = doc[:loc[0]]
+	}
+
+	p := &Preview{}
+	for _, m := range metaRe.FindAllStringSubmatch(doc, -1) {
+		attrs := parseAttrs(m[1])
+		key := attrs["property"]
+		if key == "" {
+			key = attrs["name"]
+		}
+		content := attrs["content"]
+		if content == "" {
+			continue
+		}
+		switch strings.ToLower(key) {
+		case "og:title", "twitter:title":
+			setIfEmpty(&p.Title, content)
+		case "og:description", "twitter:description", "description":
+			setIfEmpty(&p.Description, content)
+		case "og:site_name":
+			setIfEmpty(&p.SiteName, content)
+		case "og:image", "og:image:url", "og:image:secure_url", "twitter:image":
+			setIfEmpty(&p.ImageURL, content)
+		}
+	}
+
+	if p.Title == "" {
+		if m := titleRe.FindStringSubmatch(doc); m != nil {
+			p.Title = clean(tagRe.ReplaceAllString(m[1], ""))
+		}
+	}
+
+	p.Title = truncate(p.Title, maxFieldRunes)
+	p.Description = truncate(p.Description, maxFieldRunes)
+	p.SiteName = truncate(p.SiteName, maxFieldRunes)
+	return p
+}
+
+func parseAttrs(s string) map[string]string {
+	out := map[string]string{}
+	for _, m := range attrRe.FindAllStringSubmatch(s, -1) {
+		value := m[3]
+		if value == "" {
+			value = m[4]
+		}
+		if value == "" {
+			value = m[5]
+		}
+		out[strings.ToLower(m[1])] = clean(value)
+	}
+	return out
+}
+
+func setIfEmpty(dst *string, v string) {
+	if *dst == "" {
+		*dst = v
+	}
+}
+
+func clean(s string) string {
+	return strings.TrimSpace(html.UnescapeString(strings.Join(strings.Fields(s), " ")))
+}
+
+func truncate(s string, n int) string {
+	r := []rune(s)
+	if len(r) <= n {
+		return s
+	}
+	return strings.TrimSpace(string(r[:n])) + "…"
+}
@@ -0,0 +1,233 @@
+package unfurl
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+// The blocklist is the security boundary of this package: everything else is
+// plumbing around it. A URL that resolves into any of these ranges must never
+// be fetched, or the BFF becomes a proxy into the private network.
+func TestBlockedIP(t *testing.T) {
+	blocked := []string{
+		"127.0.0.1",        // loopback
+		"::1",              // loopback v6
+		"::ffff:127.0.0.1", // loopback via IPv4-mapped v6
+		"10.0.0.5",         // RFC1918
+		"172.16.4.2",       // RFC1918
+		"192.168.1.1",      // RFC1918
+		"::ffff:10.0.0.5",  // RFC1918 via IPv4-mapped v6
+		"169.254.169.254",  // cloud instance metadata
+		"fe80::1",          // link-local v6
+		"fd00::1",          // unique local v6
+		"0.0.0.0",          // unspecified
+		"::",               // unspecified v6
+		"224.0.0.1",        // multicast
+		"100.64.0.1",       // CGNAT
+		"198.18.0.1",       // benchmarking
+		"240.0.0.1",        // reserved
+	}
+	for _, s := range blocked {
+		if ip := net.ParseIP(s); ip == nil || !blockedIP(ip) {
+			t.Errorf("blockedIP(%s) = false, want true", s)
+		}
+	}
+
+	allowed := []string{"1.1.1.1", "8.8.8.8", "93.184.216.34", "2606:2800:220:1:248:1893:25c8:1946"}
+	for _, s := range allowed {
+		if ip := net.ParseIP(s); ip == nil || blockedIP(ip) {
+			t.Errorf("blockedIP(%s) = true, want false", s)
+		}
+	}
+}
+
+// A hostname that resolves to a private address must be refused at dial time —
+// this is what stops "http://internal.example.com" and DNS rebinding.
+func TestFetch_RefusesPrivateAddresses(t *testing.T) {
+	f := New() // loopback NOT allowed
+	for _, raw := range []string{
+		"http://127.0.0.1:80/",
+		"http://localhost/",
+		"http://169.254.169.254/latest/meta-data/",
+		"http://10.0.0.1/",
+		"http://[::1]/",
+	} {
+		_, err := f.Fetch(context.Background(), raw)
+		if err == nil {
+			t.Errorf("Fetch(%s) succeeded — it must be refused", raw)
+		}
+	}
+}
+
+func TestFetch_RejectsNonHTTPSchemes(t *testing.T) {
+	f := New()
+	for _, raw := range []string{
+		"file:///etc/passwd",
+		"gopher://example.org/",
+		"ftp://example.org/",
+		"redis://10.0.0.1:6379",
+		"//example.org",
+	} {
+		if _, err := f.Fetch(context.Background(), raw); !errors.Is(err, ErrBlocked) {
+			t.Errorf("Fetch(%s) err = %v, want ErrBlocked", raw, err)
+		}
+	}
+}
+
+// testFetcher permits loopback so the fetch path can be exercised against an
+// httptest server. The flag is unexported and set only here.
+func testFetcher() *Fetcher {
+	f := New()
+	f.allowLoopback = true
+	return f
+}
+
+func TestFetch_ReadsOpenGraph(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html; charset=utf-8")
+		fmt.Fprint(w, `<html><head>
+			<title>Fallback title</title>
+			<meta property="og:title" content="Bhagavad-g&#012ABC;t&#257; As It Is">
+			<meta property="og:description" content="Chapter 2, verse 13">
+			<meta property="og:site_name" content="Prabhupāda Says">
+			<meta property="og:image" content="/img/cover.jpg">
+		</head><body>ignored</body></html>`)
+	}))
+	defer srv.Close()
+
+	p, err := testFetcher().Fetch(context.Background(), srv.URL)
+	if err != nil {
+		t.Fatalf("Fetch: %v", err)
+	}
+	if p.Description != "Chapter 2, verse 13" {
+		t.Errorf("description = %q", p.Description)
+	}
+	if p.SiteName != "Prabhupāda Says" {
+		t.Errorf("site_name = %q", p.SiteName)
+	}
+	// Relative og:image must be resolved against the page it came from.
+	if !strings.HasPrefix(p.ImageURL, srv.URL) || !strings.HasSuffix(p.ImageURL, "/img/cover.jpg") {
+		t.Errorf("image = %q, want an absolute URL under %s", p.ImageURL, srv.URL)
+	}
+}
+
+func TestFetch_FallsBackToTitleTag(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html")
+		fmt.Fprint(w, `<html><head><title>  Plain &amp; simple  </title>
+			<meta name="description" content="From the name attribute"></head></html>`)
+	}))
+	defer srv.Close()
+
+	p, err := testFetcher().Fetch(context.Background(), srv.URL)
+	if err != nil {
+		t.Fatalf("Fetch: %v", err)
+	}
+	if p.Title != "Plain & simple" {
+		t.Errorf("title = %q, want entities decoded and whitespace collapsed", p.Title)
+	}
+	if p.Description != "From the name attribute" {
+		t.Errorf("description = %q", p.Description)
+	}
+}
+
+// A redirect that lands on a private address must be refused at the hop, not
+// followed because the first host looked fine.
+func TestFetch_RefusesRedirectToPrivateAddress(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, "http://169.254.169.254/latest/meta-data/", http.StatusFound)
+	}))
+	defer srv.Close()
+
+	if _, err := testFetcher().Fetch(context.Background(), srv.URL); err == nil {
+		t.Fatal("followed a redirect into link-local space")
+	}
+}
+
+func TestFetch_RejectsNonHTML(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/pdf")
+		fmt.Fprint(w, "%PDF-1.7")
+	}))
+	defer srv.Close()
+
+	if _, err := testFetcher().Fetch(context.Background(), srv.URL); !errors.Is(err, ErrNotHTML) {
+		t.Errorf("err = %v, want ErrNotHTML", err)
+	}
+}
+
+// A page with no metadata is not a preview — better no card than an empty one.
+func TestFetch_EmptyMetadataIsNotAPreview(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html")
+		fmt.Fprint(w, `<html><head></head><body>nothing here</body></html>`)
+	}))
+	defer srv.Close()
+
+	if _, err := testFetcher().Fetch(context.Background(), srv.URL); !errors.Is(err, ErrNotHTML) {
+		t.Errorf("err = %v, want ErrNotHTML", err)
+	}
+}
+
+// A body far larger than the cap must not be read whole.
+func TestFetch_CapsBodySize(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html")
+		fmt.Fprint(w, `<html><head><meta property="og:title" content="Huge"></head><body>`)
+		chunk := strings.Repeat("x", 64<<10)
+		for i := 0; i < 64; i++ { // 4 MiB, well past maxBodyBytes
+			fmt.Fprint(w, chunk)
+		}
+	}))
+	defer srv.Close()
+
+	p, err := testFetcher().Fetch(context.Background(), srv.URL)
+	if err != nil {
+		t.Fatalf("Fetch: %v", err)
+	}
+	if p.Title != "Huge" {
+		t.Errorf("title = %q", p.Title)
+	}
+}
+
+func TestExtract_LongFieldsAreTruncated(t *testing.T) {
+	long := strings.Repeat("ā", 500)
+	p := extract(`<meta property="og:title" content="` + long + `">`)
+	if len([]rune(p.Title)) > maxFieldRunes+1 {
+		t.Errorf("title kept %d runes, want it truncated", len([]rune(p.Title)))
+	}
+	if !strings.HasSuffix(p.Title, "…") {
+		t.Error("truncated title should be marked with an ellipsis")
+	}
+}
+
+func TestExtract_HandlesAttributeOrderAndQuotes(t *testing.T) {
+	p := extract(`
+		<meta content='Single quoted' property='og:title'>
+		<meta content=Unquoted name=og:site_name>
+		<meta property="og:description" content="Ordered normally">`)
+	if p.Title != "Single quoted" {
+		t.Errorf("title = %q", p.Title)
+	}
+	if p.SiteName != "Unquoted" {
+		t.Errorf("site_name = %q", p.SiteName)
+	}
+	if p.Description != "Ordered normally" {
+		t.Errorf("description = %q", p.Description)
+	}
+}
+
+// Metadata is read from <head>; a <meta> forged in the body must not win.
+func TestExtract_IgnoresBodyContent(t *testing.T) {
+	p := extract(`<html><head><meta property="og:title" content="Real"></head>
+		<body><meta property="og:title" content="Forged"></body></html>`)
+	if p.Title != "Real" {
+		t.Errorf("title = %q, want the head's value", p.Title)
+	}
+}