spsays-bff/pkg/unfurl/unfurl.go

// Package unfurl fetches link preview metadata for URLs users paste into chat.
//
// Fetching a user-supplied URL from the server is a server-side request forgery
// primitive: without care, "https://x" can be pointed at 169.254.169.254 (cloud
// metadata), at a database on 10.x, or at localhost. Everything below exists to
// make that impossible:
//
//   - only http/https,
//   - every connection is vetted at the RESOLVED IP, so a hostname that
//     resolves to a private address is refused (and re-checked on each redirect
//     hop, which is what defeats DNS rebinding),
//   - redirects capped, response size capped, timeouts on everything,
//   - only text/html is parsed.
//
// The metadata is deliberately fetched by the server rather than accepted from
// the client. A client that could supply the title and image could make any
// link claim to be anything.
package unfurl

import (
	"context"
	"errors"
	"fmt"
	"html"
	"io"
	"net"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"syscall"
	"time"
)

// ErrBlocked is returned for a URL that resolves somewhere we refuse to fetch.
var ErrBlocked = errors.New("refusing to fetch that address")

// ErrNotHTML is returned when the response isn't a page we can read metadata
// from — a PDF or an image has nothing to unfurl.
var ErrNotHTML = errors.New("not an HTML page")

const (
	maxRedirects  = 3
	maxBodyBytes  = 512 << 10 // 512 KiB is far more than any <head>
	fetchTimeout  = 6 * time.Second
	dialTimeout   = 3 * time.Second
	maxFieldRunes = 300
)

// Preview is the metadata rendered as a link card.
type Preview struct {
	URL         string `json:"url"`
	Title       string `json:"title,omitempty"`
	Description string `json:"description,omitempty"`
	SiteName    string `json:"site_name,omitempty"`
	ImageURL    string `json:"image_url,omitempty"`
}

// Fetcher retrieves previews. Safe for concurrent use.
type Fetcher struct {
	client *http.Client
	// allowLoopback exists only so the package's own tests can point at an
	// httptest server. It is unexported and never set outside this package.
	allowLoopback bool
}

// New returns a Fetcher with every guard enabled.
func New() *Fetcher {
	f := &Fetcher{}
	f.client = &http.Client{
		Timeout: fetchTimeout,
		Transport: &http.Transport{
			DialContext: (&net.Dialer{
				Timeout: dialTimeout,
				// Control runs after DNS resolution with the address actually
				// being dialled, so it sees through a hostname that points at
				// a private range — including one that changes between the
				// lookup and the connection.
				Control: func(network, address string, _ syscall.RawConn) error {
					return f.checkAddr(network, address)
				},
			}).DialContext,
			TLSHandshakeTimeout:   dialTimeout,
			ResponseHeaderTimeout: fetchTimeout,
			DisableKeepAlives:     true,
		},
		CheckRedirect: func(req *http.Request, via []*http.Request) error {
			if len(via) >= maxRedirects {
				return fmt.Errorf("too many redirects")
			}
			if !allowedScheme(req.URL) {
				return ErrBlocked
			}
			return nil
		},
	}
	return f
}

func (f *Fetcher) checkAddr(network, address string) error {
	if network != "tcp4" && network != "tcp6" && network != "tcp" {
		return ErrBlocked
	}
	host, _, err := net.SplitHostPort(address)
	if err != nil {
		return ErrBlocked
	}
	ip := net.ParseIP(host)
	if ip == nil {
		return ErrBlocked
	}
	if f.allowLoopback && ip.IsLoopback() {
		return nil
	}
	if blockedIP(ip) {
		return ErrBlocked
	}
	return nil
}

// blockedIP reports whether an address is one we refuse to talk to.
func blockedIP(ip net.IP) bool {
	// Normalise IPv4-mapped IPv6 (::ffff:127.0.0.1) so the v4 rules apply.
	if v4 := ip.To4(); v4 != nil {
		ip = v4
	}
	if ip.IsLoopback() || ip.IsPrivate() || ip.IsUnspecified() ||
		ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() ||
		ip.IsMulticast() || ip.IsInterfaceLocalMulticast() {
		return true
	}
	// Ranges Go's helpers don't cover but which still reach infrastructure.
	for _, cidr := range extraBlocked {
		if cidr.Contains(ip) {
			return true
		}
	}
	return false
}

var extraBlocked = func() []*net.IPNet {
	nets := []string{
		"100.64.0.0/10",   // CGNAT
		"192.0.0.0/24",    // IETF protocol assignments
		"192.0.2.0/24",    // TEST-NET-1
		"198.18.0.0/15",   // benchmarking
		"198.51.100.0/24", // TEST-NET-2
		"203.0.113.0/24",  // TEST-NET-3
		"240.0.0.0/4",     // reserved
		"::/128",          // unspecified
		"64:ff9b::/96",    // NAT64, can map to private v4
	}
	out := make([]*net.IPNet, 0, len(nets))
	for _, n := range nets {
		if _, cidr, err := net.ParseCIDR(n); err == nil {
			out = append(out, cidr)
		}
	}
	return out
}()

func allowedScheme(u *url.URL) bool {
	return u.Scheme == "http" || u.Scheme == "https"
}

// Fetch retrieves a preview for rawURL.
func (f *Fetcher) Fetch(ctx context.Context, rawURL string) (*Preview, error) {
	u, err := url.Parse(strings.TrimSpace(rawURL))
	if err != nil {
		return nil, fmt.Errorf("parse url: %w", err)
	}
	if !allowedScheme(u) || u.Host == "" {
		return nil, ErrBlocked
	}

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
	if err != nil {
		return nil, fmt.Errorf("build request: %w", err)
	}
	// Identify honestly; some sites serve richer metadata to a known crawler
	// and it gives operators someone to contact.
	req.Header.Set("User-Agent", "PrabhupadaSaysBot/1.0 (+https://prabhupadasays.org)")
	req.Header.Set("Accept", "text/html,application/xhtml+xml")

	resp, err := f.client.Do(req)
	if err != nil {
		// A blocked dial surfaces wrapped in a url.Error; unwrap so callers can
		// tell "refused on purpose" from "the site is down".
		if errors.Is(err, ErrBlocked) {
			return nil, ErrBlocked
		}
		return nil, fmt.Errorf("fetch: %w", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		return nil, fmt.Errorf("unfurl status %d", resp.StatusCode)
	}
	if ct := resp.Header.Get("Content-Type"); ct != "" &&
		!strings.HasPrefix(strings.ToLower(ct), "text/html") &&
		!strings.HasPrefix(strings.ToLower(ct), "application/xhtml") {
		return nil, ErrNotHTML
	}

	body, err := io.ReadAll(io.LimitReader(resp.Body, maxBodyBytes))
	if err != nil {
		return nil, fmt.Errorf("read body: %w", err)
	}

	// resp.Request.URL is the FINAL url after redirects — relative images must
	// resolve against where we actually landed.
	final := resp.Request.URL
	p := extract(string(body))
	p.URL = final.String()
	if p.ImageURL != "" {
		if abs, err := final.Parse(p.ImageURL); err == nil && allowedScheme(abs) {
			p.ImageURL = abs.String()
		} else {
			p.ImageURL = ""
		}
	}
	if p.Title == "" && p.Description == "" {
		return nil, ErrNotHTML
	}
	return p, nil
}

var (
	metaRe  = regexp.MustCompile(`(?is)<meta\s+([^>]*?)/?>`)
	attrRe  = regexp.MustCompile(`(?is)([a-z0-9:_.-]+)\s*=\s*("([^"]*)"|'([^']*)'|([^\s"'>]+))`)
	titleRe = regexp.MustCompile(`(?is)<title[^>]*>(.*?)</title>`)
	headRe  = regexp.MustCompile(`(?is)</head\s*>`)
	tagRe   = regexp.MustCompile(`(?s)<[^>]*>`)
)

// extract pulls Open Graph metadata out of a document's head.
//
// Written against the meta tags themselves rather than a full HTML parse: the
// fields wanted here are a handful of well-formed <meta> elements, and this
// keeps the BFF free of another dependency.
func extract(doc string) *Preview {
	// Metadata lives in <head>; stopping there avoids scanning a whole page.
	if loc := headRe.FindStringIndex(doc); loc != nil {
		doc = doc[:loc[0]]
	}

	p := &Preview{}
	for _, m := range metaRe.FindAllStringSubmatch(doc, -1) {
		attrs := parseAttrs(m[1])
		key := attrs["property"]
		if key == "" {
			key = attrs["name"]
		}
		content := attrs["content"]
		if content == "" {
			continue
		}
		switch strings.ToLower(key) {
		case "og:title", "twitter:title":
			setIfEmpty(&p.Title, content)
		case "og:description", "twitter:description", "description":
			setIfEmpty(&p.Description, content)
		case "og:site_name":
			setIfEmpty(&p.SiteName, content)
		case "og:image", "og:image:url", "og:image:secure_url", "twitter:image":
			setIfEmpty(&p.ImageURL, content)
		}
	}

	if p.Title == "" {
		if m := titleRe.FindStringSubmatch(doc); m != nil {
			p.Title = clean(tagRe.ReplaceAllString(m[1], ""))
		}
	}

	p.Title = truncate(p.Title, maxFieldRunes)
	p.Description = truncate(p.Description, maxFieldRunes)
	p.SiteName = truncate(p.SiteName, maxFieldRunes)
	return p
}

func parseAttrs(s string) map[string]string {
	out := map[string]string{}
	for _, m := range attrRe.FindAllStringSubmatch(s, -1) {
		value := m[3]
		if value == "" {
			value = m[4]
		}
		if value == "" {
			value = m[5]
		}
		out[strings.ToLower(m[1])] = clean(value)
	}
	return out
}

func setIfEmpty(dst *string, v string) {
	if *dst == "" {
		*dst = v
	}
}

func clean(s string) string {
	return strings.TrimSpace(html.UnescapeString(strings.Join(strings.Fields(s), " ")))
}

func truncate(s string, n int) string {
	r := []rune(s)
	if len(r) <= n {
		return s
	}
	return strings.TrimSpace(string(r[:n])) + "…"
}