spsays-bff/pkg/unfurl/unfurl_test.go

package unfurl

import (
	"context"
	"errors"
	"fmt"
	"net"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
)

// The blocklist is the security boundary of this package: everything else is
// plumbing around it. A URL that resolves into any of these ranges must never
// be fetched, or the BFF becomes a proxy into the private network.
func TestBlockedIP(t *testing.T) {
	blocked := []string{
		"127.0.0.1",        // loopback
		"::1",              // loopback v6
		"::ffff:127.0.0.1", // loopback via IPv4-mapped v6
		"10.0.0.5",         // RFC1918
		"172.16.4.2",       // RFC1918
		"192.168.1.1",      // RFC1918
		"::ffff:10.0.0.5",  // RFC1918 via IPv4-mapped v6
		"169.254.169.254",  // cloud instance metadata
		"fe80::1",          // link-local v6
		"fd00::1",          // unique local v6
		"0.0.0.0",          // unspecified
		"::",               // unspecified v6
		"224.0.0.1",        // multicast
		"100.64.0.1",       // CGNAT
		"198.18.0.1",       // benchmarking
		"240.0.0.1",        // reserved
	}
	for _, s := range blocked {
		if ip := net.ParseIP(s); ip == nil || !blockedIP(ip) {
			t.Errorf("blockedIP(%s) = false, want true", s)
		}
	}

	allowed := []string{"1.1.1.1", "8.8.8.8", "93.184.216.34", "2606:2800:220:1:248:1893:25c8:1946"}
	for _, s := range allowed {
		if ip := net.ParseIP(s); ip == nil || blockedIP(ip) {
			t.Errorf("blockedIP(%s) = true, want false", s)
		}
	}
}

// A hostname that resolves to a private address must be refused at dial time —
// this is what stops "http://internal.example.com" and DNS rebinding.
func TestFetch_RefusesPrivateAddresses(t *testing.T) {
	f := New() // loopback NOT allowed
	for _, raw := range []string{
		"http://127.0.0.1:80/",
		"http://localhost/",
		"http://169.254.169.254/latest/meta-data/",
		"http://10.0.0.1/",
		"http://[::1]/",
	} {
		_, err := f.Fetch(context.Background(), raw)
		if err == nil {
			t.Errorf("Fetch(%s) succeeded — it must be refused", raw)
		}
	}
}

func TestFetch_RejectsNonHTTPSchemes(t *testing.T) {
	f := New()
	for _, raw := range []string{
		"file:///etc/passwd",
		"gopher://example.org/",
		"ftp://example.org/",
		"redis://10.0.0.1:6379",
		"//example.org",
	} {
		if _, err := f.Fetch(context.Background(), raw); !errors.Is(err, ErrBlocked) {
			t.Errorf("Fetch(%s) err = %v, want ErrBlocked", raw, err)
		}
	}
}

// testFetcher permits loopback so the fetch path can be exercised against an
// httptest server. The flag is unexported and set only here.
func testFetcher() *Fetcher {
	f := New()
	f.allowLoopback = true
	return f
}

func TestFetch_ReadsOpenGraph(t *testing.T) {
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		fmt.Fprint(w, `<html><head>
			<title>Fallback title</title>
			<meta property="og:title" content="Bhagavad-g&#012ABC;t&#257; As It Is">
			<meta property="og:description" content="Chapter 2, verse 13">
			<meta property="og:site_name" content="Prabhupāda Says">
			<meta property="og:image" content="/img/cover.jpg">
		</head><body>ignored</body></html>`)
	}))
	defer srv.Close()

	p, err := testFetcher().Fetch(context.Background(), srv.URL)
	if err != nil {
		t.Fatalf("Fetch: %v", err)
	}
	if p.Description != "Chapter 2, verse 13" {
		t.Errorf("description = %q", p.Description)
	}
	if p.SiteName != "Prabhupāda Says" {
		t.Errorf("site_name = %q", p.SiteName)
	}
	// Relative og:image must be resolved against the page it came from.
	if !strings.HasPrefix(p.ImageURL, srv.URL) || !strings.HasSuffix(p.ImageURL, "/img/cover.jpg") {
		t.Errorf("image = %q, want an absolute URL under %s", p.ImageURL, srv.URL)
	}
}

func TestFetch_FallsBackToTitleTag(t *testing.T) {
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/html")
		fmt.Fprint(w, `<html><head><title>  Plain &amp; simple  </title>
			<meta name="description" content="From the name attribute"></head></html>`)
	}))
	defer srv.Close()

	p, err := testFetcher().Fetch(context.Background(), srv.URL)
	if err != nil {
		t.Fatalf("Fetch: %v", err)
	}
	if p.Title != "Plain & simple" {
		t.Errorf("title = %q, want entities decoded and whitespace collapsed", p.Title)
	}
	if p.Description != "From the name attribute" {
		t.Errorf("description = %q", p.Description)
	}
}

// A redirect that lands on a private address must be refused at the hop, not
// followed because the first host looked fine.
func TestFetch_RefusesRedirectToPrivateAddress(t *testing.T) {
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, "http://169.254.169.254/latest/meta-data/", http.StatusFound)
	}))
	defer srv.Close()

	if _, err := testFetcher().Fetch(context.Background(), srv.URL); err == nil {
		t.Fatal("followed a redirect into link-local space")
	}
}

func TestFetch_RejectsNonHTML(t *testing.T) {
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/pdf")
		fmt.Fprint(w, "%PDF-1.7")
	}))
	defer srv.Close()

	if _, err := testFetcher().Fetch(context.Background(), srv.URL); !errors.Is(err, ErrNotHTML) {
		t.Errorf("err = %v, want ErrNotHTML", err)
	}
}

// A page with no metadata is not a preview — better no card than an empty one.
func TestFetch_EmptyMetadataIsNotAPreview(t *testing.T) {
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/html")
		fmt.Fprint(w, `<html><head></head><body>nothing here</body></html>`)
	}))
	defer srv.Close()

	if _, err := testFetcher().Fetch(context.Background(), srv.URL); !errors.Is(err, ErrNotHTML) {
		t.Errorf("err = %v, want ErrNotHTML", err)
	}
}

// A body far larger than the cap must not be read whole.
func TestFetch_CapsBodySize(t *testing.T) {
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/html")
		fmt.Fprint(w, `<html><head><meta property="og:title" content="Huge"></head><body>`)
		chunk := strings.Repeat("x", 64<<10)
		for i := 0; i < 64; i++ { // 4 MiB, well past maxBodyBytes
			fmt.Fprint(w, chunk)
		}
	}))
	defer srv.Close()

	p, err := testFetcher().Fetch(context.Background(), srv.URL)
	if err != nil {
		t.Fatalf("Fetch: %v", err)
	}
	if p.Title != "Huge" {
		t.Errorf("title = %q", p.Title)
	}
}

func TestExtract_LongFieldsAreTruncated(t *testing.T) {
	long := strings.Repeat("ā", 500)
	p := extract(`<meta property="og:title" content="` + long + `">`)
	if len([]rune(p.Title)) > maxFieldRunes+1 {
		t.Errorf("title kept %d runes, want it truncated", len([]rune(p.Title)))
	}
	if !strings.HasSuffix(p.Title, "…") {
		t.Error("truncated title should be marked with an ellipsis")
	}
}

func TestExtract_HandlesAttributeOrderAndQuotes(t *testing.T) {
	p := extract(`
		<meta content='Single quoted' property='og:title'>
		<meta content=Unquoted name=og:site_name>
		<meta property="og:description" content="Ordered normally">`)
	if p.Title != "Single quoted" {
		t.Errorf("title = %q", p.Title)
	}
	if p.SiteName != "Unquoted" {
		t.Errorf("site_name = %q", p.SiteName)
	}
	if p.Description != "Ordered normally" {
		t.Errorf("description = %q", p.Description)
	}
}

// Metadata is read from <head>; a <meta> forged in the body must not win.
func TestExtract_IgnoresBodyContent(t *testing.T) {
	p := extract(`<html><head><meta property="og:title" content="Real"></head>
		<body><meta property="og:title" content="Forged"></body></html>`)
	if p.Title != "Real" {
		t.Errorf("title = %q, want the head's value", p.Title)
	}
}