package unfurl import ( "context" "errors" "fmt" "net" "net/http" "net/http/httptest" "strings" "testing" ) // The blocklist is the security boundary of this package: everything else is // plumbing around it. A URL that resolves into any of these ranges must never // be fetched, or the BFF becomes a proxy into the private network. func TestBlockedIP(t *testing.T) { blocked := []string{ "127.0.0.1", // loopback "::1", // loopback v6 "::ffff:127.0.0.1", // loopback via IPv4-mapped v6 "10.0.0.5", // RFC1918 "172.16.4.2", // RFC1918 "192.168.1.1", // RFC1918 "::ffff:10.0.0.5", // RFC1918 via IPv4-mapped v6 "169.254.169.254", // cloud instance metadata "fe80::1", // link-local v6 "fd00::1", // unique local v6 "0.0.0.0", // unspecified "::", // unspecified v6 "224.0.0.1", // multicast "100.64.0.1", // CGNAT "198.18.0.1", // benchmarking "240.0.0.1", // reserved } for _, s := range blocked { if ip := net.ParseIP(s); ip == nil || !blockedIP(ip) { t.Errorf("blockedIP(%s) = false, want true", s) } } allowed := []string{"1.1.1.1", "8.8.8.8", "93.184.216.34", "2606:2800:220:1:248:1893:25c8:1946"} for _, s := range allowed { if ip := net.ParseIP(s); ip == nil || blockedIP(ip) { t.Errorf("blockedIP(%s) = true, want false", s) } } } // A hostname that resolves to a private address must be refused at dial time — // this is what stops "http://internal.example.com" and DNS rebinding. func TestFetch_RefusesPrivateAddresses(t *testing.T) { f := New() // loopback NOT allowed for _, raw := range []string{ "http://127.0.0.1:80/", "http://localhost/", "http://169.254.169.254/latest/meta-data/", "http://10.0.0.1/", "http://[::1]/", } { _, err := f.Fetch(context.Background(), raw) if err == nil { t.Errorf("Fetch(%s) succeeded — it must be refused", raw) } } } func TestFetch_RejectsNonHTTPSchemes(t *testing.T) { f := New() for _, raw := range []string{ "file:///etc/passwd", "gopher://example.org/", "ftp://example.org/", "redis://10.0.0.1:6379", "//example.org", } { if _, err := f.Fetch(context.Background(), raw); !errors.Is(err, ErrBlocked) { t.Errorf("Fetch(%s) err = %v, want ErrBlocked", raw, err) } } } // testFetcher permits loopback so the fetch path can be exercised against an // httptest server. The flag is unexported and set only here. func testFetcher() *Fetcher { f := New() f.allowLoopback = true return f } func TestFetch_ReadsOpenGraph(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/html; charset=utf-8") fmt.Fprint(w, ` Fallback title ignored`) })) defer srv.Close() p, err := testFetcher().Fetch(context.Background(), srv.URL) if err != nil { t.Fatalf("Fetch: %v", err) } if p.Description != "Chapter 2, verse 13" { t.Errorf("description = %q", p.Description) } if p.SiteName != "Prabhupāda Says" { t.Errorf("site_name = %q", p.SiteName) } // Relative og:image must be resolved against the page it came from. if !strings.HasPrefix(p.ImageURL, srv.URL) || !strings.HasSuffix(p.ImageURL, "/img/cover.jpg") { t.Errorf("image = %q, want an absolute URL under %s", p.ImageURL, srv.URL) } } func TestFetch_FallsBackToTitleTag(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/html") fmt.Fprint(w, ` Plain & simple `) })) defer srv.Close() p, err := testFetcher().Fetch(context.Background(), srv.URL) if err != nil { t.Fatalf("Fetch: %v", err) } if p.Title != "Plain & simple" { t.Errorf("title = %q, want entities decoded and whitespace collapsed", p.Title) } if p.Description != "From the name attribute" { t.Errorf("description = %q", p.Description) } } // A redirect that lands on a private address must be refused at the hop, not // followed because the first host looked fine. func TestFetch_RefusesRedirectToPrivateAddress(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "http://169.254.169.254/latest/meta-data/", http.StatusFound) })) defer srv.Close() if _, err := testFetcher().Fetch(context.Background(), srv.URL); err == nil { t.Fatal("followed a redirect into link-local space") } } func TestFetch_RejectsNonHTML(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/pdf") fmt.Fprint(w, "%PDF-1.7") })) defer srv.Close() if _, err := testFetcher().Fetch(context.Background(), srv.URL); !errors.Is(err, ErrNotHTML) { t.Errorf("err = %v, want ErrNotHTML", err) } } // A page with no metadata is not a preview — better no card than an empty one. func TestFetch_EmptyMetadataIsNotAPreview(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/html") fmt.Fprint(w, `nothing here`) })) defer srv.Close() if _, err := testFetcher().Fetch(context.Background(), srv.URL); !errors.Is(err, ErrNotHTML) { t.Errorf("err = %v, want ErrNotHTML", err) } } // A body far larger than the cap must not be read whole. func TestFetch_CapsBodySize(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/html") fmt.Fprint(w, ``) chunk := strings.Repeat("x", 64<<10) for i := 0; i < 64; i++ { // 4 MiB, well past maxBodyBytes fmt.Fprint(w, chunk) } })) defer srv.Close() p, err := testFetcher().Fetch(context.Background(), srv.URL) if err != nil { t.Fatalf("Fetch: %v", err) } if p.Title != "Huge" { t.Errorf("title = %q", p.Title) } } func TestExtract_LongFieldsAreTruncated(t *testing.T) { long := strings.Repeat("ā", 500) p := extract(``) if len([]rune(p.Title)) > maxFieldRunes+1 { t.Errorf("title kept %d runes, want it truncated", len([]rune(p.Title))) } if !strings.HasSuffix(p.Title, "…") { t.Error("truncated title should be marked with an ellipsis") } } func TestExtract_HandlesAttributeOrderAndQuotes(t *testing.T) { p := extract(` `) if p.Title != "Single quoted" { t.Errorf("title = %q", p.Title) } if p.SiteName != "Unquoted" { t.Errorf("site_name = %q", p.SiteName) } if p.Description != "Ordered normally" { t.Errorf("description = %q", p.Description) } } // Metadata is read from ; a forged in the body must not win. func TestExtract_IgnoresBodyContent(t *testing.T) { p := extract(` `) if p.Title != "Real" { t.Errorf("title = %q, want the head's value", p.Title) } }