adding protocols redirect (#7296)

pkg/protocols/http/cluster.go

@@ -11,7 +11,7 @@ import (
 // TmplClusterKey generates a unique key for the request
 // to be used in the clustering process.
 func (request *Request) TmplClusterKey() uint64 {
-	inp := fmt.Sprintf("%s-%d-%t-%t-%s-%d", request.Method.String(), request.MaxRedirects, request.DisableCookie, request.Redirects, strings.Join(request.Path, "-"), utils.MapHash(request.Headers))
+	inp := fmt.Sprintf("%s-%d-%t-%t-%t-%s-%d", request.Method.String(), request.MaxRedirects, request.DisableCookie, request.Redirects, request.ProtocolRedirects, strings.Join(request.Path, "-"), utils.MapHash(request.Headers))
 	return xxhash.Sum64String(inp)
 }
 

pkg/protocols/http/cluster_test.go

@@ -3,6 +3,9 @@ package http
 import (
 	"testing"
 
+	"github.com/projectdiscovery/nuclei/v3/pkg/fuzz"
+	"github.com/projectdiscovery/nuclei/v3/pkg/operators"
+	"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
 	"github.com/stretchr/testify/require"
 )
 
@@ -16,3 +19,179 @@ func TestCanCluster(t *testing.T) {
 	require.True(t, req.IsClusterable(), "could not cluster GET request")
 	require.Equal(t, req.TmplClusterKey(), newReq.TmplClusterKey(), "cluster keys should be equal")
 }
+
+func TestIsClusterable(t *testing.T) {
+	t.Run("simple GET is clusterable", func(t *testing.T) {
+		req := &Request{Path: []string{"{{BaseURL}}"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}}
+		require.True(t, req.IsClusterable())
+	})
+
+	t.Run("payloads prevent clustering", func(t *testing.T) {
+		req := &Request{
+			Path:     []string{"{{BaseURL}}"},
+			Method:   HTTPMethodTypeHolder{MethodType: HTTPGet},
+			Payloads: map[string]interface{}{"user": "admin"},
+		}
+		require.False(t, req.IsClusterable())
+	})
+
+	t.Run("fuzzing prevents clustering", func(t *testing.T) {
+		req := &Request{
+			Path:    []string{"{{BaseURL}}"},
+			Method:  HTTPMethodTypeHolder{MethodType: HTTPGet},
+			Fuzzing: []*fuzz.Rule{{}},
+		}
+		require.False(t, req.IsClusterable())
+	})
+
+	t.Run("raw requests prevent clustering", func(t *testing.T) {
+		req := &Request{
+			Raw: []string{"GET / HTTP/1.1\r\nHost: {{Hostname}}\r\n\r\n"},
+		}
+		require.False(t, req.IsClusterable())
+	})
+
+	t.Run("body prevents clustering", func(t *testing.T) {
+		req := &Request{
+			Path:   []string{"{{BaseURL}}"},
+			Method: HTTPMethodTypeHolder{MethodType: HTTPPost},
+			Body:   "key=value",
+		}
+		require.False(t, req.IsClusterable())
+	})
+
+	t.Run("unsafe prevents clustering", func(t *testing.T) {
+		req := &Request{Unsafe: true, Raw: []string{"GET / HTTP/1.1\r\n\r\n"}}
+		require.False(t, req.IsClusterable())
+	})
+
+	t.Run("named request prevents clustering", func(t *testing.T) {
+		req := &Request{
+			Path:   []string{"{{BaseURL}}"},
+			Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
+			Name:   "login-check",
+		}
+		require.False(t, req.IsClusterable())
+	})
+
+	t.Run("redirects do not prevent clustering", func(t *testing.T) {
+		req := &Request{
+			Path:      []string{"{{BaseURL}}"},
+			Method:    HTTPMethodTypeHolder{MethodType: HTTPGet},
+			Redirects: true,
+		}
+		require.True(t, req.IsClusterable())
+	})
+
+	t.Run("protocol-redirects do not prevent clustering", func(t *testing.T) {
+		req := &Request{
+			Path:              []string{"{{BaseURL}}"},
+			Method:            HTTPMethodTypeHolder{MethodType: HTTPGet},
+			Redirects:         true,
+			ProtocolRedirects: true,
+		}
+		require.True(t, req.IsClusterable())
+	})
+}
+
+func TestTmplClusterKeyIdempotent(t *testing.T) {
+	req := &Request{
+		Path:         []string{"{{BaseURL}}/admin"},
+		Method:       HTTPMethodTypeHolder{MethodType: HTTPGet},
+		MaxRedirects: 5,
+		Redirects:    true,
+		Headers:      map[string]string{"X-Custom": "test"},
+	}
+	key1 := req.TmplClusterKey()
+	key2 := req.TmplClusterKey()
+	require.Equal(t, key1, key2, "cluster key should be deterministic")
+}
+
+func TestTmplClusterKeyIdenticalRequests(t *testing.T) {
+	make := func() *Request {
+		return &Request{
+			Path:          []string{"{{BaseURL}}"},
+			Method:        HTTPMethodTypeHolder{MethodType: HTTPGet},
+			MaxRedirects:  3,
+			DisableCookie: true,
+			Redirects:     true,
+			Headers:       map[string]string{"Accept": "text/html"},
+		}
+	}
+	require.Equal(t, make().TmplClusterKey(), make().TmplClusterKey())
+}
+
+func TestTmplClusterKeyDiffersOnMethod(t *testing.T) {
+	base := &Request{Path: []string{"{{BaseURL}}"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}}
+	other := &Request{Path: []string{"{{BaseURL}}"}, Method: HTTPMethodTypeHolder{MethodType: HTTPPost}}
+	require.NotEqual(t, base.TmplClusterKey(), other.TmplClusterKey())
+}
+
+func TestTmplClusterKeyDiffersOnMaxRedirects(t *testing.T) {
+	base := &Request{Path: []string{"{{BaseURL}}"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, MaxRedirects: 5}
+	other := &Request{Path: []string{"{{BaseURL}}"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, MaxRedirects: 10}
+	require.NotEqual(t, base.TmplClusterKey(), other.TmplClusterKey())
+}
+
+func TestTmplClusterKeyDiffersOnDisableCookie(t *testing.T) {
+	base := &Request{Path: []string{"{{BaseURL}}"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, DisableCookie: false}
+	other := &Request{Path: []string{"{{BaseURL}}"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, DisableCookie: true}
+	require.NotEqual(t, base.TmplClusterKey(), other.TmplClusterKey())
+}
+
+func TestTmplClusterKeyDiffersOnRedirects(t *testing.T) {
+	base := &Request{Path: []string{"{{BaseURL}}"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, Redirects: false}
+	other := &Request{Path: []string{"{{BaseURL}}"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, Redirects: true}
+	require.NotEqual(t, base.TmplClusterKey(), other.TmplClusterKey())
+}
+
+func TestTmplClusterKeyDiffersOnProtocolRedirects(t *testing.T) {
+	base := &Request{Path: []string{"{{BaseURL}}"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, Redirects: true}
+	withProto := &Request{Path: []string{"{{BaseURL}}"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, Redirects: true, ProtocolRedirects: true}
+	require.NotEqual(t, base.TmplClusterKey(), withProto.TmplClusterKey())
+}
+
+func TestTmplClusterKeyDiffersOnPath(t *testing.T) {
+	base := &Request{Path: []string{"{{BaseURL}}/a"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}}
+	other := &Request{Path: []string{"{{BaseURL}}/b"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}}
+	require.NotEqual(t, base.TmplClusterKey(), other.TmplClusterKey())
+}
+
+func TestTmplClusterKeyDiffersOnMultiplePaths(t *testing.T) {
+	base := &Request{Path: []string{"{{BaseURL}}/a", "{{BaseURL}}/b"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}}
+	other := &Request{Path: []string{"{{BaseURL}}/a", "{{BaseURL}}/c"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}}
+	require.NotEqual(t, base.TmplClusterKey(), other.TmplClusterKey())
+}
+
+func TestTmplClusterKeyDiffersOnHeaders(t *testing.T) {
+	base := &Request{
+		Path:    []string{"{{BaseURL}}"},
+		Method:  HTTPMethodTypeHolder{MethodType: HTTPGet},
+		Headers: map[string]string{"X-Test": "a"},
+	}
+	other := &Request{
+		Path:    []string{"{{BaseURL}}"},
+		Method:  HTTPMethodTypeHolder{MethodType: HTTPGet},
+		Headers: map[string]string{"X-Test": "b"},
+	}
+	require.NotEqual(t, base.TmplClusterKey(), other.TmplClusterKey())
+}
+
+func TestTmplClusterKeyNilVsEmptyHeaders(t *testing.T) {
+	noHeaders := &Request{Path: []string{"{{BaseURL}}"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}}
+	emptyHeaders := &Request{Path: []string{"{{BaseURL}}"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, Headers: map[string]string{}}
+	require.Equal(t, noHeaders.TmplClusterKey(), emptyHeaders.TmplClusterKey(), "nil and empty headers should produce the same key")
+}
+
+func TestIsClusterableWithReqCondition(t *testing.T) {
+	req := &Request{
+		Path:   []string{"{{BaseURL}}"},
+		Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
+		Operators: operators.Operators{
+			Matchers: []*matchers.Matcher{
+				{DSL: []string{"status_code_1 == 200"}},
+			},
+		},
+	}
+	require.False(t, req.IsClusterable(), "request with req-condition matchers should not be clusterable")
+}

pkg/protocols/http/http.go

@@ -188,6 +188,11 @@ type Request struct {
 	//   This can be used in conjunction with `max-redirects` to control the HTTP request redirects.
 	HostRedirects bool `yaml:"host-redirects,omitempty" json:"host-redirects,omitempty" jsonschema:"title=follow same host http redirects,description=Specifies whether redirects to the same host should be followed by the HTTP Client"`
 	// description: |
+	//   ProtocolRedirects specifies whether only redirects within the same protocol should be followed.
+	//
+	//   When set to true with redirects enabled, cross-protocol redirects (e.g. HTTP to HTTPS) will be blocked.
+	ProtocolRedirects bool `yaml:"protocol-redirects,omitempty" json:"protocol-redirects,omitempty" jsonschema:"title=follow same protocol http redirects,description=Specifies whether only same-protocol redirects should be followed by the HTTP Client"`
+	// description: |
 	//   Pipeline defines if the attack should be performed with HTTP 1.1 Pipelining
 	//
 	//   All requests must be idempotent (GET/POST). This can be used for race conditions/billions requests.
@@ -337,6 +342,9 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
 	if request.HostRedirects || options.Options.FollowHostRedirects {
 		connectionConfiguration.RedirectFlow = httpclientpool.FollowSameHostRedirect
 	}
+	if request.ProtocolRedirects {
+		connectionConfiguration.RedirectFlow = httpclientpool.FollowSameSchemeRedirect
+	}
 
 	// If we have request level timeout, ignore http client timeouts
 	for _, req := range request.Raw {

pkg/protocols/http/httpclientpool/clientpool.go

@@ -362,6 +362,7 @@ const (
 	DontFollowRedirect RedirectFlow = iota
 	FollowSameHostRedirect
 	FollowAllRedirect
+	FollowSameSchemeRedirect
 )
 
 const defaultMaxRedirects = 10
@@ -387,6 +388,12 @@ func makeCheckRedirectFunc(redirectType RedirectFlow, maxRedirects int) checkRed
 			return checkMaxRedirects(req, via, maxRedirects)
 		case FollowAllRedirect:
 			return checkMaxRedirects(req, via, maxRedirects)
+		case FollowSameSchemeRedirect:
+			previousScheme := via[len(via)-1].URL.Scheme
+			if req.URL.Scheme != previousScheme {
+				return http.ErrUseLastResponse
+			}
+			return checkMaxRedirects(req, via, maxRedirects)
 		}
 		return nil
 	}

pkg/protocols/http/httpclientpool/clientpool_test.go

@@ -170,6 +170,85 @@ func TestFollowAllRedirect(t *testing.T) {
 	}
 }
 
+func TestFollowSameSchemeRedirect(t *testing.T) {
+	tests := []struct {
+		name        string
+		oldURL      string
+		newURL      string
+		shouldAllow bool
+	}{
+		// same scheme should be allowed
+		{"http to http", "http://example.com/a", "http://example.com/b", true},
+		{"https to https", "https://example.com/a", "https://example.com/b", true},
+		{"http to http different host", "http://example.com/a", "http://other.com/b", true},
+		{"https to https different host", "https://example.com/a", "https://other.com/b", true},
+
+		// cross-scheme should be blocked
+		{"http to https", "http://example.com/a", "https://example.com/b", false},
+		{"https to http", "https://example.com/a", "http://example.com/b", false},
+		{"http to https different host", "http://example.com/a", "https://other.com/b", false},
+		{"https to http different host", "https://example.com/a", "http://other.com/b", false},
+
+		// same scheme with ports
+		{"http with port to http", "http://example.com:8080/a", "http://example.com:9090/b", true},
+		{"https with port to https", "https://example.com:8443/a", "https://example.com:443/b", true},
+		{"http with port to https", "http://example.com:80/a", "https://example.com:443/b", false},
+
+		// ipv4
+		{"ipv4 http to http", "http://127.0.0.1/a", "http://127.0.0.1/b", true},
+		{"ipv4 http to https", "http://127.0.0.1/a", "https://127.0.0.1/b", false},
+
+		// ipv6
+		{"ipv6 https to https", "https://[::1]/a", "https://[::1]/b", true},
+		{"ipv6 https to http", "https://[::1]/a", "http://[::1]/b", false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			checkFn := makeCheckRedirectFunc(FollowSameSchemeRedirect, 10)
+			oldReq, _ := http.NewRequest("GET", tt.oldURL, nil)
+			newReq, _ := http.NewRequest("GET", tt.newURL, nil)
+			err := checkFn(newReq, []*http.Request{oldReq})
+			allowed := err == nil
+			if allowed != tt.shouldAllow {
+				t.Errorf("redirect from %q to %q: allowed=%v, want %v", tt.oldURL, tt.newURL, allowed, tt.shouldAllow)
+			}
+		})
+	}
+}
+
+func TestFollowSameSchemeRedirectChain(t *testing.T) {
+	checkFn := makeCheckRedirectFunc(FollowSameSchemeRedirect, 10)
+
+	// multi-hop chain within same scheme should work
+	req1, _ := http.NewRequest("GET", "http://example.com/a", nil)
+	req2, _ := http.NewRequest("GET", "http://example.com/b", nil)
+	req3, _ := http.NewRequest("GET", "http://example.com/c", nil)
+	err := checkFn(req3, []*http.Request{req1, req2})
+	if err != nil {
+		t.Errorf("same-scheme redirect chain should be allowed, got: %v", err)
+	}
+
+	// chain where last hop changes scheme should be blocked
+	req4, _ := http.NewRequest("GET", "https://example.com/d", nil)
+	err = checkFn(req4, []*http.Request{req1, req2})
+	if err == nil {
+		t.Errorf("cross-scheme redirect at end of chain should be blocked")
+	}
+}
+
+func TestFollowSameSchemeRedirectMaxRedirects(t *testing.T) {
+	checkFn := makeCheckRedirectFunc(FollowSameSchemeRedirect, 2)
+
+	req, _ := http.NewRequest("GET", "http://example.com/c", nil)
+	via := make([]*http.Request, 3)
+	for i := range via {
+		via[i], _ = http.NewRequest("GET", "http://example.com/"+string(rune('a'+i)), nil)
+	}
+	if err := checkFn(req, via); err == nil {
+		t.Errorf("should block after exceeding maxRedirects=2 with 3 via requests")
+	}
+}
+
 func TestMaxRedirects(t *testing.T) {
 	// Exceeding explicit max
 	checkFn := makeCheckRedirectFunc(FollowAllRedirect, 2)

pkg/protocols/http/validate.go

@@ -11,5 +11,9 @@ func (request *Request) validate() error {
 		return errors.New("'redirects' and 'host-redirects' can't be used together")
 	}
 
+	if request.ProtocolRedirects && !request.Redirects && !request.HostRedirects {
+		return errors.New("'protocol-redirects' requires 'redirects' or 'host-redirects' to be enabled")
+	}
+
 	return nil
 }

pkg/protocols/http/validate_test.go

@@ -0,0 +1,53 @@
+package http
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestValidateRedirectsCombinations(t *testing.T) {
+	t.Run("redirects and host-redirects conflict", func(t *testing.T) {
+		req := &Request{Redirects: true, HostRedirects: true}
+		err := req.validate()
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "'redirects' and 'host-redirects' can't be used together")
+	})
+
+	t.Run("redirects alone is valid", func(t *testing.T) {
+		req := &Request{Redirects: true}
+		err := req.validate()
+		require.NoError(t, err)
+	})
+
+	t.Run("host-redirects alone is valid", func(t *testing.T) {
+		req := &Request{HostRedirects: true}
+		err := req.validate()
+		require.NoError(t, err)
+	})
+
+	t.Run("protocol-redirects without redirects or host-redirects is invalid", func(t *testing.T) {
+		req := &Request{ProtocolRedirects: true}
+		err := req.validate()
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "'protocol-redirects' requires 'redirects' or 'host-redirects'")
+	})
+
+	t.Run("protocol-redirects with redirects is valid", func(t *testing.T) {
+		req := &Request{Redirects: true, ProtocolRedirects: true}
+		err := req.validate()
+		require.NoError(t, err)
+	})
+
+	t.Run("protocol-redirects with host-redirects is valid", func(t *testing.T) {
+		req := &Request{HostRedirects: true, ProtocolRedirects: true}
+		err := req.validate()
+		require.NoError(t, err)
+	})
+
+	t.Run("no redirect options is valid", func(t *testing.T) {
+		req := &Request{}
+		err := req.validate()
+		require.NoError(t, err)
+	})
+}