send identical requests once per scan and share the response across templates

pkg/protocols/common/protocolstate/dialers.go

@@ -16,6 +16,7 @@ type Dialers struct {
 	HTTPClientPool             *HTTPPool
 	PerHostRateLimitPool       any // *hostratelimit.PerHostRateLimitPool
 	HTTPToHTTPSPortTracker     any // *httpclientpool.HTTPToHTTPSPortTracker
+	SharedResponses            any // *httpclientpool.sharedResponses
 	HTTPDesyncHosts            *ExpiringSet
 	NetworkPolicy              *networkpolicy.NetworkPolicy
 	LocalFileAccessAllowed     bool

pkg/protocols/http/http.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"math"
 	"net/http"
+	"slices"
 	"strings"
 	"time"
 
@@ -148,6 +149,9 @@ type Request struct {
 	replayClient      *http.Client // set when -replay-proxy is given
 	dialer            *fastdialer.Dialer
 	bodySize          *bodySizeEstimate
+	// sharesResponse lets other templates reuse a response this request gets,
+	// and this request reuse theirs (see httpclientpool.WithSharedResponse)
+	sharesResponse bool
 
 	// description: |
 	//   SelfContained specifies if the request is self-contained.
@@ -587,9 +591,37 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
 	if shouldDisableKeepAliveForHTTPProxy(request, options) {
 		request.connConfiguration.Connection.DisableKeepAlive = true
 	}
+	request.sharesResponse = canShareResponse(request, options.Flow)
 	return nil
 }
 
+// canShareResponse reports whether a response another template got for the
+// same request is as good as sending it again. A shared response arrives at
+// once, so nothing that measures time may take one: race requests, timing
+// analyzers, and a request whose matchers, extractors or flow read the
+// duration.
+func canShareResponse(request *Request, flow string) bool {
+	if request.Race || request.Analyzer != nil || strings.Contains(flow, "duration") {
+		return false
+	}
+	readsDuration := func(part string, dsl []string) bool {
+		return strings.Contains(part, "duration") || slices.ContainsFunc(dsl, func(expression string) bool {
+			return strings.Contains(expression, "duration")
+		})
+	}
+	for _, matcher := range request.Matchers {
+		if readsDuration(matcher.Part, matcher.DSL) {
+			return false
+		}
+	}
+	for _, extractor := range request.Extractors {
+		if readsDuration(extractor.Part, extractor.DSL) {
+			return false
+		}
+	}
+	return true
+}
+
 // RebuildGenerator rebuilds the generator for the request
 func (request *Request) RebuildGenerator() error {
 	generator, err := generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Catalog, request.options.Options.AttackType, request.options.Options)

pkg/protocols/http/httpclientpool/clientpool.go

@@ -514,7 +514,11 @@ func wrappedGet(options *types.Options, configuration *Configuration, host strin
 			}
 		}
 
-		return &connTrackingTransport{base: transport}, nil
+		tracking := &connTrackingTransport{base: transport}
+		if shared := getSharedResponses(options); shared != nil {
+			return &sharingTransport{base: tracking, shared: shared}, nil
+		}
+		return tracking, nil
 	}
 	redirectFlow := configuration.RedirectFlow
 	maxRedirects := configuration.MaxRedirects

pkg/protocols/http/httpclientpool/clientpool_idleconn_test.go

@@ -175,7 +175,9 @@ func TestIdleConnPoolSizeIsWiredToTransport(t *testing.T) {
 	client, err := Get(opts, &Configuration{}, "example.com")
 	require.NoError(t, err)
 
-	tracking, ok := client.HTTPClient.Transport.(*connTrackingTransport)
+	sharing, ok := client.HTTPClient.Transport.(*sharingTransport)
+	require.True(t, ok, "expected the shared-response transport wrapper")
+	tracking, ok := sharing.base.(*connTrackingTransport)
 	require.True(t, ok, "expected the connection-tracking transport wrapper")
 	transport, ok := tracking.base.(*http.Transport)
 	require.True(t, ok, "expected an *http.Transport underneath")

pkg/protocols/http/httpclientpool/shared_responses.go

@@ -0,0 +1,226 @@
+package httpclientpool
+
+import (
+	"bytes"
+	"container/list"
+	"context"
+	"io"
+	"maps"
+	"net/http"
+	"slices"
+	"strings"
+	"sync"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
+	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	"github.com/projectdiscovery/useragent"
+)
+
+// Templates send many requests another template already sent to the same
+// target, byte for byte: clustering merges only single-request templates. A
+// request marked with WithSharedResponse is sent once per scan and every
+// other template sending the same bytes gets a copy of the response. The key
+// is the request as it goes on the wire, cookies included, and the client
+// still processes each copy, so redirects and cookies behave as if it was
+// sent again.
+const (
+	// sharedResponseMaxBody keeps large downloads out of the store; they are
+	// sent each time
+	sharedResponseMaxBody   = 1 << 20
+	sharedResponsesMaxBytes = 64 << 20
+)
+
+type sharedResponseKey struct{}
+
+// WithSharedResponse marks the request of templateID as one whose response
+// other templates may reuse. A template sending the same request again gets
+// it sent again: a repeat within one template is intended.
+func WithSharedResponse(ctx context.Context, templateID string) context.Context {
+	return context.WithValue(ctx, sharedResponseKey{}, templateID)
+}
+
+type sharedResponses struct {
+	mu      sync.Mutex
+	entries map[string]*sharedEntry
+	// order holds stored entries oldest first, for eviction
+	order *list.List
+	bytes int
+}
+
+type sharedEntry struct {
+	key        string
+	templateID string
+	ready      chan struct{}
+	response   *storedResponse
+}
+
+type storedResponse struct {
+	response *http.Response
+	body     []byte
+}
+
+func getSharedResponses(options *types.Options) *sharedResponses {
+	dialers := protocolstate.GetDialersWithId(options.ExecutionId)
+	if dialers == nil {
+		return nil
+	}
+	dialers.Lock()
+	defer dialers.Unlock()
+	if dialers.SharedResponses == nil {
+		dialers.SharedResponses = &sharedResponses{entries: make(map[string]*sharedEntry), order: list.New()}
+	}
+	shared, _ := dialers.SharedResponses.(*sharedResponses)
+	return shared
+}
+
+type sharingTransport struct {
+	base   http.RoundTripper
+	shared *sharedResponses
+}
+
+func (t *sharingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	templateID, marked := req.Context().Value(sharedResponseKey{}).(string)
+	if !marked || (req.Method != http.MethodGet && req.Method != http.MethodHead) || hasBody(req) {
+		return t.base.RoundTrip(req)
+	}
+	key := sharedRequestKey(req)
+
+	t.shared.mu.Lock()
+	entry, found := t.shared.entries[key]
+	if !found {
+		entry = &sharedEntry{key: key, templateID: templateID, ready: make(chan struct{})}
+		t.shared.entries[key] = entry
+		t.shared.mu.Unlock()
+		return t.send(req, entry)
+	}
+	t.shared.mu.Unlock()
+	if entry.templateID == templateID {
+		return t.base.RoundTrip(req)
+	}
+	select {
+	case <-entry.ready:
+	case <-req.Context().Done():
+		return nil, req.Context().Err()
+	}
+	if entry.response == nil {
+		// the first send failed or was too large to keep
+		return t.base.RoundTrip(req)
+	}
+	return entry.response.copyFor(req), nil
+}
+
+func (t *sharingTransport) CloseIdleConnections() {
+	type closeIdler interface{ CloseIdleConnections() }
+	if ci, ok := t.base.(closeIdler); ok {
+		ci.CloseIdleConnections()
+	}
+}
+
+// send sends the first request of key and stores its response for the rest.
+func (t *sharingTransport) send(req *http.Request, entry *sharedEntry) (*http.Response, error) {
+	defer close(entry.ready)
+	resp, err := t.base.RoundTrip(req)
+	if err != nil {
+		t.forget(entry)
+		return resp, err
+	}
+	body, err := io.ReadAll(io.LimitReader(resp.Body, sharedResponseMaxBody+1))
+	if err != nil || len(body) > sharedResponseMaxBody {
+		t.forget(entry)
+		// hand back what was read followed by the rest
+		resp.Body = struct {
+			io.Reader
+			io.Closer
+		}{io.MultiReader(bytes.NewReader(body), resp.Body), resp.Body}
+		return resp, nil
+	}
+	_ = resp.Body.Close()
+	entry.response = &storedResponse{response: resp, body: body}
+	t.store(entry)
+	return entry.response.copyFor(req), nil
+}
+
+// hasBody reports whether req sends a body. retryablehttp gives every
+// request one, empty for a GET, so its content is what counts.
+func hasBody(req *http.Request) bool {
+	if req.Body == nil || req.Body == http.NoBody {
+		return false
+	}
+	if req.ContentLength > 0 || req.GetBody == nil {
+		return true
+	}
+	body, err := req.GetBody()
+	if err != nil {
+		return true
+	}
+	defer func() { _ = body.Close() }()
+	var first [1]byte
+	n, _ := body.Read(first[:])
+	return n > 0
+}
+
+func (t *sharingTransport) forget(entry *sharedEntry) {
+	t.shared.mu.Lock()
+	defer t.shared.mu.Unlock()
+	if t.shared.entries[entry.key] == entry {
+		delete(t.shared.entries, entry.key)
+	}
+}
+
+func (t *sharingTransport) store(entry *sharedEntry) {
+	t.shared.mu.Lock()
+	defer t.shared.mu.Unlock()
+	t.shared.order.PushBack(entry)
+	t.shared.bytes += len(entry.response.body)
+	for t.shared.bytes > sharedResponsesMaxBytes && t.shared.order.Len() > 1 {
+		oldest := t.shared.order.Remove(t.shared.order.Front()).(*sharedEntry)
+		t.shared.bytes -= len(oldest.response.body)
+		delete(t.shared.entries, oldest.key)
+	}
+}
+
+func (s *storedResponse) copyFor(req *http.Request) *http.Response {
+	resp := *s.response
+	resp.Header = s.response.Header.Clone()
+	resp.Trailer = s.response.Trailer.Clone()
+	resp.Body = io.NopCloser(bytes.NewReader(s.body))
+	resp.ContentLength = int64(len(s.body))
+	resp.Request = req
+	return &resp
+}
+
+// randomUserAgents are the values nuclei picks a User-Agent from for every
+// request a template does not set one for. Two templates sending the same
+// request almost never pick the same one, and either could have picked the
+// other's, so a picked User-Agent is left out of the key.
+var randomUserAgents = sync.OnceValue(func() map[string]struct{} {
+	values := make(map[string]struct{}, len(useragent.UserAgents))
+	for _, userAgent := range useragent.UserAgents {
+		values[userAgent.Raw] = struct{}{}
+	}
+	return values
+})
+
+// sharedRequestKey is the request as the server receives it, apart from the
+// order of headers and a picked User-Agent.
+func sharedRequestKey(req *http.Request) string {
+	var key strings.Builder
+	key.WriteString(req.Method)
+	key.WriteByte(' ')
+	key.WriteString(req.URL.String())
+	key.WriteByte('\n')
+	key.WriteString(req.Host)
+	for _, name := range slices.Sorted(maps.Keys(req.Header)) {
+		values := req.Header[name]
+		if _, picked := randomUserAgents()[req.Header.Get("User-Agent")]; name == "User-Agent" && len(values) == 1 && picked {
+			continue
+		}
+		for _, value := range values {
+			key.WriteByte('\n')
+			key.WriteString(name)
+			key.WriteString(": ")
+			key.WriteString(value)
+		}
+	}
+	return key.String()
+}

pkg/protocols/http/httpclientpool/shared_responses_test.go

@@ -0,0 +1,157 @@
+package httpclientpool
+
+import (
+	"container/list"
+	"context"
+	"errors"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/projectdiscovery/retryablehttp-go"
+	"github.com/projectdiscovery/useragent"
+	"github.com/stretchr/testify/require"
+)
+
+func newSharingTransport(base http.RoundTripper) *sharingTransport {
+	return &sharingTransport{base: base, shared: &sharedResponses{entries: make(map[string]*sharedEntry), order: list.New()}}
+}
+
+func sharedGet(t *testing.T, transport http.RoundTripper, method, url, templateID string, header http.Header) string {
+	t.Helper()
+	ctx := context.Background()
+	if templateID != "" {
+		ctx = WithSharedResponse(ctx, templateID)
+	}
+	req, err := http.NewRequestWithContext(ctx, method, url, nil)
+	require.NoError(t, err)
+	for name, values := range header {
+		req.Header[name] = values
+	}
+	resp, err := transport.RoundTrip(req)
+	require.NoError(t, err)
+	defer func() { _ = resp.Body.Close() }()
+	body, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	return string(body)
+}
+
+func TestSharedResponsesSendEachRequestOncePerScan(t *testing.T) {
+	var hits atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		n := hits.Add(1)
+		_, _ = io.WriteString(w, r.URL.Path+strings.Repeat("!", int(n)))
+	}))
+	defer server.Close()
+	transport := newSharingTransport(http.DefaultTransport)
+
+	first := sharedGet(t, transport, http.MethodGet, server.URL+"/a", "one", nil)
+	require.Equal(t, first, sharedGet(t, transport, http.MethodGet, server.URL+"/a", "two", nil), "another template gets the stored response")
+	require.EqualValues(t, 1, hits.Load())
+
+	// nuclei builds a GET with an empty body
+	empty, err := retryablehttp.NewRequestWithContext(WithSharedResponse(context.Background(), "four"), http.MethodGet, server.URL+"/a", strings.NewReader(""))
+	require.NoError(t, err)
+	resp, err := transport.RoundTrip(empty.Request)
+	require.NoError(t, err)
+	_ = resp.Body.Close()
+	require.EqualValues(t, 1, hits.Load(), "an empty body is no body")
+
+	sharedGet(t, transport, http.MethodGet, server.URL+"/a", "one", nil)
+	require.EqualValues(t, 2, hits.Load(), "a template repeating its own request sends it")
+
+	sharedGet(t, transport, http.MethodGet, server.URL+"/a", "", nil)
+	require.EqualValues(t, 3, hits.Load(), "an unmarked request is always sent")
+
+	sharedGet(t, transport, http.MethodGet, server.URL+"/a", "three", http.Header{"Cookie": {"session=1"}})
+	require.EqualValues(t, 4, hits.Load(), "other cookies make another request")
+
+	picked := func(i int) http.Header { return http.Header{"User-Agent": {useragent.UserAgents[i].Raw}} }
+	sharedGet(t, transport, http.MethodGet, server.URL+"/b", "one", picked(0))
+	sharedGet(t, transport, http.MethodGet, server.URL+"/b", "two", picked(1))
+	require.EqualValues(t, 5, hits.Load(), "a User-Agent nuclei picked at random is not part of the request")
+	sharedGet(t, transport, http.MethodGet, server.URL+"/b", "three", http.Header{"User-Agent": {"custom-scanner"}})
+	require.EqualValues(t, 6, hits.Load(), "one a template sets is")
+
+	req, err := http.NewRequestWithContext(WithSharedResponse(context.Background(), "one"), http.MethodPost, server.URL+"/a", strings.NewReader("x"))
+	require.NoError(t, err)
+	for range 2 {
+		resp, err := transport.RoundTrip(req.Clone(req.Context()))
+		require.NoError(t, err)
+		_ = resp.Body.Close()
+	}
+	require.EqualValues(t, 8, hits.Load(), "a request with a body is always sent")
+}
+
+func TestSharedResponsesWaitForTheRequestInFlight(t *testing.T) {
+	var hits atomic.Int32
+	release := make(chan struct{})
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		hits.Add(1)
+		<-release
+		_, _ = io.WriteString(w, "slow")
+	}))
+	defer server.Close()
+	transport := newSharingTransport(http.DefaultTransport)
+
+	var wg sync.WaitGroup
+	bodies := make([]string, 5)
+	for i := range bodies {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			bodies[i] = sharedGet(t, transport, http.MethodGet, server.URL, "template-"+string(rune('a'+i)), nil)
+		}()
+	}
+	require.Eventually(t, func() bool { return hits.Load() > 0 }, 5*time.Second, time.Millisecond)
+	close(release)
+	wg.Wait()
+	require.EqualValues(t, 1, hits.Load())
+	require.Equal(t, []string{"slow", "slow", "slow", "slow", "slow"}, bodies)
+}
+
+func TestSharedResponsesSendLargeBodiesEachTime(t *testing.T) {
+	var hits atomic.Int32
+	large := strings.Repeat("x", sharedResponseMaxBody+10)
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		hits.Add(1)
+		_, _ = io.WriteString(w, large)
+	}))
+	defer server.Close()
+	transport := newSharingTransport(http.DefaultTransport)
+
+	require.Equal(t, large, sharedGet(t, transport, http.MethodGet, server.URL, "one", nil), "the whole body still arrives")
+	require.Equal(t, large, sharedGet(t, transport, http.MethodGet, server.URL, "two", nil))
+	require.EqualValues(t, 2, hits.Load())
+}
+
+type failOnce struct {
+	failed atomic.Bool
+	base   http.RoundTripper
+}
+
+func (f *failOnce) RoundTrip(req *http.Request) (*http.Response, error) {
+	if f.failed.CompareAndSwap(false, true) {
+		return nil, errors.New("connection reset")
+	}
+	return f.base.RoundTrip(req)
+}
+
+func TestSharedResponsesDoNotKeepFailures(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = io.WriteString(w, "ok")
+	}))
+	defer server.Close()
+	transport := newSharingTransport(&failOnce{base: http.DefaultTransport})
+
+	req, err := http.NewRequestWithContext(WithSharedResponse(context.Background(), "one"), http.MethodGet, server.URL, nil)
+	require.NoError(t, err)
+	_, err = transport.RoundTrip(req)
+	require.Error(t, err)
+	require.Equal(t, "ok", sharedGet(t, transport, http.MethodGet, server.URL, "two", nil), "the next template sends it again")
+}

pkg/protocols/http/request.go

@@ -1340,6 +1340,9 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 				}
 			}
 
+			if request.sharesResponse {
+				generatedRequest.request.WithContext(httpclientpool.WithSharedResponse(generatedRequest.request.Context(), request.options.TemplateID))
+			}
 			resp, err = executingClient.Do(generatedRequest.request)
 			sent = true
 

pkg/protocols/http/share_response_test.go

@@ -0,0 +1,19 @@
+package http
+
+import (
+	"testing"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/operators"
+	"github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors"
+	"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCanShareResponseKeepsTimingChecksOnTheirOwnRequest(t *testing.T) {
+	require.True(t, canShareResponse(&Request{Operators: operators.Operators{Matchers: []*matchers.Matcher{{Words: []string{"admin"}}}}}, ""))
+	require.False(t, canShareResponse(&Request{Operators: operators.Operators{Matchers: []*matchers.Matcher{{DSL: []string{"duration>=5"}}}}}, ""))
+	require.False(t, canShareResponse(&Request{Operators: operators.Operators{Matchers: []*matchers.Matcher{{Part: "duration"}}}}, ""))
+	require.False(t, canShareResponse(&Request{Operators: operators.Operators{Extractors: []*extractors.Extractor{{DSL: []string{"duration_2"}}}}}, ""))
+	require.False(t, canShareResponse(&Request{}, "http(1) && template.duration > 5"))
+	require.False(t, canShareResponse(&Request{Race: true}, ""))
+}