Merge pull request #8107 from projectdiscovery/feat/shared-responses

send identical requests once per scan

pkg/core/execute_options.go

@@ -9,6 +9,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types/scanstrategy"
@@ -35,6 +36,7 @@ func (e *Engine) ExecuteWithResults(ctx context.Context, templatesList []*templa
 func (e *Engine) ExecuteScanWithOpts(ctx context.Context, templatesList []*templates.Template, target provider.InputProvider, noCluster bool) *atomic.Bool {
 	results := &atomic.Bool{}
 	selfcontainedWg := &sync.WaitGroup{}
+	httpclientpool.ResetSharedResponses(e.options)
 
 	totalReqBeforeCluster := getRequestCount(templatesList) * int(target.Count())
 

pkg/protocols/common/protocolstate/dialers.go

@@ -16,6 +16,7 @@ type Dialers struct {
 	HTTPClientPool             *HTTPPool
 	PerHostRateLimitPool       any // *hostratelimit.PerHostRateLimitPool
 	HTTPToHTTPSPortTracker     any // *httpclientpool.HTTPToHTTPSPortTracker
+	SharedResponses            any // *httpclientpool.sharedResponses
 	HTTPDesyncHosts            *ExpiringSet
 	NetworkPolicy              *networkpolicy.NetworkPolicy
 	LocalFileAccessAllowed     bool

pkg/protocols/http/http.go

@@ -148,6 +148,9 @@ type Request struct {
 	replayClient      *http.Client // set when -replay-proxy is given
 	dialer            *fastdialer.Dialer
 	bodySize          *bodySizeEstimate
+	// sharesResponse lets other templates reuse a response this request gets,
+	// and this request reuse theirs (see httpclientpool.WithSharedResponse)
+	sharesResponse bool
 
 	// description: |
 	//   SelfContained specifies if the request is self-contained.
@@ -587,9 +590,24 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
 	if shouldDisableKeepAliveForHTTPProxy(request, options) {
 		request.connConfiguration.Connection.DisableKeepAlive = true
 	}
+	request.sharesResponse = canShareResponse(request, options.Flow)
 	return nil
 }
 
+// canShareResponse reports whether a response another template got for the
+// same request is as good as sending it again. A shared response arrives at
+// once, so nothing that measures time may take one: race requests, timing
+// analyzers, and a request whose operators or flow read the duration. The
+// operators are searched whole, so a new field that can read it, as LLM
+// matcher inputs do, is covered without being listed.
+func canShareResponse(request *Request, flow string) bool {
+	if request.Race || request.Analyzer != nil || strings.Contains(flow, "duration") {
+		return false
+	}
+	operators, err := json.Marshal(request.Operators)
+	return err == nil && !bytes.Contains(operators, []byte("duration"))
+}
+
 // RebuildGenerator rebuilds the generator for the request
 func (request *Request) RebuildGenerator() error {
 	generator, err := generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Catalog, request.options.Options.AttackType, request.options.Options)

pkg/protocols/http/httpclientpool/clientpool.go

@@ -514,7 +514,11 @@ func wrappedGet(options *types.Options, configuration *Configuration, host strin
 			}
 		}
 
-		return &connTrackingTransport{base: transport}, nil
+		tracking := &connTrackingTransport{base: transport}
+		if shared := getSharedResponses(options); shared != nil {
+			return &sharingTransport{base: tracking, shared: shared}, nil
+		}
+		return tracking, nil
 	}
 	redirectFlow := configuration.RedirectFlow
 	maxRedirects := configuration.MaxRedirects

pkg/protocols/http/httpclientpool/clientpool_idleconn_test.go

@@ -175,7 +175,9 @@ func TestIdleConnPoolSizeIsWiredToTransport(t *testing.T) {
 	client, err := Get(opts, &Configuration{}, "example.com")
 	require.NoError(t, err)
 
-	tracking, ok := client.HTTPClient.Transport.(*connTrackingTransport)
+	sharing, ok := client.HTTPClient.Transport.(*sharingTransport)
+	require.True(t, ok, "expected the shared-response transport wrapper")
+	tracking, ok := sharing.base.(*connTrackingTransport)
 	require.True(t, ok, "expected the connection-tracking transport wrapper")
 	transport, ok := tracking.base.(*http.Transport)
 	require.True(t, ok, "expected an *http.Transport underneath")

pkg/protocols/http/httpclientpool/shared_responses.go

@@ -0,0 +1,281 @@
+package httpclientpool
+
+import (
+	"bytes"
+	"container/list"
+	"context"
+	"io"
+	"maps"
+	"net/http"
+	"slices"
+	"strings"
+	"sync"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
+	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	"github.com/projectdiscovery/useragent"
+)
+
+// Templates send many requests another template already sent to the same
+// target, byte for byte: clustering merges only single-request templates. A
+// request marked with WithSharedResponse is sent once per scan and every
+// other template sending the same bytes gets a copy of the response. The key
+// is the request as it goes on the wire, cookies included, and the client
+// still processes each copy, so redirects and cookies behave as if it was
+// sent again.
+const (
+	// sharedResponseMaxBody keeps large downloads out of the store; they are
+	// sent each time
+	sharedResponseMaxBody   = 1 << 20
+	sharedResponsesMaxBytes = 64 << 20
+)
+
+type sharedResponseKey struct{}
+
+// WithSharedResponse marks the request of templateID as one whose response
+// other templates may reuse. A template sending the same request again gets
+// it sent again: a repeat within one template is intended.
+func WithSharedResponse(ctx context.Context, templateID string) context.Context {
+	return context.WithValue(ctx, sharedResponseKey{}, templateID)
+}
+
+type sharedResponses struct {
+	mu      sync.Mutex
+	entries map[string]*sharedEntry
+	// order holds stored entries oldest first, for eviction
+	order    *list.List
+	bytes    int
+	maxBytes int
+}
+
+type sharedEntry struct {
+	key        string
+	templateID string
+	ready      chan struct{}
+	response   *storedResponse
+}
+
+type storedResponse struct {
+	response *http.Response
+	body     []byte
+}
+
+func getSharedResponses(options *types.Options) *sharedResponses {
+	dialers := protocolstate.GetDialersWithId(options.ExecutionId)
+	if dialers == nil {
+		return nil
+	}
+	dialers.Lock()
+	defer dialers.Unlock()
+	if dialers.SharedResponses == nil {
+		dialers.SharedResponses = &sharedResponses{entries: make(map[string]*sharedEntry), order: list.New(), maxBytes: sharedResponsesMaxBytes}
+	}
+	shared, _ := dialers.SharedResponses.(*sharedResponses)
+	return shared
+}
+
+// ResetSharedResponses forgets the responses of earlier scans, so a scan
+// that reuses the execution, as an SDK engine does, sends its requests again.
+func ResetSharedResponses(options *types.Options) {
+	shared := getSharedResponses(options)
+	if shared == nil {
+		return
+	}
+	shared.mu.Lock()
+	defer shared.mu.Unlock()
+	clear(shared.entries)
+	shared.order.Init()
+	shared.bytes = 0
+}
+
+type sharingTransport struct {
+	base   http.RoundTripper
+	shared *sharedResponses
+}
+
+func (t *sharingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	templateID, marked := req.Context().Value(sharedResponseKey{}).(string)
+	if !marked || (req.Method != http.MethodGet && req.Method != http.MethodHead) || hasBody(req) {
+		return t.base.RoundTrip(req)
+	}
+	key := sharedRequestKey(req)
+
+	t.shared.mu.Lock()
+	entry, found := t.shared.entries[key]
+	if !found {
+		entry = &sharedEntry{key: key, templateID: templateID, ready: make(chan struct{})}
+		t.shared.entries[key] = entry
+		t.shared.mu.Unlock()
+		return t.send(req, entry)
+	}
+	t.shared.mu.Unlock()
+	if entry.templateID == templateID {
+		return t.base.RoundTrip(req)
+	}
+	select {
+	case <-entry.ready:
+	case <-req.Context().Done():
+		return nil, req.Context().Err()
+	}
+	if entry.response == nil {
+		// the first send failed or was too large to keep
+		return t.base.RoundTrip(req)
+	}
+	return entry.response.copyFor(req), nil
+}
+
+func (t *sharingTransport) CloseIdleConnections() {
+	type closeIdler interface{ CloseIdleConnections() }
+	if ci, ok := t.base.(closeIdler); ok {
+		ci.CloseIdleConnections()
+	}
+}
+
+// send sends the first request of key and stores its response for the rest.
+func (t *sharingTransport) send(req *http.Request, entry *sharedEntry) (*http.Response, error) {
+	defer close(entry.ready)
+	resp, err := t.base.RoundTrip(req)
+	if err != nil {
+		t.forget(entry)
+		return resp, err
+	}
+	body, err := io.ReadAll(io.LimitReader(resp.Body, sharedResponseMaxBody+1))
+	if err != nil || len(body) > sharedResponseMaxBody || !shareableStatus(resp.StatusCode) {
+		t.forget(entry)
+		// The bytes already taken off the body have to go back in front of it.
+		// A short read is still this caller's response: a Content-Length that
+		// is longer than the body is what some matchers look for. It is not
+		// stored, and neither is a body too large to keep or a refusal.
+		resp.Body = struct {
+			io.Reader
+			io.Closer
+		}{io.MultiReader(bytes.NewReader(body), resp.Body), resp.Body}
+		return resp, nil
+	}
+	_ = resp.Body.Close()
+	entry.response = &storedResponse{response: resp, body: body}
+	t.store(entry)
+	return entry.response.copyFor(req), nil
+}
+
+// shareableStatus reports whether code is an answer a later template can
+// reuse. 404 and 500 are answers templates match on. 429 and the gateway
+// errors mean the server did not answer, so the next template must send.
+func shareableStatus(code int) bool {
+	switch code {
+	case http.StatusTooManyRequests, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout:
+		return false
+	default:
+		return true
+	}
+}
+
+// hasBody reports whether req sends a body. retryablehttp gives every
+// request one, empty for a GET, so its content is what counts.
+func hasBody(req *http.Request) bool {
+	if req.Body == nil || req.Body == http.NoBody {
+		return false
+	}
+	if req.ContentLength > 0 || req.GetBody == nil {
+		return true
+	}
+	body, err := req.GetBody()
+	if err != nil {
+		return true
+	}
+	defer func() { _ = body.Close() }()
+	var first [1]byte
+	n, _ := body.Read(first[:])
+	return n > 0
+}
+
+func (t *sharingTransport) forget(entry *sharedEntry) {
+	t.shared.mu.Lock()
+	defer t.shared.mu.Unlock()
+	if t.shared.entries[entry.key] == entry {
+		delete(t.shared.entries, entry.key)
+	}
+}
+
+func (t *sharingTransport) store(entry *sharedEntry) {
+	t.shared.mu.Lock()
+	defer t.shared.mu.Unlock()
+	// a reset while it was in flight dropped it
+	if t.shared.entries[entry.key] != entry {
+		return
+	}
+	t.shared.order.PushBack(entry)
+	t.shared.bytes += entry.size()
+	for t.shared.bytes > t.shared.maxBytes && t.shared.order.Len() > 1 {
+		oldest := t.shared.order.Remove(t.shared.order.Front()).(*sharedEntry)
+		t.shared.bytes -= oldest.size()
+		if t.shared.entries[oldest.key] == oldest {
+			delete(t.shared.entries, oldest.key)
+		}
+	}
+}
+
+// sharedEntryOverhead stands for the response struct and bookkeeping, so
+// bodyless responses, as every HEAD gets, still count toward the cap.
+const sharedEntryOverhead = 512
+
+func (e *sharedEntry) size() int {
+	size := sharedEntryOverhead + len(e.key) + len(e.response.body)
+	for name, values := range e.response.response.Header {
+		size += len(name)
+		for _, value := range values {
+			size += len(value)
+		}
+	}
+	return size
+}
+
+func (s *storedResponse) copyFor(req *http.Request) *http.Response {
+	resp := *s.response
+	resp.Header = s.response.Header.Clone()
+	resp.Trailer = s.response.Trailer.Clone()
+	resp.Body = io.NopCloser(bytes.NewReader(s.body))
+	// A HEAD response has an empty body and a Content-Length for the body it
+	// did not send. Matchers read that length, so a reused HEAD keeps it.
+	if req.Method != http.MethodHead {
+		resp.ContentLength = int64(len(s.body))
+	}
+	resp.Request = req
+	return &resp
+}
+
+// randomUserAgents are the values nuclei picks a User-Agent from for every
+// request a template does not set one for. Two templates sending the same
+// request almost never pick the same one, and either could have picked the
+// other's, so a picked User-Agent is left out of the key.
+var randomUserAgents = sync.OnceValue(func() map[string]struct{} {
+	values := make(map[string]struct{}, len(useragent.UserAgents))
+	for _, userAgent := range useragent.UserAgents {
+		values[userAgent.Raw] = struct{}{}
+	}
+	return values
+})
+
+// sharedRequestKey is the request as the server receives it, apart from the
+// order of headers and a picked User-Agent.
+func sharedRequestKey(req *http.Request) string {
+	var key strings.Builder
+	key.WriteString(req.Method)
+	key.WriteByte(' ')
+	key.WriteString(req.URL.String())
+	key.WriteByte('\n')
+	key.WriteString(req.Host)
+	for _, name := range slices.Sorted(maps.Keys(req.Header)) {
+		values := req.Header[name]
+		if _, picked := randomUserAgents()[req.Header.Get("User-Agent")]; name == "User-Agent" && len(values) == 1 && picked {
+			continue
+		}
+		for _, value := range values {
+			key.WriteByte('\n')
+			key.WriteString(name)
+			key.WriteString(": ")
+			key.WriteString(value)
+		}
+	}
+	return key.String()
+}

pkg/protocols/http/httpclientpool/shared_responses_test.go

@@ -0,0 +1,308 @@
+package httpclientpool
+
+import (
+	"container/list"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/projectdiscovery/retryablehttp-go"
+	"github.com/projectdiscovery/useragent"
+	"github.com/stretchr/testify/require"
+)
+
+func newSharingTransport(base http.RoundTripper) *sharingTransport {
+	return &sharingTransport{base: base, shared: &sharedResponses{entries: make(map[string]*sharedEntry), order: list.New(), maxBytes: sharedResponsesMaxBytes}}
+}
+
+func sharedGet(t *testing.T, transport http.RoundTripper, method, url, templateID string, header http.Header) string {
+	t.Helper()
+	ctx := context.Background()
+	if templateID != "" {
+		ctx = WithSharedResponse(ctx, templateID)
+	}
+	req, err := http.NewRequestWithContext(ctx, method, url, nil)
+	require.NoError(t, err)
+	for name, values := range header {
+		req.Header[name] = values
+	}
+	resp, err := transport.RoundTrip(req)
+	require.NoError(t, err)
+	defer func() { _ = resp.Body.Close() }()
+	body, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	return string(body)
+}
+
+func TestSharedResponsesSendEachRequestOncePerScan(t *testing.T) {
+	var hits atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		n := hits.Add(1)
+		_, _ = io.WriteString(w, r.URL.Path+strings.Repeat("!", int(n)))
+	}))
+	defer server.Close()
+	transport := newSharingTransport(http.DefaultTransport)
+
+	first := sharedGet(t, transport, http.MethodGet, server.URL+"/a", "one", nil)
+	require.Equal(t, first, sharedGet(t, transport, http.MethodGet, server.URL+"/a", "two", nil), "another template gets the stored response")
+	require.EqualValues(t, 1, hits.Load())
+
+	// nuclei builds a GET with an empty body
+	empty, err := retryablehttp.NewRequestWithContext(WithSharedResponse(context.Background(), "four"), http.MethodGet, server.URL+"/a", strings.NewReader(""))
+	require.NoError(t, err)
+	resp, err := transport.RoundTrip(empty.Request)
+	require.NoError(t, err)
+	_ = resp.Body.Close()
+	require.EqualValues(t, 1, hits.Load(), "an empty body is no body")
+
+	sharedGet(t, transport, http.MethodGet, server.URL+"/a", "one", nil)
+	require.EqualValues(t, 2, hits.Load(), "a template repeating its own request sends it")
+
+	sharedGet(t, transport, http.MethodGet, server.URL+"/a", "", nil)
+	require.EqualValues(t, 3, hits.Load(), "an unmarked request is always sent")
+
+	sharedGet(t, transport, http.MethodGet, server.URL+"/a", "three", http.Header{"Cookie": {"session=1"}})
+	require.EqualValues(t, 4, hits.Load(), "other cookies make another request")
+
+	picked := func(i int) http.Header { return http.Header{"User-Agent": {useragent.UserAgents[i].Raw}} }
+	sharedGet(t, transport, http.MethodGet, server.URL+"/b", "one", picked(0))
+	sharedGet(t, transport, http.MethodGet, server.URL+"/b", "two", picked(1))
+	require.EqualValues(t, 5, hits.Load(), "a User-Agent nuclei picked at random is not part of the request")
+	sharedGet(t, transport, http.MethodGet, server.URL+"/b", "three", http.Header{"User-Agent": {"custom-scanner"}})
+	require.EqualValues(t, 6, hits.Load(), "one a template sets is")
+
+	for range 2 {
+		// A new request each time. Clone shares the body, so a second clone
+		// of one request can send ContentLength with an already-read body.
+		req, err := http.NewRequestWithContext(WithSharedResponse(context.Background(), "one"), http.MethodPost, server.URL+"/a", strings.NewReader("x"))
+		require.NoError(t, err)
+		resp, err := transport.RoundTrip(req)
+		require.NoError(t, err)
+		_ = resp.Body.Close()
+	}
+	require.EqualValues(t, 8, hits.Load(), "a request with a body is always sent")
+}
+
+func TestSharedResponsesWaitForTheRequestInFlight(t *testing.T) {
+	var hits atomic.Int32
+	release := make(chan struct{})
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		hits.Add(1)
+		<-release
+		_, _ = io.WriteString(w, "slow")
+	}))
+	defer server.Close()
+	transport := newSharingTransport(http.DefaultTransport)
+
+	var wg sync.WaitGroup
+	bodies := make([]string, 5)
+	for i := range bodies {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			bodies[i] = sharedGet(t, transport, http.MethodGet, server.URL, "template-"+string(rune('a'+i)), nil)
+		}()
+	}
+	require.Eventually(t, func() bool { return hits.Load() > 0 }, 5*time.Second, time.Millisecond)
+	close(release)
+	wg.Wait()
+	require.EqualValues(t, 1, hits.Load())
+	require.Equal(t, []string{"slow", "slow", "slow", "slow", "slow"}, bodies)
+}
+
+func TestSharedResponsesSendLargeBodiesEachTime(t *testing.T) {
+	var hits atomic.Int32
+	large := strings.Repeat("x", sharedResponseMaxBody+10)
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		hits.Add(1)
+		_, _ = io.WriteString(w, large)
+	}))
+	defer server.Close()
+	transport := newSharingTransport(http.DefaultTransport)
+
+	require.Equal(t, large, sharedGet(t, transport, http.MethodGet, server.URL, "one", nil), "the whole body still arrives")
+	require.Equal(t, large, sharedGet(t, transport, http.MethodGet, server.URL, "two", nil))
+	require.EqualValues(t, 2, hits.Load())
+}
+
+type failOnce struct {
+	failed atomic.Bool
+	base   http.RoundTripper
+}
+
+func (f *failOnce) RoundTrip(req *http.Request) (*http.Response, error) {
+	if f.failed.CompareAndSwap(false, true) {
+		return nil, errors.New("connection reset")
+	}
+	return f.base.RoundTrip(req)
+}
+
+func TestSharedResponsesDoNotKeepFailures(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = io.WriteString(w, "ok")
+	}))
+	defer server.Close()
+	transport := newSharingTransport(&failOnce{base: http.DefaultTransport})
+
+	req, err := http.NewRequestWithContext(WithSharedResponse(context.Background(), "one"), http.MethodGet, server.URL, nil)
+	require.NoError(t, err)
+	_, err = transport.RoundTrip(req)
+	require.Error(t, err)
+	require.Equal(t, "ok", sharedGet(t, transport, http.MethodGet, server.URL, "two", nil), "the next template sends it again")
+}
+
+func TestSharedResponsesStartEmptyEachScan(t *testing.T) {
+	var hits atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		hits.Add(1)
+		_, _ = io.WriteString(w, "ok")
+	}))
+	defer server.Close()
+	opts := newTestOptions(t, "test-shared-responses-reset")
+	transport := &sharingTransport{base: http.DefaultTransport, shared: getSharedResponses(opts)}
+
+	sharedGet(t, transport, http.MethodGet, server.URL, "one", nil)
+	sharedGet(t, transport, http.MethodGet, server.URL, "two", nil)
+	require.EqualValues(t, 1, hits.Load())
+
+	// an SDK engine runs its next scan on the same execution
+	ResetSharedResponses(opts)
+	sharedGet(t, transport, http.MethodGet, server.URL, "two", nil)
+	require.EqualValues(t, 2, hits.Load(), "a new scan does not see the responses of the last one")
+}
+
+func TestSharedResponsesCapCountsBodylessResponses(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+	defer server.Close()
+	transport := newSharingTransport(http.DefaultTransport)
+	transport.shared.maxBytes = 20 * sharedEntryOverhead
+
+	for i := range 40 {
+		sharedGet(t, transport, http.MethodHead, fmt.Sprintf("%s/%d", server.URL, i), "one", nil)
+	}
+	require.LessOrEqual(t, transport.shared.bytes, transport.shared.maxBytes)
+	require.Less(t, len(transport.shared.entries), 20, "HEAD responses are evicted too")
+}
+
+func TestSharedResponsesKeepHeadContentLength(t *testing.T) {
+	var hits atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		hits.Add(1)
+		w.Header().Set("Content-Length", "1234")
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer server.Close()
+	transport := newSharingTransport(http.DefaultTransport)
+
+	length := func(templateID string) int64 {
+		t.Helper()
+		req, err := http.NewRequestWithContext(WithSharedResponse(context.Background(), templateID), http.MethodHead, server.URL, nil)
+		require.NoError(t, err)
+		resp, err := transport.RoundTrip(req)
+		require.NoError(t, err)
+		t.Cleanup(func() { _ = resp.Body.Close() })
+		return resp.ContentLength
+	}
+
+	require.Equal(t, int64(1234), length("one"))
+	require.Equal(t, int64(1234), length("two"), "a reused HEAD keeps the length the server advertised")
+	require.EqualValues(t, 1, hits.Load())
+}
+
+func TestSharedResponsesDoNotKeepRefusals(t *testing.T) {
+	for _, status := range []int{http.StatusTooManyRequests, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout} {
+		t.Run(http.StatusText(status), func(t *testing.T) {
+			var hits atomic.Int32
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				hits.Add(1)
+				http.Error(w, "no", status)
+			}))
+			defer server.Close()
+			transport := newSharingTransport(http.DefaultTransport)
+
+			sharedGet(t, transport, http.MethodGet, server.URL, "one", nil)
+			sharedGet(t, transport, http.MethodGet, server.URL, "two", nil)
+			require.EqualValues(t, 2, hits.Load())
+		})
+	}
+}
+
+func TestSharedResponsesKeepNotFound(t *testing.T) {
+	var hits atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		hits.Add(1)
+		http.NotFound(w, r)
+	}))
+	defer server.Close()
+	transport := newSharingTransport(http.DefaultTransport)
+
+	sharedGet(t, transport, http.MethodGet, server.URL, "one", nil)
+	sharedGet(t, transport, http.MethodGet, server.URL, "two", nil)
+	require.EqualValues(t, 1, hits.Load())
+}
+
+// shortBody yields a prefix and then ErrUnexpectedEOF, which is what a
+// response with a Content-Length longer than its body looks like.
+type shortBody struct {
+	prefix []byte
+	sent   bool
+}
+
+func (s *shortBody) Read(p []byte) (int, error) {
+	if !s.sent {
+		s.sent = true
+		n := copy(p, s.prefix)
+		return n, io.ErrUnexpectedEOF
+	}
+	return 0, io.ErrUnexpectedEOF
+}
+
+func (s *shortBody) Close() error { return nil }
+
+type countingTransport struct {
+	hits atomic.Int32
+}
+
+func (c *countingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	c.hits.Add(1)
+	return &http.Response{
+		StatusCode:    http.StatusOK,
+		Header:        http.Header{"Content-Length": {"50000"}},
+		ContentLength: 50000,
+		Body:          &shortBody{prefix: []byte("fourteen bytes")},
+		Request:       req,
+	}, nil
+}
+
+func TestSharedResponsesDoNotKeepShortReads(t *testing.T) {
+	base := &countingTransport{}
+	transport := newSharingTransport(base)
+
+	send := func(templateID string) *http.Response {
+		t.Helper()
+		req, err := http.NewRequestWithContext(WithSharedResponse(context.Background(), templateID), http.MethodGet, "http://example.com", nil)
+		require.NoError(t, err)
+		resp, err := transport.RoundTrip(req)
+		require.NoError(t, err)
+		return resp
+	}
+
+	first := send("one")
+	body, err := io.ReadAll(first.Body)
+	require.ErrorIs(t, err, io.ErrUnexpectedEOF)
+	require.Equal(t, "fourteen bytes", string(body))
+	require.Equal(t, int64(50000), first.ContentLength)
+	_ = first.Body.Close()
+
+	second := send("two")
+	_ = second.Body.Close()
+	require.EqualValues(t, 2, base.hits.Load(), "a short read is not reused")
+}

pkg/protocols/http/request.go

@@ -1340,6 +1340,9 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 				}
 			}
 
+			if request.sharesResponse {
+				generatedRequest.request.WithContext(httpclientpool.WithSharedResponse(generatedRequest.request.Context(), request.options.TemplateID))
+			}
 			resp, err = executingClient.Do(generatedRequest.request)
 			sent = true
 

pkg/protocols/http/share_response_test.go

@@ -0,0 +1,20 @@
+package http
+
+import (
+	"testing"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/operators"
+	"github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors"
+	"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCanShareResponseKeepsTimingChecksOnTheirOwnRequest(t *testing.T) {
+	require.True(t, canShareResponse(&Request{Operators: operators.Operators{Matchers: []*matchers.Matcher{{Words: []string{"admin"}}}}}, ""))
+	require.False(t, canShareResponse(&Request{Operators: operators.Operators{Matchers: []*matchers.Matcher{{DSL: []string{"duration>=5"}}}}}, ""))
+	require.False(t, canShareResponse(&Request{Operators: operators.Operators{Matchers: []*matchers.Matcher{{Part: "duration"}}}}, ""))
+	require.False(t, canShareResponse(&Request{Operators: operators.Operators{Extractors: []*extractors.Extractor{{DSL: []string{"duration_2"}}}}}, ""))
+	require.False(t, canShareResponse(&Request{Operators: operators.Operators{Matchers: []*matchers.Matcher{{Type: matchers.MatcherTypeHolder{MatcherType: matchers.LLMMatcher}, Inputs: []string{"{{duration_1}}"}}}}}, ""), "llm inputs read the event too")
+	require.False(t, canShareResponse(&Request{}, "http(1) && template.duration > 5"))
+	require.False(t, canShareResponse(&Request{Race: true}, ""))
+}

pkg/tmplexec/flow/flow_executor_test.go

@@ -19,6 +19,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/progress"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
 	"github.com/projectdiscovery/nuclei/v3/pkg/scan"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
 	"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
@@ -377,6 +378,8 @@ func TestFlowProgressEndsComplete(t *testing.T) {
 	for i, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
 			hits.Store(0)
+			// each case is its own scan, which the engine starts this way
+			httpclientpool.ResetSharedResponses(executerOpts.Options)
 			p := &countingProgress{}
 			opts := *executerOpts
 			opts.Progress = p