fix(headless): synchronize page wrapper copies with history updates

(cherry picked from commit 190fb46436)

README.md

@@ -294,6 +294,8 @@ UNCOVER:
 RATE-LIMIT:
    -rl, -rate-limit int               maximum number of requests to send per second (default 150)
    -rld, -rate-limit-duration value   maximum number of requests to send per second (default 1s)
+   -host-backoff                      slow down a host that starts blocking (429, repeated 403s, connection failures)
+   -host-backoff-max value            maximum delay applied to a host that is blocking (default 30s)
    -per-host-rate-limit               enable per-host rate limiting (global rate limit becomes unlimited when enabled)
    -rlm, -rate-limit-minute int       maximum number of requests to send per minute (DEPRECATED)
    -bs, -bulk-size int                maximum number of hosts to be analyzed in parallel per template (default 25)
@@ -337,6 +339,7 @@ DEBUG:
    -dresp, -debug-resp        show all received responses
    -p, -proxy string[]        list of http/socks5 proxy to use (comma separated or file input)
    -pi, -proxy-internal       proxy all internal requests
+   -rp, -replay-proxy string  http/socks5 proxy to re-send requests that produced a finding through (e.g. burp)
    -ldf, -list-dsl-function   list all supported DSL function signatures
    -tlog, -trace-log string   file to write sent requests trace log
    -elog, -error-log string   file to write sent requests error log

cmd/nuclei/main.go

@@ -432,6 +432,8 @@ on extensive configurability, massive extensibility and ease of use.`)
 	flagSet.CreateGroup("rate-limit", "Rate-Limit",
 		flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "maximum number of requests to send per second"),
 		flagSet.DurationVarP(&options.RateLimitDuration, "rate-limit-duration", "rld", time.Second, "maximum number of requests to send per second"),
+		flagSet.BoolVar(&options.HostBackoff, "host-backoff", false, "slow down a host that starts blocking (429, repeated 403s, connection failures)"),
+		flagSet.DurationVar(&options.HostBackoffMax, "host-backoff-max", 30*time.Second, "maximum delay applied to a host that is blocking"),
 		flagSet.BoolVar(&options.PerHostRateLimit, "per-host-rate-limit", false, "enable per-host rate limiting (global rate limit becomes unlimited when enabled)"),
 		flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute (DEPRECATED)"),
 		flagSet.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "maximum number of hosts to be analyzed in parallel per template"),
@@ -482,6 +484,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.BoolVarP(&options.DebugResponse, "debug-resp", "dresp", false, "show all received responses"),
 		flagSet.StringSliceVarP(&options.Proxy, "proxy", "p", nil, "list of http/socks5 proxy to use (comma separated or file input)", goflags.FileCommaSeparatedStringSliceOptions),
 		flagSet.BoolVarP(&options.ProxyInternal, "proxy-internal", "pi", false, "proxy all internal requests"),
+		flagSet.StringVarP(&options.ReplayProxy, "replay-proxy", "rp", "", "http/socks5 proxy to re-send requests that produced a finding through (e.g. burp)"),
 		flagSet.BoolVarP(&options.ListDslSignatures, "list-dsl-function", "ldf", false, "list all supported DSL function signatures"),
 		flagSet.StringVarP(&options.TraceLogFile, "trace-log", "tlog", "", "file to write sent requests trace log"),
 		flagSet.StringVarP(&options.ErrorLogFile, "error-log", "elog", "", "file to write sent requests error log"),

go.mod

@@ -102,7 +102,7 @@ require (
 	github.com/projectdiscovery/tlsx v1.4.1
 	github.com/projectdiscovery/uncover v1.2.1
 	github.com/projectdiscovery/useragent v0.0.109
-	github.com/projectdiscovery/utils v0.11.9
+	github.com/projectdiscovery/utils v0.11.10-0.20261019085724-c5c64e928d17
 	github.com/projectdiscovery/wappalyzergo v0.3.4
 	github.com/projectdiscovery/yamldoc-go v1.0.7
 	github.com/redis/go-redis/v9 v9.19.0

go.sum

@@ -1003,8 +1003,8 @@ github.com/projectdiscovery/uncover v1.2.1 h1:8U46T/96CLT7BPoXBgkTvWqB06lOyeTSLv
 github.com/projectdiscovery/uncover v1.2.1/go.mod h1:0p8onrWxfpXQEYs90ZDzTSpu1107gWmodX1NWqu/+z4=
 github.com/projectdiscovery/useragent v0.0.109 h1:b86BPdZvmgbJousa04jAugDRY4Y+rAvEamE0TFCPmeE=
 github.com/projectdiscovery/useragent v0.0.109/go.mod h1:sRyBqDAcD31RKYrO8fyXGowLd36fNwY1g0MNW9mp7zk=
-github.com/projectdiscovery/utils v0.11.9 h1:aKLje3zPcZuOGmopN51PasWxPX/CypPwzP2ke/Qa2bA=
-github.com/projectdiscovery/utils v0.11.9/go.mod h1:d3BcbsHhuxW9T3jpvrgzwshvqhaL9qPSkoi+3h/DY8E=
+github.com/projectdiscovery/utils v0.11.10-0.20261019085724-c5c64e928d17 h1:aSNbcIUA6wZyHaGdZjFzmZ5irbMMg5Yc5WmuXdmnvyI=
+github.com/projectdiscovery/utils v0.11.10-0.20261019085724-c5c64e928d17/go.mod h1:d3BcbsHhuxW9T3jpvrgzwshvqhaL9qPSkoi+3h/DY8E=
 github.com/projectdiscovery/wappalyzergo v0.3.4 h1:yO+xel+C0d/vOFWcUKVZRpIDAfcEmsjpVDjQ0jifAUw=
 github.com/projectdiscovery/wappalyzergo v0.3.4/go.mod h1:6xTRCUQhX8m+L1eqJDQIALJlRv2Bnhrmku80YU9bBXE=
 github.com/projectdiscovery/yamldoc-go v1.0.7 h1:Odmjd1o4EkMff2F6QHi1dIYPIQPZUFy24YqmQCprmTk=

internal/runner/options.go

@@ -56,10 +56,18 @@ func ConfigureOptions() error {
 	return nil
 }
 
+// readsTargetsFromStdin reports whether targets are read from a stdin pipe.
+// Targets given with -u, -l, or uncover take its place: under cron, CI or an
+// agent stdin is often a pipe nothing writes to or closes, and reading it
+// would hold the scan until it does.
+func readsTargetsFromStdin(options *types.Options, stdinIsPipe bool) bool {
+	hasOtherTargets := len(options.Targets) > 0 || options.TargetsFilePath != "" || options.Uncover || len(options.UncoverQuery) > 0
+	return stdinIsPipe && !options.DisableStdin && !hasOtherTargets
+}
+
 // ParseOptions parses the command line flags provided by a user
 func ParseOptions(options *types.Options) {
-	// Check if stdin pipe was given
-	options.Stdin = !options.DisableStdin && fileutil.HasStdin()
+	options.Stdin = readsTargetsFromStdin(options, fileutil.HasStdin())
 
 	// Read the inputs from env variables that not passed by flag.
 	readEnvInputVars(options)
@@ -171,6 +179,9 @@ func ValidateOptions(options *types.Options) error {
 	if err := loadProxyServers(options); err != nil {
 		return err
 	}
+	if err := validateReplayProxy(options.ReplayProxy); err != nil {
+		return err
+	}
 	if options.Validate {
 		validateTemplatePaths(options.Logger, config.DefaultConfig.TemplatesDirectory, options.Templates, options.Workflows)
 	}

internal/runner/options_test.go

@@ -59,3 +59,24 @@ func TestParseHeadlessOptionalArguments(t *testing.T) {
 		})
 	}
 }
+
+func TestReadsTargetsFromStdin(t *testing.T) {
+	for _, tc := range []struct {
+		name        string
+		options     *types.Options
+		stdinIsPipe bool
+		want        bool
+	}{
+		{"piped stdin alone", &types.Options{}, true, true},
+		{"no pipe", &types.Options{}, false, false},
+		{"disabled", &types.Options{DisableStdin: true}, true, false},
+		{"targets given with -u", &types.Options{Targets: goflags.StringSlice{"example.com"}}, true, false},
+		{"targets given with -l", &types.Options{TargetsFilePath: "targets.txt"}, true, false},
+		{"uncover query", &types.Options{UncoverQuery: goflags.StringSlice{"example"}}, true, false},
+		{"uncover enabled", &types.Options{Uncover: true}, true, false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			require.Equal(t, tc.want, readsTargetsFromStdin(tc.options, tc.stdinIsPipe))
+		})
+	}
+}

internal/runner/proxy.go

@@ -65,3 +65,21 @@ func loadProxyServers(options *types.Options) error {
 	}
 	return nil
 }
+
+// validateReplayProxy rejects a -replay-proxy value the replay client cannot dial
+func validateReplayProxy(raw string) error {
+	if raw == "" {
+		return nil
+	}
+	proxyURL, err := url.Parse(raw)
+	if err != nil {
+		return errkit.Wrapf(err, "invalid replay proxy %q", raw)
+	}
+	switch proxyURL.Scheme {
+	case proxyutils.HTTP, proxyutils.HTTPS, proxyutils.SOCKS5:
+		if proxyURL.Host != "" {
+			return nil
+		}
+	}
+	return fmt.Errorf("invalid replay proxy %q: want an http, https or socks5 URL", raw)
+}

internal/runner/proxy_test.go

@@ -0,0 +1,16 @@
+package runner
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestValidateReplayProxy(t *testing.T) {
+	for _, valid := range []string{"", "http://127.0.0.1:8080", "https://proxy.example:8443", "socks5://127.0.0.1:1080"} {
+		require.NoError(t, validateReplayProxy(valid), valid)
+	}
+	for _, invalid := range []string{"127.0.0.1:8080", "ftp://127.0.0.1:21", "http://", "://bad"} {
+		require.Error(t, validateReplayProxy(invalid), invalid)
+	}
+}

internal/runner/runner.go

@@ -56,6 +56,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/globalmatchers"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/honeypotdetector"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hostbackoff"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit"
@@ -727,6 +728,10 @@ func (r *Runner) RunEnumeration() error {
 		executorOpts.HostErrorsCache = cache
 	}
 
+	if r.options.HostBackoff {
+		executorOpts.HostBackoff = hostbackoff.New(hostbackoff.Config{Max: r.options.HostBackoffMax})
+	}
+
 	executorEngine := core.New(r.options)
 	executorEngine.SetExecuterOptions(executorOpts)
 

internal/server/nuclei_sdk.go

@@ -28,6 +28,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/progress"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/globalmatchers"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hostbackoff"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/excludematchers"
@@ -106,6 +107,10 @@ func newNucleiExecutor(opts *NucleiExecutorOptions) (*nucleiExecutor, error) {
 		executorOpts.HostErrorsCache = cache
 	}
 
+	if opts.Options.HostBackoff {
+		executorOpts.HostBackoff = hostbackoff.New(hostbackoff.Config{Max: opts.Options.HostBackoffMax})
+	}
+
 	executorEngine := core.New(opts.Options)
 	executorEngine.SetExecuterOptions(executorOpts)
 

pkg/output/output_stats.go

@@ -13,7 +13,7 @@ type StatsOutputWriter struct {
 
 var _ Writer = &StatsOutputWriter{}
 
-// NewStatsOutputWriter returns a new StatsOutputWriter instance.
+// NewTrackerWriter returns a new StatsOutputWriter instance.
 func NewTrackerWriter(t *stats.Tracker) *StatsOutputWriter {
 	return &StatsOutputWriter{
 		colorizer: aurora.New(aurora.WithColors(true)),

pkg/protocols/common/helpers/deserialization/java.go

@@ -89,7 +89,7 @@ func generateCommonsCollections40Payload(cmd string) []byte {
 	return buffer.Bytes()
 }
 
-// generateCommonsCollections440PPayload generates commons-collections 3.1
+// generateCommonsCollections31Payload generates commons-collections 3.1
 // deserialization payload for a command.
 func generateCommonsCollections31Payload(cmd string) []byte {
 	buffer := &bytes.Buffer{}

pkg/protocols/common/hostbackoff/hostbackoff.go

@@ -0,0 +1,276 @@
+// Package hostbackoff paces requests to a host that signals it is being
+// overloaded, so a scan slows down instead of being blocked outright.
+//
+// nuclei already skips a host after repeated transport errors
+// (pkg/protocols/common/hosterrorscache), which is all or nothing. A target
+// that answers 429, or starts answering 403 to everything, is asking for less
+// traffic rather than none, and the useful response is to keep scanning it more
+// slowly.
+//
+// The control rule follows shipped practice rather than anything novel: a
+// per-host delay that grows multiplicatively while the host complains and
+// decays while it is healthy, with the host's own Retry-After honoured when it
+// asks for longer. Scrapy's AutoThrottle, Heritrix and Googlebot's crawl
+// capacity limit all work this way, and the asymmetry (a bad response may only
+// slow the scan, never speed it up) is what keeps one lucky success from
+// undoing a backoff.
+package hostbackoff
+
+import (
+	"context"
+	"math"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/projectdiscovery/gcache"
+)
+
+const (
+	// DefaultStep is the delay applied the first time a host signals overload.
+	DefaultStep = 250 * time.Millisecond
+	// DefaultMax caps the per-host delay. Past this the host is better handled
+	// by the host error cache, which skips it entirely.
+	DefaultMax = 30 * time.Second
+	// DefaultFactor grows the delay on each further complaint.
+	DefaultFactor = 2.0
+	// DefaultDecay shrinks the delay after a healthy response. It is far gentler
+	// than the growth so that a scan recovers gradually rather than immediately
+	// re-flooding a host that just recovered.
+	DefaultDecay = 0.8
+	// DefaultForbiddenStreak is how many consecutive 403s count as blocking. A
+	// single 403 is ordinary (an endpoint the scan is not authorised for), but a
+	// run of them usually means the host started refusing the scanner.
+	DefaultForbiddenStreak = 5
+	// DefaultMaxHosts bounds how many hosts are tracked at once, matching the
+	// host error cache. A scan can carry millions of targets, so the state has
+	// to be evictable; losing a host's entry only resets its delay, and the next
+	// complaint earns it again.
+	DefaultMaxHosts = 10000
+)
+
+// Config tunes the governor. The zero value is replaced by the defaults above.
+type Config struct {
+	Step            time.Duration
+	Max             time.Duration
+	Factor          float64
+	Decay           float64
+	ForbiddenStreak int
+	MaxHosts        int
+}
+
+func (c *Config) applyDefaults() {
+	if c.Step <= 0 {
+		c.Step = DefaultStep
+	}
+	if c.Max <= 0 {
+		c.Max = DefaultMax
+	}
+	if c.Factor <= 1 {
+		c.Factor = DefaultFactor
+	}
+	if c.Decay <= 0 || c.Decay >= 1 {
+		c.Decay = DefaultDecay
+	}
+	if c.ForbiddenStreak <= 0 {
+		c.ForbiddenStreak = DefaultForbiddenStreak
+	}
+	if c.MaxHosts <= 0 {
+		c.MaxHosts = DefaultMaxHosts
+	}
+}
+
+type hostState struct {
+	delay     time.Duration
+	forbidden int
+	// next is the earliest start time still free. Concurrent waits reserve
+	// successive slots so a delay paces the host instead of holding a burst
+	// and releasing it together.
+	next time.Time
+}
+
+// Governor holds the per-host delay for a scan.
+type Governor struct {
+	cfg   Config
+	mu    sync.Mutex
+	hosts gcache.Cache[string, *hostState]
+}
+
+// New returns a governor using cfg, with unset fields defaulted.
+func New(cfg Config) *Governor {
+	cfg.applyDefaults()
+	return &Governor{cfg: cfg, hosts: gcache.New[string, *hostState](cfg.MaxHosts).ARC().Build()}
+}
+
+// Delay reports the current pause for a host, zero when it looks healthy.
+func (g *Governor) Delay(host string) time.Duration {
+	if g == nil || host == "" {
+		return 0
+	}
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	if state, err := g.hosts.GetIFPresent(host); err == nil && state != nil {
+		return state.delay
+	}
+	return 0
+}
+
+// Wait pauses until this caller’s slot on the host. Callers that arrive
+// together take successive slots of the current delay, so they do not all
+// wake at once. It returns the context error if the scan is cancelled while
+// waiting, so a stop is not held up by a long backoff.
+func (g *Governor) Wait(ctx context.Context, host string) error {
+	if g == nil || host == "" {
+		return nil
+	}
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
+	g.mu.Lock()
+	state, err := g.hosts.GetIFPresent(host)
+	if err != nil || state == nil || state.delay <= 0 {
+		g.mu.Unlock()
+		return nil
+	}
+	now := time.Now()
+	start := state.next
+	if start.Before(now) {
+		start = now.Add(state.delay)
+	}
+	state.next = start.Add(state.delay)
+	_ = g.hosts.Set(host, state)
+	wait := start.Sub(now)
+	g.mu.Unlock()
+	if wait <= 0 {
+		return nil
+	}
+	timer := time.NewTimer(wait)
+	defer timer.Stop()
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-timer.C:
+		return nil
+	}
+}
+
+// Observe feeds one response back to the governor. retryAfter is the parsed
+// Retry-After header, zero when absent. err is the transport error, if any.
+func (g *Governor) Observe(host string, statusCode int, retryAfter time.Duration, err error) {
+	if g == nil || host == "" {
+		return
+	}
+	g.mu.Lock()
+	defer g.mu.Unlock()
+
+	state, lookupErr := g.hosts.GetIFPresent(host)
+	if lookupErr != nil || state == nil {
+		state = &hostState{}
+	}
+	defer func() { _ = g.hosts.Set(host, state) }()
+
+	if statusCode == http.StatusForbidden {
+		state.forbidden++
+	} else if healthyStatus(statusCode) {
+		state.forbidden = 0
+	}
+	// A 403 that names Retry-After is the host asking for a wait. The streak
+	// is only for bare 403s, which are often just an unauthorised endpoint.
+	if statusCode == http.StatusForbidden && retryAfter > 0 && state.forbidden < g.cfg.ForbiddenStreak {
+		state.forbidden = g.cfg.ForbiddenStreak
+	}
+
+	if !g.isBlocking(statusCode, err, state) {
+		// Only a response that actually served (2xx/3xx) is evidence the host
+		// recovered. A 404 or 500 is what most templates get back, and treating
+		// it as healthy would wipe the delay on the next probe.
+		if healthyStatus(statusCode) {
+			state.delay = time.Duration(float64(state.delay) * g.cfg.Decay)
+			if state.delay < g.cfg.Step {
+				state.delay = 0
+				state.next = time.Time{}
+			}
+		}
+		return
+	}
+
+	next := state.delay
+	if next <= 0 {
+		next = g.cfg.Step
+	} else {
+		next = time.Duration(math.Min(float64(next)*g.cfg.Factor, float64(g.cfg.Max)))
+	}
+	// The host named a wait of its own; respect it when it is the longer of the
+	// two, since it is the only figure that is not a guess.
+	if retryAfter > next {
+		next = retryAfter
+	}
+	if next > g.cfg.Max {
+		next = g.cfg.Max
+	}
+	state.delay = next
+}
+
+// isBlocking reports whether a response means the host wants less traffic.
+func (g *Governor) isBlocking(statusCode int, err error, state *hostState) bool {
+	if err != nil {
+		return true
+	}
+	switch statusCode {
+	case http.StatusTooManyRequests, http.StatusServiceUnavailable:
+		return true
+	case http.StatusForbidden:
+		return state.forbidden >= g.cfg.ForbiddenStreak
+	}
+	return false
+}
+
+// healthyStatus reports whether the host served a normal response. Template
+// misses and server errors are not recovery.
+func healthyStatus(statusCode int) bool {
+	return statusCode >= http.StatusOK && statusCode < http.StatusBadRequest
+}
+
+// RetryAfter parses a Retry-After header, in either its delay-seconds or
+// HTTP-date form. Delay-seconds is a non-negative decimal integer (RFC 9110);
+// fractional and compound duration strings are rejected. A digit string that
+// does not fit in a Duration saturates instead of being ignored. The result
+// is zero when the header is absent or unparsable, and never a negative wait
+// for a date already in the past.
+func RetryAfter(header string) time.Duration {
+	header = strings.TrimSpace(header)
+	if header == "" {
+		return 0
+	}
+	if seconds, ok := delaySeconds(header); ok {
+		return seconds
+	}
+	if when, err := http.ParseTime(header); err == nil {
+		if wait := time.Until(when); wait > 0 {
+			return wait
+		}
+	}
+	return 0
+}
+
+// delaySeconds parses a Retry-After delay-seconds value. The boolean is false
+// when header is not a decimal integer.
+func delaySeconds(header string) (time.Duration, bool) {
+	if header == "" {
+		return 0, false
+	}
+	for i := 0; i < len(header); i++ {
+		if header[i] < '0' || header[i] > '9' {
+			return 0, false
+		}
+	}
+	const maxDuration = time.Duration(1<<63 - 1)
+	const maxSeconds = uint64(maxDuration / time.Second)
+	seconds, err := strconv.ParseUint(header, 10, 64)
+	if err != nil || seconds > maxSeconds {
+		return maxDuration, true
+	}
+	return time.Duration(seconds) * time.Second, true
+}

pkg/protocols/common/hostbackoff/hostbackoff_test.go

@@ -0,0 +1,212 @@
+package hostbackoff
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestHealthyHostIsNotDelayed(t *testing.T) {
+	g := New(Config{})
+	for i := 0; i < 10; i++ {
+		g.Observe("acme.test", http.StatusOK, 0, nil)
+	}
+	require.Zero(t, g.Delay("acme.test"), "a host that answers normally must not be slowed")
+}
+
+func TestTooManyRequestsBacksOff(t *testing.T) {
+	g := New(Config{Step: 100 * time.Millisecond, Factor: 2, Max: time.Second})
+
+	g.Observe("acme.test", http.StatusTooManyRequests, 0, nil)
+	require.Equal(t, 100*time.Millisecond, g.Delay("acme.test"))
+
+	g.Observe("acme.test", http.StatusTooManyRequests, 0, nil)
+	require.Equal(t, 200*time.Millisecond, g.Delay("acme.test"), "a repeat complaint grows the delay")
+}
+
+func TestServiceUnavailableBacksOff(t *testing.T) {
+	g := New(Config{Step: 100 * time.Millisecond})
+	g.Observe("acme.test", http.StatusServiceUnavailable, 0, nil)
+	require.Equal(t, 100*time.Millisecond, g.Delay("acme.test"))
+}
+
+func TestTransportErrorBacksOff(t *testing.T) {
+	g := New(Config{Step: 100 * time.Millisecond})
+	g.Observe("acme.test", 0, 0, errors.New("connection reset"))
+	require.Equal(t, 100*time.Millisecond, g.Delay("acme.test"))
+}
+
+// A single 403 is ordinary: plenty of templates probe endpoints the scan is not
+// authorised for. A run of them is the host refusing the scanner.
+func TestForbiddenOnlyBacksOffAsAStreak(t *testing.T) {
+	g := New(Config{Step: 100 * time.Millisecond, ForbiddenStreak: 3})
+
+	g.Observe("acme.test", http.StatusForbidden, 0, nil)
+	g.Observe("acme.test", http.StatusForbidden, 0, nil)
+	require.Zero(t, g.Delay("acme.test"), "an occasional 403 is not a block")
+
+	g.Observe("acme.test", http.StatusForbidden, 0, nil)
+	require.Equal(t, 100*time.Millisecond, g.Delay("acme.test"), "a streak is")
+}
+
+func TestOkResetsTheForbiddenStreak(t *testing.T) {
+	g := New(Config{Step: 100 * time.Millisecond, ForbiddenStreak: 3})
+	g.Observe("acme.test", http.StatusForbidden, 0, nil)
+	g.Observe("acme.test", http.StatusForbidden, 0, nil)
+	g.Observe("acme.test", http.StatusOK, 0, nil)
+	g.Observe("acme.test", http.StatusForbidden, 0, nil)
+	require.Zero(t, g.Delay("acme.test"), "the streak must start again after a healthy response")
+}
+
+// The host's own figure is the only one that is not a guess, so it wins when it
+// is longer than what the governor would have chosen.
+func TestRetryAfterIsHonouredWhenLonger(t *testing.T) {
+	g := New(Config{Step: 100 * time.Millisecond, Max: time.Minute})
+
+	g.Observe("acme.test", http.StatusTooManyRequests, 5*time.Second, nil)
+	require.Equal(t, 5*time.Second, g.Delay("acme.test"))
+
+	// a shorter Retry-After does not undo a longer backoff already earned
+	g.Observe("acme.test", http.StatusTooManyRequests, time.Millisecond, nil)
+	require.Equal(t, 10*time.Second, g.Delay("acme.test"))
+}
+
+func TestDelayIsCapped(t *testing.T) {
+	g := New(Config{Step: time.Second, Factor: 10, Max: 2 * time.Second})
+	for i := 0; i < 5; i++ {
+		g.Observe("acme.test", http.StatusTooManyRequests, time.Hour, nil)
+	}
+	require.Equal(t, 2*time.Second, g.Delay("acme.test"), "neither growth nor Retry-After may exceed the cap")
+}
+
+func TestHealthyResponsesDecayTheDelay(t *testing.T) {
+	g := New(Config{Step: 100 * time.Millisecond, Factor: 2, Decay: 0.5})
+	g.Observe("acme.test", http.StatusTooManyRequests, 0, nil)
+	g.Observe("acme.test", http.StatusTooManyRequests, 0, nil)
+	require.Equal(t, 200*time.Millisecond, g.Delay("acme.test"))
+
+	g.Observe("acme.test", http.StatusOK, 0, nil)
+	require.Equal(t, 100*time.Millisecond, g.Delay("acme.test"))
+
+	// below the first step there is nothing useful left to wait for
+	g.Observe("acme.test", http.StatusOK, 0, nil)
+	require.Zero(t, g.Delay("acme.test"))
+}
+
+func TestHostsAreIndependent(t *testing.T) {
+	g := New(Config{Step: 100 * time.Millisecond})
+	g.Observe("blocked.test", http.StatusTooManyRequests, 0, nil)
+	require.Equal(t, 100*time.Millisecond, g.Delay("blocked.test"))
+	require.Zero(t, g.Delay("healthy.test"), "one host complaining must not slow another")
+}
+
+func TestWaitSleepsAndRespectsCancellation(t *testing.T) {
+	g := New(Config{Step: 50 * time.Millisecond})
+
+	start := time.Now()
+	require.NoError(t, g.Wait(context.Background(), "quiet.test"))
+	require.Less(t, time.Since(start), 10*time.Millisecond, "a healthy host is not waited on")
+
+	g.Observe("slow.test", http.StatusTooManyRequests, 0, nil)
+	start = time.Now()
+	require.NoError(t, g.Wait(context.Background(), "slow.test"))
+	require.GreaterOrEqual(t, time.Since(start), 50*time.Millisecond)
+
+	// a cancelled scan must not be held up by a long backoff
+	g.Observe("stuck.test", http.StatusTooManyRequests, time.Hour, nil)
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	require.ErrorIs(t, g.Wait(ctx, "stuck.test"), context.Canceled)
+}
+
+func TestWaitSpacesConcurrentCallers(t *testing.T) {
+	const step = 60 * time.Millisecond
+	g := New(Config{Step: step, Max: time.Second})
+	g.Observe("burst.test", http.StatusTooManyRequests, 0, nil)
+
+	start := time.Now()
+	var wg sync.WaitGroup
+	wg.Add(2)
+	for i := 0; i < 2; i++ {
+		go func() {
+			defer wg.Done()
+			require.NoError(t, g.Wait(context.Background(), "burst.test"))
+		}()
+	}
+	wg.Wait()
+	require.GreaterOrEqual(t, time.Since(start), 2*step-10*time.Millisecond)
+}
+
+func TestForbiddenRetryAfterBacksOffImmediately(t *testing.T) {
+	g := New(Config{Step: 100 * time.Millisecond, ForbiddenStreak: 5, Max: time.Minute})
+	g.Observe("acme.test", http.StatusForbidden, 2*time.Second, nil)
+	require.Equal(t, 2*time.Second, g.Delay("acme.test"))
+}
+
+func TestNilGovernorIsInert(t *testing.T) {
+	var g *Governor
+	require.Zero(t, g.Delay("acme.test"))
+	require.NoError(t, g.Wait(context.Background(), "acme.test"))
+	require.NotPanics(t, func() { g.Observe("acme.test", http.StatusTooManyRequests, 0, nil) })
+}
+
+func TestRetryAfterParsing(t *testing.T) {
+	require.Equal(t, 5*time.Second, RetryAfter("5"))
+	require.Equal(t, 5*time.Second, RetryAfter(" 5 "))
+	require.Zero(t, RetryAfter(""))
+	require.Zero(t, RetryAfter("not-a-number"))
+	require.Zero(t, RetryAfter("-3"), "a negative wait is meaningless")
+	require.Zero(t, RetryAfter(time.Now().Add(-time.Hour).UTC().Format(http.TimeFormat)), "a date in the past is not a wait")
+
+	future := RetryAfter(time.Now().Add(30 * time.Second).UTC().Format(http.TimeFormat))
+	require.Greater(t, future, 25*time.Second)
+	require.LessOrEqual(t, future, 30*time.Second)
+}
+
+// The previous parser appended "s" and called time.ParseDuration, which accepts
+// fractional seconds and Go duration strings, and returns an error (then zero)
+// when the integer does not fit.
+func TestRetryAfterRejectsDurationSyntax(t *testing.T) {
+	fractional, err := time.ParseDuration("1.5s")
+	require.NoError(t, err)
+	require.Equal(t, 1500*time.Millisecond, fractional)
+	require.Zero(t, RetryAfter("1.5"))
+
+	compound, err := time.ParseDuration("1m3s")
+	require.NoError(t, err)
+	require.Equal(t, 63*time.Second, compound)
+	require.Zero(t, RetryAfter("1m3"))
+
+	_, err = time.ParseDuration("999999999999999999999s")
+	require.Error(t, err)
+	require.Equal(t, time.Duration(1<<63-1), RetryAfter("999999999999999999999"))
+}
+
+func TestNotFoundDoesNotClearBackoff(t *testing.T) {
+	g := New(Config{Step: 100 * time.Millisecond, Decay: 0.5})
+	g.Observe("acme.test", http.StatusTooManyRequests, 0, nil)
+	g.Observe("acme.test", http.StatusNotFound, 0, nil)
+	require.Equal(t, 100*time.Millisecond, g.Delay("acme.test"))
+
+	g.Observe("acme.test", http.StatusInternalServerError, 0, nil)
+	require.Equal(t, 100*time.Millisecond, g.Delay("acme.test"))
+
+	g.Observe("acme.test", http.StatusOK, 0, nil)
+	require.Zero(t, g.Delay("acme.test"))
+}
+
+func TestForbiddenStreakSurvivesAMiss(t *testing.T) {
+	g := New(Config{Step: 100 * time.Millisecond, ForbiddenStreak: 3})
+	g.Observe("acme.test", http.StatusForbidden, 0, nil)
+	g.Observe("acme.test", http.StatusForbidden, 0, nil)
+	g.Observe("acme.test", http.StatusNotFound, 0, nil)
+	require.Zero(t, g.Delay("acme.test"))
+
+	g.Observe("acme.test", http.StatusForbidden, 0, nil)
+	require.Equal(t, 100*time.Millisecond, g.Delay("acme.test"))
+}

pkg/protocols/common/interactsh/interactsh.go

@@ -242,7 +242,7 @@ func (c *Client) poll() error {
 	return nil
 }
 
-// requestShouldStopAtFirstmatch checks if further interactions should be stopped
+// requestShouldStopAtFirstMatch checks if further interactions should be stopped
 // note: extra care should be taken while using this function since internalEvent is
 // synchronized all the time and if caller functions has already acquired lock its best to explicitly specify that
 // we could use `TryLock()` but that may over complicate things and need to differentiate

pkg/protocols/common/protocolstate/context.go

@@ -21,14 +21,14 @@ func WithExecutionID(ctx context.Context, executionContext *ExecutionContext) co
 	return context.WithValue(ctx, executionIDKey, executionContext)
 }
 
-// HasExecutionID checks if the context has an execution ID
+// HasExecutionContext checks if the context has an execution context.
 func HasExecutionContext(ctx context.Context) bool {
 	_, ok := ctx.Value(executionIDKey).(*ExecutionContext)
 	return ok
 }
 
-// GetExecutionID retrieves the execution ID from the context
-// Returns empty string if no execution ID is set
+// GetExecutionContext retrieves the execution context from the context.
+// It returns nil if no execution context is set.
 func GetExecutionContext(ctx context.Context) *ExecutionContext {
 	if id, ok := ctx.Value(executionIDKey).(*ExecutionContext); ok {
 		return id

pkg/protocols/common/protocolstate/memguardian.go

@@ -5,6 +5,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/docker/go-units"
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/utils/env"
 	httputil "github.com/projectdiscovery/utils/http"
@@ -16,6 +17,7 @@ var (
 	MaxBytesBufferAllocOnLowMemory = env.GetEnvOrDefault("MEMGUARDIAN_ALLOC", 0)
 	memTimer                       *time.Ticker
 	cancelFunc                     context.CancelFunc
+	memWatcherDone                 chan struct{}
 	muGlobalChange                 sync.Mutex
 )
 
@@ -25,12 +27,18 @@ func StartActiveMemGuardian(ctx context.Context) {
 	if memguardian.DefaultMemGuardian == nil || memTimer != nil {
 		return
 	}
+	if err := refreshMemoryWarning(); err != nil {
+		gologger.Warning().Msgf("could not check memory pressure: %s", err)
+	}
 
 	memTimer = time.NewTicker(memguardian.DefaultInterval)
 	ctx, cancelFunc = context.WithCancel(ctx)
+	memWatcherDone = make(chan struct{})
 
 	ticker := memTimer
+	done := memWatcherDone
 	go func(t *time.Ticker) {
+		defer close(done)
 		if t == nil {
 			return
 		}
@@ -39,6 +47,9 @@ func StartActiveMemGuardian(ctx context.Context) {
 			case <-ctx.Done():
 				return
 			case <-t.C:
+				if err := refreshMemoryWarning(); err != nil {
+					gologger.Warning().Msgf("could not check memory pressure: %s", err)
+				}
 				if IsLowOnMemory() {
 					_ = GlobalGuardBytesBufferAlloc()
 				} else {
@@ -49,6 +60,24 @@ func StartActiveMemGuardian(ctx context.Context) {
 	}(ticker)
 }
 
+func refreshMemoryWarning() error {
+	usedRatio, usedBytes, err := memguardian.UsedRam()
+	if err != nil {
+		return err
+	}
+
+	warning := usedRatio >= memguardian.DefaultMaxUsedRamRatio
+	if maxRam := env.GetEnvOrDefault(memguardian.MemGuardianMaxUsedMemoryENV, ""); maxRam != "" {
+		maxBytes, err := units.FromHumanSize(maxRam)
+		if err != nil {
+			return err
+		}
+		warning = warning || maxBytes > 0 && usedBytes >= uint64(maxBytes)
+	}
+	memguardian.DefaultMemGuardian.Warning.Store(warning)
+	return nil
+}
+
 func StopActiveMemGuardian() {
 	muGlobalChange.Lock()
 	defer muGlobalChange.Unlock()
@@ -61,6 +90,10 @@ func StopActiveMemGuardian() {
 		cancelFunc()
 		cancelFunc = nil
 	}
+	if memWatcherDone != nil {
+		<-memWatcherDone
+		memWatcherDone = nil
+	}
 	if memTimer != nil {
 		memTimer.Stop()
 		memTimer = nil
@@ -77,7 +110,7 @@ func IsLowOnMemory() bool {
 // GuardThreads on caller
 func GuardThreadsOrDefault(current int) int {
 	if MaxThreadsOnLowMemory > 0 {
-		return MaxThreadsOnLowMemory
+		return min(current, MaxThreadsOnLowMemory)
 	}
 
 	fraction := int(current / 5)

pkg/protocols/common/protocolstate/memguardian_test.go

@@ -127,3 +127,45 @@ func TestMemGuardianReset(t *testing.T) {
 	StopActiveMemGuardian()
 	time.Sleep(10 * time.Millisecond) // Allow cleanup
 }
+
+func TestActiveMemGuardianUpdatesLowMemoryWarning(t *testing.T) {
+	StopActiveMemGuardian()
+	t.Setenv(memguardian.MemGuardianMaxUsedMemoryENV, "")
+	original := memguardian.DefaultMemGuardian
+	originalInterval := memguardian.DefaultInterval
+	originalRatio := memguardian.DefaultMaxUsedRamRatio
+	memguardian.DefaultInterval = 5 * time.Millisecond
+	memguardian.DefaultMaxUsedRamRatio = 1
+	guardian, err := memguardian.New(
+		memguardian.WitInterval(memguardian.DefaultInterval),
+		memguardian.WithMaxRamRatioWarning(1),
+	)
+	require.NoError(t, err)
+	memguardian.DefaultMemGuardian = guardian
+	t.Cleanup(func() {
+		StopActiveMemGuardian()
+		guardian.Close()
+		memguardian.DefaultMemGuardian = original
+		memguardian.DefaultInterval = originalInterval
+		memguardian.DefaultMaxUsedRamRatio = originalRatio
+	})
+
+	StartActiveMemGuardian(context.Background())
+	require.Eventually(t, IsLowOnMemory, 100*time.Millisecond, 5*time.Millisecond)
+	StopActiveMemGuardian()
+	memguardian.DefaultMaxUsedRamRatio = 100
+	require.NoError(t, refreshMemoryWarning())
+	require.False(t, IsLowOnMemory())
+	t.Setenv(memguardian.MemGuardianMaxUsedMemoryENV, "0")
+	require.NoError(t, refreshMemoryWarning())
+	require.False(t, IsLowOnMemory())
+}
+
+func TestGuardThreadsNeverIncreasesConcurrency(t *testing.T) {
+	previous := MaxThreadsOnLowMemory
+	MaxThreadsOnLowMemory = 5
+	t.Cleanup(func() { MaxThreadsOnLowMemory = previous })
+
+	require.Equal(t, 2, GuardThreadsOrDefault(2))
+	require.Equal(t, 5, GuardThreadsOrDefault(25))
+}

pkg/protocols/headless/engine/page.go

@@ -52,9 +52,10 @@ type HistoryData struct {
 
 // Options contains additional configuration options for the browser instance
 type Options struct {
-	Timeout       time.Duration
-	DisableCookie bool
-	Options       *types.Options
+	Timeout        time.Duration
+	DisableCookie  bool
+	Options        *types.Options
+	OnActionTiming func(ActionType, time.Duration)
 }
 
 // Run runs a list of actions by creating a new page in the browser.

pkg/protocols/headless/engine/page_actions.go

@@ -52,6 +52,7 @@ func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action) (ou
 
 	type deferredWaitFunc struct {
 		actionIndex int
+		actionType  ActionType
 		run         func() error
 	}
 
@@ -76,6 +77,7 @@ func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action) (ou
 		actionIndex := len(p.ActionDurations)
 		timeStart := time.Now()
 		var deferredWaitDuration time.Duration
+		deferredWaitFailed := false
 		switch act.ActionType.ActionType {
 		case ActionNavigate:
 			err = p.NavigateURL(act, outData)
@@ -84,18 +86,24 @@ func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action) (ou
 				for _, waitFunc := range waitFuncs {
 					if waitFunc.run != nil {
 						waitStart := time.Now()
-						if err := waitFunc.run(); err != nil {
-							waitDuration := time.Since(waitStart)
-							addActionDuration(p.ActionDurations, waitFunc.actionIndex, waitDuration)
-							return nil, errkit.Wrap(err, "error occurred while executing waitFunc")
-						}
+						waitErr := waitFunc.run()
 						waitDuration := time.Since(waitStart)
 						addActionDuration(p.ActionDurations, waitFunc.actionIndex, waitDuration)
+						if p.options.OnActionTiming != nil {
+							p.options.OnActionTiming(waitFunc.actionType, waitDuration)
+						}
 						deferredWaitDuration += waitDuration
+						if waitErr != nil {
+							err = waitErr
+							deferredWaitFailed = true
+							break
+						}
 					}
 				}
 
-				p.lastActionNavigate = act
+				if err == nil {
+					p.lastActionNavigate = act
+				}
 			}
 		case ActionScript:
 			err = p.RunScript(act, outData)
@@ -141,7 +149,7 @@ func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action) (ou
 			var waitFunc func() error
 			waitFunc, err = p.WaitEvent(act, outData)
 			if waitFunc != nil {
-				waitFuncs = append(waitFuncs, deferredWaitFunc{actionIndex: actionIndex, run: waitFunc})
+				waitFuncs = append(waitFuncs, deferredWaitFunc{actionIndex: actionIndex, actionType: act.ActionType.ActionType, run: waitFunc})
 			}
 		case ActionWaitDialog:
 			err = p.HandleDialog(act, outData)
@@ -177,11 +185,20 @@ func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action) (ou
 		default:
 			executed = false
 		}
-		if err != nil {
-			return nil, errors.Wrap(err, "error occurred executing action")
-		}
 		if executed {
-			p.ActionDurations = append(p.ActionDurations, max(time.Since(timeStart)-deferredWaitDuration, 0))
+			duration := max(time.Since(timeStart)-deferredWaitDuration, 0)
+			if p.options.OnActionTiming != nil {
+				p.options.OnActionTiming(act.ActionType.ActionType, duration)
+			}
+			if err == nil {
+				p.ActionDurations = append(p.ActionDurations, duration)
+			}
+		}
+		if err != nil {
+			if deferredWaitFailed {
+				return nil, errkit.Wrap(err, "error occurred while executing waitFunc")
+			}
+			return nil, errors.Wrap(err, "error occurred executing action")
 		}
 	}
 	return outData, nil
@@ -230,7 +247,9 @@ func (p *Page) WaitVisible(act *Action, out ActionData) error {
 }
 
 func (p *Page) Sleeper(pollTimeout, timeout time.Duration) *Page {
+	p.mutex.RLock()
 	page := *p
+	p.mutex.RUnlock()
 	page.page = page.Page().Sleeper(func() utils.Sleeper {
 		return createBackOffSleeper(pollTimeout, timeout)
 	})
@@ -238,7 +257,9 @@ func (p *Page) Sleeper(pollTimeout, timeout time.Duration) *Page {
 }
 
 func (p *Page) Timeout(timeout time.Duration) *Page {
+	p.mutex.RLock()
 	page := *p
+	p.mutex.RUnlock()
 	page.page = page.Page().Timeout(timeout)
 	return &page
 }
@@ -732,6 +753,18 @@ func (p *Page) SelectInputElement(act *Action, out ActionData) error {
 
 // WaitPageLifecycleEvent waits for specified page lifecycle event name
 func (p *Page) WaitPageLifecycleEvent(act *Action, out ActionData, event proto.PageLifecycleEventName) error {
+	if event == proto.PageLifecycleEventNameLoad {
+		dur, err := getTimeout(p, act)
+		if err != nil {
+			return err
+		}
+		if err := p.page.Timeout(dur).WaitLoad(); err != nil {
+			return err
+		}
+		p.updateLastNavigatedURL()
+		return nil
+	}
+
 	fn, err := getNavigationFunc(p, act, event)
 	if err != nil {
 		return err

pkg/protocols/headless/engine/page_actions_test.go

@@ -18,6 +18,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/go-rod/rod"
+	"github.com/go-rod/rod/lib/proto"
 	"github.com/stretchr/testify/require"
 
 	"github.com/projectdiscovery/nuclei/v3/internal/tests/testheadless"
@@ -153,6 +155,26 @@ func TestLocatorInteractshTracking(t *testing.T) {
 	})
 }
 
+func TestSleeperConcurrentHistoryUpdate(t *testing.T) {
+	page := &Page{page: &rod.Page{}, mutex: &sync.RWMutex{}}
+	start := make(chan struct{})
+	var done sync.WaitGroup
+	done.Add(1)
+	go func() {
+		defer done.Done()
+		<-start
+		for range 10000 {
+			page.addToHistory(HistoryData{})
+		}
+	}()
+
+	close(start)
+	for range 10000 {
+		page.Sleeper(time.Millisecond, time.Second)
+	}
+	done.Wait()
+}
+
 func TestActionNavigate(t *testing.T) {
 	response := `
 		<html>
@@ -642,6 +664,47 @@ func TestActionWaitLoad(t *testing.T) {
 	})
 }
 
+func TestActionWaitLoadAfterPageAlreadyLoaded(t *testing.T) {
+	actions := []*Action{{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}}
+	testHeadlessSimpleResponse(t, "<html><body>ready</body></html>", actions, 10*time.Second, func(page *Page, err error, _ ActionData) {
+		require.NoError(t, err)
+		require.NoError(t, page.Page().WaitLoad())
+
+		wait := &Action{ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}, Data: map[string]string{"timeout": "2s"}}
+		started := time.Now()
+		require.NoError(t, page.WaitPageLifecycleEvent(wait, nil, proto.PageLifecycleEventNameLoad))
+		require.Less(t, time.Since(started), 1500*time.Millisecond, "a completed load must not consume the full wait timeout")
+	})
+}
+
+func TestActionWaitLoadWaitsForPendingResource(t *testing.T) {
+	resourceDone := make(chan struct{})
+	actions := []*Action{
+		{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}},
+		{ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}},
+	}
+	testHeadless(t, actions, 10*time.Second, func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/slow.js" {
+			w.Header().Set("Content-Type", "text/javascript")
+			time.Sleep(300 * time.Millisecond)
+			_, _ = fmt.Fprint(w, "window.resourceLoaded = true")
+			close(resourceDone)
+			return
+		}
+		_, _ = fmt.Fprint(w, `<html><script src="/slow.js"></script></html>`)
+	}, func(page *Page, err error, _ ActionData) {
+		require.NoError(t, err)
+		select {
+		case <-resourceDone:
+		default:
+			t.Fatal("waitload returned before the pending resource completed")
+		}
+		loaded, evalErr := page.Page().Eval(`() => window.resourceLoaded`)
+		require.NoError(t, evalErr)
+		require.True(t, loaded.Value.Bool())
+	})
+}
+
 func TestActionGetResource(t *testing.T) {
 	response := `
 		<html>
@@ -839,6 +902,25 @@ func TestActionWaitEventDuration(t *testing.T) {
 	})
 }
 
+func TestActionNavigateDurationIncludesResponseHeaderWait(t *testing.T) {
+	actions := []*Action{
+		{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}},
+		{ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}},
+	}
+
+	handler := func(w http.ResponseWriter, _ *http.Request) {
+		time.Sleep(350 * time.Millisecond)
+		_, _ = fmt.Fprintln(w, `<html><body>loaded</body></html>`)
+	}
+
+	testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, _ ActionData) {
+		require.NoError(t, err)
+		require.Len(t, page.ActionDurations, 2)
+		require.GreaterOrEqual(t, page.ActionDurations[0], 300*time.Millisecond)
+		require.Less(t, page.ActionDurations[1], time.Second)
+	})
+}
+
 func TestActionWaitEventSeesEventFiredBeforeWaiting(t *testing.T) {
 	actions := []*Action{
 		{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}},
@@ -1071,10 +1153,21 @@ func TestHeadlessRunHonorsParentCancellation(t *testing.T) {
 	}}
 
 	startedAt := time.Now()
-	_, page, err := instance.Run(input, actions, nil, &Options{Timeout: 5 * time.Second, Options: opts})
+	var timedAction ActionType
+	var timedDuration time.Duration
+	_, page, err := instance.Run(input, actions, nil, &Options{
+		Timeout: 5 * time.Second,
+		Options: opts,
+		OnActionTiming: func(action ActionType, duration time.Duration) {
+			timedAction = action
+			timedDuration = duration
+		},
+	})
 
 	require.ErrorIs(t, err, context.Canceled)
 	require.Nil(t, page)
+	require.Equal(t, ActionNavigate, timedAction)
+	require.Positive(t, timedDuration)
 	require.Less(t, time.Since(startedAt), 2*time.Second)
 }
 

pkg/protocols/headless/request.go

@@ -79,6 +79,10 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
 	if len(request.Fuzzing) > 0 {
 		return request.executeFuzzingRule(input, vars, previous, wrappedCallback)
 	}
+	timing := newHeadlessTimingTrace(request.options.TemplateID, input.MetaInput.Input)
+	if timing != nil {
+		defer timing.emit(request.options.Logger)
+	}
 	if request.generator != nil {
 		iterator := request.generator.NewIterator()
 		for {
@@ -103,13 +107,13 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
 				return errors.Wrap(err, "could not evaluate payload helper expressions")
 			}
 
-			if err := request.executeRequestWithPayloads(input, renderedValue.Values, previous, renderedValue.InteractURLs, wrappedCallback); err != nil {
+			if err := request.executeRequestWithPayloads(input, renderedValue.Values, previous, renderedValue.InteractURLs, wrappedCallback, timing); err != nil {
 				return err
 			}
 		}
 	} else {
 		value := maps.Clone(vars)
-		if err := request.executeRequestWithPayloads(input, value, previous, interactshURLs, wrappedCallback); err != nil {
+		if err := request.executeRequestWithPayloads(input, value, previous, interactshURLs, wrappedCallback, timing); err != nil {
 			return err
 		}
 	}
@@ -132,9 +136,15 @@ func extractBaseURLFromActions(steps []*engine.Action) (string, error) {
 	return "", errors.New("no navigation action found")
 }
 
-func (request *Request) executeRequestWithPayloads(input *contextargs.Context, payloads map[string]interface{}, previous output.InternalEvent, interactshURLs []string, callback protocols.OutputEventCallback) error {
+func (request *Request) executeRequestWithPayloads(input *contextargs.Context, payloads map[string]interface{}, previous output.InternalEvent, interactshURLs []string, callback protocols.OutputEventCallback, timing *headlessTimingTrace) error {
+	if timing != nil {
+		timing.attempts++
+	}
 	pageTimeout := time.Duration(request.options.Options.PageTimeout) * time.Second
 	if _, err := url.Parse(input.MetaInput.Input); err != nil {
+		if timing != nil {
+			timing.errors++
+		}
 		request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), err)
 		request.options.Progress.IncrementFailedRequestsBy(1)
 		return errors.Wrap(err, errCouldNotGetHtmlElement)
@@ -144,35 +154,82 @@ func (request *Request) executeRequestWithPayloads(input *contextargs.Context, p
 		DisableCookie: request.DisableCookie,
 		Options:       request.options.Options,
 	}
+	var actionTimings []headlessActionTiming
+	if timing != nil {
+		options.OnActionTiming = func(action engine.ActionType, duration time.Duration) {
+			actionTimings = append(actionTimings, headlessActionTiming{action: action, duration: duration})
+		}
+	}
 
 	if !options.DisableCookie && input.CookieJar == nil {
+		if timing != nil {
+			timing.errors++
+		}
 		return errors.New("cookie reuse enabled but cookie-jar is nil")
 	}
 
 	// limited as one request per run of the steps, the unit progress counts
-	if err := request.options.RateLimitTakeContext(input.Context(), input.MetaInput.Input); err != nil {
-		request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), err)
+	var rateWaitStarted time.Time
+	if timing != nil {
+		rateWaitStarted = time.Now()
+	}
+	rateWaitErr := request.options.RateLimitTakeContext(input.Context(), input.MetaInput.Input)
+	if timing != nil {
+		timing.rateWait += time.Since(rateWaitStarted)
+	}
+	if rateWaitErr != nil {
+		if timing != nil {
+			timing.errors++
+		}
+		request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), rateWaitErr)
 		request.options.Progress.IncrementFailedRequestsBy(1)
-		return errors.Wrap(err, "could not take rate limit")
+		return errors.Wrap(rateWaitErr, "could not take rate limit")
 	}
 
+	var createStarted time.Time
+	if timing != nil {
+		createStarted = time.Now()
+	}
 	instance, err := request.options.Browser.NewInstanceWithContext(input.Context(), pageTimeout)
+	if timing != nil {
+		timing.contextCreate += time.Since(createStarted)
+	}
 	if err != nil {
+		if timing != nil {
+			timing.errors++
+		}
 		request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), err)
 		request.options.Progress.IncrementFailedRequestsBy(1)
 		return errors.Wrap(err, errCouldNotGetHtmlElement)
 	}
 	var page *engine.Page
 	defer func() {
+		var cleanupStarted time.Time
+		if timing != nil {
+			cleanupStarted = time.Now()
+		}
 		if closeErr := instance.CloseWithPage(page); closeErr != nil {
 			gologger.Warning().Msgf("Could not close headless browser instance: %s", closeErr)
 		}
+		if timing != nil {
+			timing.cleanup += time.Since(cleanupStarted)
+		}
 	}()
 	instance.SetInteractsh(request.options.Interactsh)
 
 	timeStart := time.Now()
 	out, createdPage, err := instance.Run(input, request.Steps, payloads, options)
 	runDuration := time.Since(timeStart)
+	if timing != nil {
+		if err != nil {
+			timing.errors++
+			timing.recordFailedRun(actionTimings, runDuration)
+		} else if createdPage != nil {
+			timing.recordRun(request.Steps, createdPage.ActionDurations, runDuration)
+		} else {
+			timing.runOther += runDuration
+		}
+	}
 	if err != nil {
 		request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), err)
 		request.options.Progress.IncrementFailedRequestsBy(1)
@@ -294,7 +351,7 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, payloads
 		newInput := input.Clone()
 		newInput.MetaInput.Input = gr.Request.String()
 
-		if err := request.executeRequestWithPayloads(newInput, gr.DynamicValues, previous, gr.InteractURLs, callback); err != nil {
+		if err := request.executeRequestWithPayloads(newInput, gr.DynamicValues, previous, gr.InteractURLs, callback, nil); err != nil {
 			return false
 		}
 

pkg/protocols/headless/timing.go

@@ -0,0 +1,141 @@
+package headless
+
+import (
+	"crypto/sha256"
+	"encoding/binary"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
+)
+
+const (
+	headlessTimingTemplatesEnv = "NUCLEI_HEADLESS_TIMING_TEMPLATES"
+	headlessTimingSampleRate   = 50
+	headlessTimingSampleMin    = 30 * time.Second
+	headlessTimingOutlierMin   = 5 * time.Minute
+)
+
+type headlessTimingTrace struct {
+	templateID        string
+	targetFingerprint string
+	sampled           bool
+	started           time.Time
+	attempts          int
+	errors            int
+	rateWait          time.Duration
+	contextCreate     time.Duration
+	navigate          time.Duration
+	waitLoad          time.Duration
+	waitDialog        time.Duration
+	otherActions      time.Duration
+	runOther          time.Duration
+	cleanup           time.Duration
+}
+
+type headlessActionTiming struct {
+	action   engine.ActionType
+	duration time.Duration
+}
+
+func newHeadlessTimingTrace(templateID, target string) *headlessTimingTrace {
+	selectedTemplates := os.Getenv(headlessTimingTemplatesEnv)
+	if selectedTemplates == "" {
+		return nil
+	}
+	selected := false
+	for _, candidate := range strings.Split(selectedTemplates, ",") {
+		if strings.TrimSpace(candidate) == templateID {
+			selected = true
+			break
+		}
+	}
+	if !selected {
+		return nil
+	}
+	fingerprint := sha256.Sum256([]byte(target))
+	return &headlessTimingTrace{
+		templateID:        templateID,
+		targetFingerprint: fmt.Sprintf("%x", fingerprint[:8]),
+		sampled:           binary.BigEndian.Uint64(fingerprint[:8])%headlessTimingSampleRate == 0,
+		started:           time.Now(),
+	}
+}
+
+func (t *headlessTimingTrace) recordRun(steps []*engine.Action, durations []time.Duration, elapsed time.Duration) {
+	actions := time.Duration(0)
+	if len(steps) != len(durations) {
+		for _, duration := range durations {
+			actions += duration
+		}
+		t.otherActions += actions
+		if elapsed > actions {
+			t.runOther += elapsed - actions
+		}
+		return
+	}
+	for index, duration := range durations {
+		actions += duration
+		if steps[index] == nil {
+			t.otherActions += duration
+			continue
+		}
+		switch steps[index].ActionType.ActionType {
+		case engine.ActionNavigate:
+			t.navigate += duration
+		case engine.ActionWaitLoad:
+			t.waitLoad += duration
+		case engine.ActionWaitDialog:
+			t.waitDialog += duration
+		default:
+			t.otherActions += duration
+		}
+	}
+	if elapsed > actions {
+		t.runOther += elapsed - actions
+	}
+}
+
+func (t *headlessTimingTrace) recordFailedRun(actions []headlessActionTiming, elapsed time.Duration) {
+	measured := time.Duration(0)
+	for _, action := range actions {
+		measured += action.duration
+		switch action.action {
+		case engine.ActionNavigate:
+			t.navigate += action.duration
+		case engine.ActionWaitLoad:
+			t.waitLoad += action.duration
+		case engine.ActionWaitDialog:
+			t.waitDialog += action.duration
+		default:
+			t.otherActions += action.duration
+		}
+	}
+	t.runOther += max(elapsed-measured, 0)
+}
+
+func (t *headlessTimingTrace) summary(elapsed time.Duration) (string, bool) {
+	if elapsed < headlessTimingOutlierMin && (!t.sampled || elapsed < headlessTimingSampleMin) {
+		return "", false
+	}
+	measured := t.rateWait + t.contextCreate + t.navigate + t.waitLoad + t.waitDialog + t.otherActions + t.runOther + t.cleanup
+	unattributed := max(elapsed-measured, 0)
+	return fmt.Sprintf(
+		"slow headless timing template_id=%q target_fingerprint=%s attempts=%d errors=%d total_ms=%d rate_wait_ms=%d context_create_ms=%d navigate_ms=%d waitload_ms=%d waitdialog_ms=%d other_actions_ms=%d run_other_ms=%d cleanup_ms=%d unattributed_ms=%d",
+		t.templateID, t.targetFingerprint, t.attempts, t.errors, elapsed.Milliseconds(), t.rateWait.Milliseconds(),
+		t.contextCreate.Milliseconds(), t.navigate.Milliseconds(), t.waitLoad.Milliseconds(), t.waitDialog.Milliseconds(),
+		t.otherActions.Milliseconds(), t.runOther.Milliseconds(), t.cleanup.Milliseconds(), unattributed.Milliseconds(),
+	), true
+}
+
+func (t *headlessTimingTrace) emit(logger *gologger.Logger) {
+	if message, ok := t.summary(time.Since(t.started)); ok {
+		if logger == nil {
+			logger = gologger.DefaultLogger
+		}
+		logger.Info().Msg(message)
+	}
+}

pkg/protocols/headless/timing_test.go

@@ -0,0 +1,127 @@
+package headless
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/gologger/formatter"
+	"github.com/projectdiscovery/gologger/levels"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
+	"github.com/stretchr/testify/require"
+)
+
+type timingCaptureWriter struct{ buffer *bytes.Buffer }
+
+func (w timingCaptureWriter) Write(data []byte, _ levels.Level) {
+	w.buffer.Write(data)
+}
+
+func TestHeadlessTimingTraceUsesExecutionLogger(t *testing.T) {
+	previous := gologger.DefaultLogger
+	t.Cleanup(func() { gologger.DefaultLogger = previous })
+	var output bytes.Buffer
+	silentLogger := &gologger.Logger{}
+	silentLogger.SetFormatter(formatter.NewCLI(false))
+	silentLogger.SetWriter(timingCaptureWriter{buffer: &output})
+	silentLogger.SetMaxLevel(levels.LevelSilent)
+	gologger.DefaultLogger = silentLogger
+	logger := &gologger.Logger{}
+	logger.SetFormatter(formatter.NewCLI(false))
+	logger.SetWriter(timingCaptureWriter{buffer: &output})
+	logger.SetMaxLevel(levels.LevelInfo)
+	trace := &headlessTimingTrace{templateID: "headless-open-redirect", sampled: true, started: time.Now().Add(-31 * time.Second)}
+
+	trace.emit(silentLogger)
+	trace.emit(nil)
+	require.Empty(t, output.String())
+
+	trace.emit(logger)
+	require.Contains(t, output.String(), "slow headless timing template_id=\"headless-open-redirect\"")
+}
+
+func TestHeadlessTimingTraceIsTemplateOptIn(t *testing.T) {
+	t.Setenv(headlessTimingTemplatesEnv, "")
+	require.Nil(t, newHeadlessTimingTrace("headless-open-redirect", "https://example.com"))
+
+	t.Setenv(headlessTimingTemplatesEnv, "CVE-2025-8191, headless-open-redirect")
+	require.Nil(t, newHeadlessTimingTrace("other-template", "https://example.com"))
+
+	trace := newHeadlessTimingTrace("headless-open-redirect", "https://example.com")
+	require.NotNil(t, trace)
+	require.Len(t, trace.targetFingerprint, 16)
+	require.NotContains(t, trace.targetFingerprint, "example.com")
+}
+
+func TestHeadlessTimingTraceAccountsForActionsAndOverhead(t *testing.T) {
+	trace := &headlessTimingTrace{
+		templateID:        "headless-open-redirect",
+		targetFingerprint: "0123456789abcdef",
+		sampled:           true,
+		attempts:          2,
+		errors:            1,
+		rateWait:          time.Second,
+		contextCreate:     time.Second,
+		cleanup:           250 * time.Millisecond,
+	}
+	steps := []*engine.Action{
+		{ActionType: engine.ActionTypeHolder{ActionType: engine.ActionNavigate}},
+		{ActionType: engine.ActionTypeHolder{ActionType: engine.ActionWaitLoad}},
+		{ActionType: engine.ActionTypeHolder{ActionType: engine.ActionWaitDialog}},
+	}
+	trace.recordRun(steps, []time.Duration{2 * time.Second, 500 * time.Millisecond, 3 * time.Second}, 6*time.Second)
+
+	message, ok := trace.summary(30 * time.Second)
+	require.True(t, ok)
+	for _, field := range []string{
+		"attempts=2", "errors=1", "total_ms=30000", "rate_wait_ms=1000",
+		"context_create_ms=1000", "navigate_ms=2000", "waitload_ms=500",
+		"waitdialog_ms=3000", "run_other_ms=500", "cleanup_ms=250", "unattributed_ms=21750",
+	} {
+		require.True(t, strings.Contains(message, field), "missing %s in %s", field, message)
+	}
+	require.NotContains(t, message, "https://")
+}
+
+func TestHeadlessTimingTraceBoundsOutput(t *testing.T) {
+	trace := &headlessTimingTrace{sampled: false}
+	_, ok := trace.summary(4 * time.Minute)
+	require.False(t, ok)
+	_, ok = trace.summary(5 * time.Minute)
+	require.True(t, ok)
+
+	trace.sampled = true
+	_, ok = trace.summary(29 * time.Second)
+	require.False(t, ok)
+	_, ok = trace.summary(30 * time.Second)
+	require.True(t, ok)
+}
+
+func TestHeadlessTimingTraceDoesNotMisattributePartialActionDurations(t *testing.T) {
+	trace := &headlessTimingTrace{sampled: true}
+	steps := []*engine.Action{
+		{ActionType: engine.ActionTypeHolder{ActionType: engine.ActionNavigate}},
+		{ActionType: engine.ActionTypeHolder{ActionType: engine.ActionWaitLoad}},
+	}
+	trace.recordRun(steps, []time.Duration{2 * time.Second}, 3*time.Second)
+
+	require.Zero(t, trace.navigate)
+	require.Zero(t, trace.waitLoad)
+	require.Equal(t, 2*time.Second, trace.otherActions)
+	require.Equal(t, time.Second, trace.runOther)
+}
+
+func TestHeadlessTimingTraceAttributesFailedActions(t *testing.T) {
+	trace := &headlessTimingTrace{sampled: true}
+	trace.recordFailedRun([]headlessActionTiming{
+		{action: engine.ActionNavigate, duration: 2 * time.Second},
+		{action: engine.ActionWaitLoad, duration: 500 * time.Millisecond},
+	}, 3*time.Second)
+
+	require.Equal(t, 2*time.Second, trace.navigate)
+	require.Equal(t, 500*time.Millisecond, trace.waitLoad)
+	require.Zero(t, trace.otherActions)
+	require.Equal(t, 500*time.Millisecond, trace.runOther)
+}

pkg/protocols/http/hostbackoff_key_test.go

@@ -0,0 +1,98 @@
+package http
+
+import (
+	"context"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hostbackoff"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hostratelimit"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httputils"
+	"github.com/projectdiscovery/retryablehttp-go"
+	urlutil "github.com/projectdiscovery/utils/url"
+	"github.com/stretchr/testify/require"
+)
+
+func TestHostBackoffKeyKeepsHTTPSPort(t *testing.T) {
+	raw, err := retryablehttp.NewRequest(http.MethodGet, "https://example.com/secret", nil)
+	require.NoError(t, err)
+	gr := &generatedRequest{request: raw}
+
+	key := hostBackoffKey(gr, nil)
+	require.Equal(t, "example.com:443", hostratelimit.NormalizeHostPort(key))
+	// request.URL.Host drops the default port. That is what executeRequest used
+	// to observe, and it lands on port 80.
+	require.Equal(t, "example.com", raw.URL.Host)
+	require.Equal(t, "example.com:80", hostratelimit.NormalizeHostPort(raw.URL.Host))
+}
+
+func TestObserveIgnoresCanceledRequest(t *testing.T) {
+	g := hostbackoff.New(hostbackoff.Config{Step: 100 * time.Millisecond})
+	req := &Request{options: &protocols.ExecutorOptions{HostBackoff: g}}
+	target := "https://example.com/x"
+	key := hostratelimit.NormalizeHostPort(target)
+
+	req.observeHostBackoff(target, nil, context.Canceled)
+	require.Zero(t, g.Delay(key))
+
+	resp := &http.Response{StatusCode: http.StatusTooManyRequests, Header: http.Header{}}
+	req.observeHostBackoff(target, resp, context.Canceled)
+	require.Equal(t, 100*time.Millisecond, g.Delay(key))
+}
+
+func TestHostBackoffKeyKeepsIPv6OnHTTPS(t *testing.T) {
+	raw, err := retryablehttp.NewRequest(http.MethodGet, "https://[2001:db8::1]/secret", nil)
+	require.NoError(t, err)
+	key := hostratelimit.NormalizeHostPort(hostBackoffKey(&generatedRequest{request: raw}, nil))
+	// URL.Host has no port, so the old observation key took the http default.
+	require.NotEqual(t, key, hostratelimit.NormalizeHostPort(raw.URL.Host))
+	require.Contains(t, key, "443")
+}
+
+func TestHostBackoffKeyFallsBackToInput(t *testing.T) {
+	input := contextargs.NewWithInput(context.Background(), "https://example.com/from-input")
+	require.Equal(t, "https://example.com/from-input", hostBackoffKey(nil, input))
+	require.Equal(t, "example.com:443", hostratelimit.NormalizeHostPort(hostBackoffKey(nil, input)))
+}
+
+// A pipeline or unsafe request stored parsed.Host, which is empty when ParseURL
+// fails or when it clears a single-label host. The wait used the original URL.
+func TestParsedHostCanMissTheWaitKey(t *testing.T) {
+	_, err := urlutil.ParseURL("http:///no-host", true)
+	require.Error(t, err)
+
+	parsed, err := urlutil.ParseURL("http://intranet/admin", true)
+	require.NoError(t, err)
+	require.Empty(t, parsed.Host)
+	require.Equal(t, "intranet:80", hostratelimit.NormalizeHostPort("http://intranet/admin"))
+}
+
+func TestWaitHostBackoffStopsWhenHandlerTriggers(t *testing.T) {
+	g := hostbackoff.New(hostbackoff.Config{Step: 2 * time.Second, Max: 2 * time.Second})
+	req := &Request{options: &protocols.ExecutorOptions{HostBackoff: g}}
+	target := "https://example.com/x"
+	req.observeHostBackoff(target, &http.Response{StatusCode: http.StatusTooManyRequests, Header: http.Header{}}, nil)
+
+	parent, cancelParent := context.WithCancel(context.Background())
+	defer cancelParent()
+	handler := httputils.NewNonBlockingSPMHandler[struct{}](parent, 1, true)
+	waitCtx, cancelWait := stopWithHandler(parent, handler)
+	defer cancelWait()
+
+	done := make(chan error, 1)
+	go func() {
+		done <- req.waitHostBackoff(waitCtx, target)
+	}()
+	time.Sleep(20 * time.Millisecond)
+	handler.Trigger()
+
+	select {
+	case err := <-done:
+		require.ErrorIs(t, err, context.Canceled)
+	case <-time.After(500 * time.Millisecond):
+		t.Fatal("backoff wait outlived the handler")
+	}
+}

pkg/protocols/http/http.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"fmt"
 	"math"
+	"net/http"
 	"strings"
 	"time"
 
@@ -20,7 +21,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/network/networkclientpool"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types/scanstrategy"
@@ -144,6 +144,7 @@ type Request struct {
 	customHeaders     map[string]string
 	generator         *generators.PayloadGenerator // optional, only enabled when using payloads
 	rawhttpClient     *rawhttp.Client
+	replayClient      *http.Client // set when -replay-proxy is given
 	dialer            *fastdialer.Dialer
 
 	// description: |
@@ -431,6 +432,13 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
 		}
 		request.rawhttpClient = httpclientpool.GetRawHTTP(options)
 	}
+	if options.Options != nil && options.Options.ReplayProxy != "" {
+		client, err := newReplayClient(options.Options)
+		if err != nil {
+			return err
+		}
+		request.replayClient = client
+	}
 	if len(request.Matchers) > 0 || len(request.Extractors) > 0 {
 		compiled := &request.Operators
 		compiled.ExcludeMatchers = options.ExcludeMatchers
@@ -538,12 +546,8 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
 		}
 	}
 	if len(request.Payloads) > 0 {
-		// Due to a known issue (https://github.com/projectdiscovery/nuclei/issues/5015),
-		// dynamic extractors cannot be used with payloads. To address this,
-		// execution is handled by the standard engine without concurrency,
-		// achieved by setting the thread count to 0.
-
-		// this limitation will be removed once we have a better way to handle dynamic extractors with payloads
+		// Multi-request matchers and dynamic extractors depend on ordered responses
+		// across a payload iteration, so they use the standard engine.
 		hasMultipleRequests := false
 		if len(request.Raw)+len(request.Path) > 1 {
 			hasMultipleRequests = true
@@ -556,14 +560,10 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
 				break
 			}
 		}
-		if hasNamedInternalExtractor && hasMultipleRequests {
+		if hasMultipleRequests && (hasNamedInternalExtractor || request.NeedsRequestCondition()) {
 			stats.Increment(SetThreadToCountZero)
 			request.Threads = 0
 		} else {
-			// specifically for http requests high concurrency and threads will lead to memory exhaustion, hence reduce the maximum parallelism
-			if protocolstate.IsLowOnMemory() {
-				request.Threads = protocolstate.GuardThreadsOrDefault(request.Threads)
-			}
 			request.Threads = options.GetThreadsForNPayloadRequests(request.Requests(), request.Threads)
 		}
 	}
@@ -613,7 +613,7 @@ const (
 )
 
 func init() {
-	stats.NewEntry(SetThreadToCountZero, "Setting thread count to 0 for %d templates, dynamic extractors are not supported with payloads yet")
+	stats.NewEntry(SetThreadToCountZero, "Setting thread count to 0 for %d templates with payloads and dependent requests")
 }
 
 // UpdateOptions replaces this request's options with a new copy

pkg/protocols/http/httpclientpool/clientpool.go

@@ -394,21 +394,9 @@ func wrappedGet(options *types.Options, configuration *Configuration, host strin
 	transportKey := transportHash(host, disableKeepAlives, maxIdleConns, maxIdleConnsPerHost, maxConnsPerHost, responseHeaderTimeout)
 
 	createTransport := func() (http.RoundTripper, error) {
-		// Set the base TLS configuration definition
-		tlsConfig := &tls.Config{
-			Renegotiation:      tls.RenegotiateOnceAsClient,
-			InsecureSkipVerify: true,
-			MinVersion:         tls.VersionTLS10,
-			ClientSessionCache: sharedTLSSessionCache,
-		}
-
-		if options.SNI != "" {
-			tlsConfig.ServerName = options.SNI
-		}
-
-		tlsConfig, err := utils.AddConfiguredClientCertToRequest(tlsConfig, options)
+		tlsConfig, err := NewTLSConfig(options)
 		if err != nil {
-			return nil, errors.Wrap(err, "could not create client certificate")
+			return nil, err
 		}
 
 		transport := &http.Transport{
@@ -753,3 +741,26 @@ func RecordHTTPToHTTPSPortMismatch(options *types.Options, hostname string) {
 
 	tracker.RecordHTTPToHTTPSPort(hostname)
 }
+
+// NewTLSConfig returns the TLS configuration HTTP requests to scan targets use,
+// honoring -sni and client certificates.
+func NewTLSConfig(options *types.Options) (*tls.Config, error) {
+	tlsConfig := &tls.Config{
+		Renegotiation: tls.RenegotiateOnceAsClient,
+		// Scan targets present untrusted certificates. This is the same configuration the HTTP client used inline.
+		InsecureSkipVerify: true, // codeql[go/disabled-certificate-check]
+		// Keep the scan client's existing minimum so older targets still connect.
+		MinVersion:         tls.VersionTLS10, // codeql[go/insecure-tls]
+		ClientSessionCache: sharedTLSSessionCache,
+	}
+
+	if options.SNI != "" {
+		tlsConfig.ServerName = options.SNI
+	}
+
+	tlsConfig, err := utils.AddConfiguredClientCertToRequest(tlsConfig, options)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not create client certificate")
+	}
+	return tlsConfig, nil
+}

pkg/protocols/http/httputils/spm.go

@@ -58,7 +58,7 @@ func NewBlockingSPMHandler[T comparable](ctx context.Context, size int, maxResul
 		maxResults:  maxResults,
 	}
 	s.internalWg.Add(1)
-	go s.run(ctx)
+	go s.run(ctx1)
 	return s
 }
 
@@ -77,7 +77,7 @@ func NewNonBlockingSPMHandler[T comparable](ctx context.Context, maxResults int,
 		maxResults:  maxResults,
 	}
 	s.internalWg.Add(1)
-	go s.run(ctx)
+	go s.run(ctx1)
 	return s
 }
 
@@ -132,18 +132,25 @@ func (h *StopAtFirstMatchHandler[T]) run(ctx context.Context) {
 			if !ok {
 				return
 			}
-			if h.onResult != nil {
-				h.onResult(val)
-			}
-			if len(h.results) >= h.maxResults {
-				// skip or do not store the result
-				continue
-			}
-			h.results[val] = struct{}{}
+			h.recordResult(val)
 		}
 	}
 }
 
+func (h *StopAtFirstMatchHandler[T]) recordResult(val T) {
+	if h.onResult != nil {
+		h.onResult(val)
+	}
+	if len(h.results) < h.maxResults {
+		h.results[val] = struct{}{}
+	}
+}
+
+// Context is canceled when the handler stops.
+func (h *StopAtFirstMatchHandler[T]) Context() context.Context {
+	return h.ctx
+}
+
 // Done returns a channel with the context done signal when stop at first match is detected
 func (h *StopAtFirstMatchHandler[T]) Done() <-chan struct{} {
 	return h.ctx.Done()
@@ -205,6 +212,9 @@ func (h *StopAtFirstMatchHandler[T]) Wait() {
 	// after waiting it closes the error channel
 	close(h.ResultChan)
 	h.internalWg.Wait()
+	for val := range h.ResultChan {
+		h.recordResult(val)
+	}
 }
 
 // CombinedResults returns the combined results

pkg/protocols/http/httputils/spm_test.go

@@ -0,0 +1,79 @@
+package httputils
+
+import (
+	"context"
+	"testing"
+	"time"
+)
+
+func TestCancelStopsResultReceiver(t *testing.T) {
+	constructors := map[string]func(context.Context) *StopAtFirstMatchHandler[int]{
+		"blocking": func(ctx context.Context) *StopAtFirstMatchHandler[int] {
+			return NewBlockingSPMHandler[int](ctx, 1, 1, true)
+		},
+		"nonblocking": func(ctx context.Context) *StopAtFirstMatchHandler[int] {
+			return NewNonBlockingSPMHandler[int](ctx, 1, true)
+		},
+	}
+
+	for name, construct := range constructors {
+		t.Run(name, func(t *testing.T) {
+			parent, cancelParent := context.WithCancel(context.Background())
+			defer cancelParent()
+			handler := construct(parent)
+			done := make(chan struct{})
+			go func() {
+				handler.internalWg.Wait()
+				close(done)
+			}()
+
+			handler.Cancel()
+			select {
+			case <-done:
+			case <-time.After(time.Second):
+				t.Fatal("result receiver remained alive after cancellation")
+			}
+		})
+	}
+}
+
+func TestCancelPreservesQueuedResults(t *testing.T) {
+	constructors := map[string]func() *StopAtFirstMatchHandler[int]{
+		"blocking": func() *StopAtFirstMatchHandler[int] {
+			return NewBlockingSPMHandler[int](context.Background(), 1, 2, false)
+		},
+		"nonblocking": func() *StopAtFirstMatchHandler[int] {
+			return NewNonBlockingSPMHandler[int](context.Background(), 2, false)
+		},
+	}
+
+	for name, construct := range constructors {
+		t.Run(name, func(t *testing.T) {
+			for range 25 {
+				handler := construct()
+				entered := make(chan struct{})
+				release := make(chan struct{})
+				handler.SetOnResultCallback(func(value int) {
+					if value == 1 {
+						close(entered)
+						<-release
+					}
+				})
+				handler.ResultChan <- 1
+				<-entered
+				handler.ResultChan <- 2
+				handler.Cancel()
+				close(release)
+				handler.Wait()
+
+				found := make(map[int]bool)
+				for _, result := range handler.CombinedResults() {
+					found[result] = true
+				}
+				if !found[1] || !found[2] {
+					t.Fatalf("queued results lost after cancellation: %v", found)
+				}
+			}
+		})
+	}
+}

pkg/protocols/http/replay.go

@@ -0,0 +1,77 @@
+package http
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+	"net/http"
+	"net/url"
+
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
+	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	"github.com/projectdiscovery/utils/errkit"
+)
+
+// newReplayClient builds the client that re-sends findings through proxy. It
+// uses the scan's TLS configuration so a replay reaches the target the same way.
+func newReplayClient(options *types.Options) (*http.Client, error) {
+	proxyURL, err := url.Parse(options.ReplayProxy)
+	if err != nil {
+		return nil, errkit.Wrapf(err, "invalid replay proxy %q", options.ReplayProxy)
+	}
+	tlsConfig, err := httpclientpool.NewTLSConfig(options)
+	if err != nil {
+		return nil, err
+	}
+	return &http.Client{
+		Timeout: options.GetTimeouts().HttpTimeout,
+		Transport: &http.Transport{
+			Proxy:           http.ProxyURL(proxyURL),
+			TLSClientConfig: tlsConfig,
+			// keep the request as sent, without an added Accept-Encoding
+			DisableCompression: true,
+		},
+		// the proxy must record the request that matched, not where it redirects
+		CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
+	}, nil
+}
+
+// replayRequest re-sends a request that produced a finding through the replay
+// proxy, so it lands in a tool such as Burp without routing the whole scan
+// through it. The response is discarded and failures are only logged: a replay
+// must never affect the scan.
+func (request *Request) replayRequest(dumpedRequest []byte, formedURL string) {
+	if request.replayClient == nil {
+		return
+	}
+	dump := bufio.NewReader(bytes.NewReader(dumpedRequest))
+	replay, err := http.ReadRequest(dump)
+	if err != nil {
+		gologger.Debug().Msgf("[%s] could not parse request to replay: %s", request.options.TemplateID, err)
+		return
+	}
+	// unsafe raw requests are dumped as written and may carry a body without
+	// declaring its length; ReadRequest would then treat them as bodyless
+	if replay.ContentLength <= 0 && len(replay.TransferEncoding) == 0 {
+		if rest, _ := io.ReadAll(dump); len(rest) > 0 {
+			replay.Body = io.NopCloser(bytes.NewReader(rest))
+			replay.ContentLength = int64(len(rest))
+		}
+	}
+	target, err := url.Parse(formedURL)
+	if err != nil {
+		gologger.Debug().Msgf("[%s] could not parse URL to replay %q: %s", request.options.TemplateID, formedURL, err)
+		return
+	}
+	replay.RequestURI = ""
+	replay.URL = target
+
+	resp, err := request.replayClient.Do(replay)
+	if err != nil {
+		gologger.Debug().Msgf("[%s] could not replay %s through %s: %s", request.options.TemplateID, formedURL, request.options.Options.ReplayProxy, err)
+		return
+	}
+	_, _ = io.Copy(io.Discard, resp.Body)
+	_ = resp.Body.Close()
+}

pkg/protocols/http/replay_test.go

@@ -0,0 +1,359 @@
+package http
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	neturl "net/url"
+	"sync"
+	"testing"
+
+	"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
+	"github.com/projectdiscovery/nuclei/v3/pkg/authprovider"
+	"github.com/projectdiscovery/nuclei/v3/pkg/authprovider/authx"
+	"github.com/projectdiscovery/nuclei/v3/pkg/model"
+	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
+	"github.com/projectdiscovery/nuclei/v3/pkg/operators"
+	"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
+	"github.com/projectdiscovery/nuclei/v3/pkg/output"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/globalmatchers"
+	urlutil "github.com/projectdiscovery/utils/url"
+	"github.com/stretchr/testify/require"
+)
+
+var _ authprovider.AuthProvider = staticReplayAuth{}
+
+type replayedRequest struct {
+	method, url, header, body string
+}
+
+func TestReplayProxyResendsOnlyMatchedRequests(t *testing.T) {
+	target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/hit" {
+			_, _ = w.Write([]byte("match-me"))
+			return
+		}
+		_, _ = w.Write([]byte("nothing here"))
+	}))
+	defer target.Close()
+
+	// a forward proxy receives absolute-form request targets for http URLs
+	var mu sync.Mutex
+	var replayed []replayedRequest
+	proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, _ := io.ReadAll(r.Body)
+		mu.Lock()
+		replayed = append(replayed, replayedRequest{method: r.Method, url: r.URL.String(), header: r.Header.Get("X-Marker"), body: string(body)})
+		mu.Unlock()
+	}))
+	defer proxy.Close()
+
+	options := testutils.DefaultOptions.Copy()
+	options.SetExecutionID(t.Name())
+	options.ReplayProxy = proxy.URL
+	testutils.Init(options)
+	t.Cleanup(func() { testutils.Cleanup(options) })
+	templateID := "replay-proxy"
+	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   templateID,
+		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
+	})
+
+	request := &Request{
+		ID:     templateID,
+		Method: HTTPMethodTypeHolder{MethodType: HTTPPost},
+		Path:   []string{"{{BaseURL}}/hit", "{{BaseURL}}/miss"},
+		Body:   "a=1&b=2",
+		Headers: map[string]string{
+			"X-Marker": "replayed",
+		},
+		Operators: operators.Operators{
+			Matchers: []*matchers.Matcher{{
+				Type:  matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
+				Words: []string{"match-me"},
+			}},
+		},
+	}
+	require.NoError(t, request.Compile(executerOpts))
+
+	var findings int
+	input := contextargs.NewWithInput(context.Background(), target.URL)
+	err := request.ExecuteWithResults(input, output.InternalEvent{}, output.InternalEvent{}, func(event *output.InternalWrappedEvent) {
+		if event.OperatorsResult != nil && event.OperatorsResult.Matched {
+			findings++
+		}
+	})
+	require.NoError(t, err)
+	require.Equal(t, 1, findings)
+
+	mu.Lock()
+	defer mu.Unlock()
+	require.Equal(t, []replayedRequest{{method: http.MethodPost, url: target.URL + "/hit", header: "replayed", body: "a=1&b=2"}}, replayed)
+}
+
+func TestReplayProxyUnsetSendsNothing(t *testing.T) {
+	options := testutils.DefaultOptions.Copy()
+	options.SetExecutionID(t.Name())
+	testutils.Init(options)
+	t.Cleanup(func() { testutils.Cleanup(options) })
+	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ID: "replay-unset"})
+
+	request := &Request{ID: "replay-unset", Path: []string{"{{BaseURL}}/"}}
+	require.NoError(t, request.Compile(executerOpts))
+	require.Nil(t, request.replayClient)
+}
+
+func TestReplayRequestKeepsBodyWithoutContentLength(t *testing.T) {
+	var got replayedRequest
+	proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, _ := io.ReadAll(r.Body)
+		got = replayedRequest{method: r.Method, url: r.URL.String(), body: string(body)}
+	}))
+	defer proxy.Close()
+
+	options := testutils.DefaultOptions.Copy()
+	options.ReplayProxy = proxy.URL
+	client, err := newReplayClient(options)
+	require.NoError(t, err)
+	request := &Request{replayClient: client, options: &protocols.ExecutorOptions{Options: options}}
+
+	// unsafe raw requests are dumped as written, here without Content-Length
+	request.replayRequest([]byte("POST /hit HTTP/1.1\r\nHost: target.example\r\n\r\nx=1"), "http://target.example/hit")
+	require.Equal(t, replayedRequest{method: http.MethodPost, url: "http://target.example/hit", body: "x=1"}, got)
+}
+
+func TestReplayProxySendsRedirectChainOnce(t *testing.T) {
+	target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/start" {
+			w.Header().Set("Location", "/next")
+			w.WriteHeader(http.StatusFound)
+			_, _ = w.Write([]byte("match-me"))
+			return
+		}
+		_, _ = w.Write([]byte("match-me"))
+	}))
+	defer target.Close()
+
+	var mu sync.Mutex
+	var replayed []string
+	proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		mu.Lock()
+		replayed = append(replayed, r.URL.String())
+		mu.Unlock()
+	}))
+	defer proxy.Close()
+
+	options := testutils.DefaultOptions.Copy()
+	options.SetExecutionID(t.Name())
+	options.ReplayProxy = proxy.URL
+	testutils.Init(options)
+	t.Cleanup(func() { testutils.Cleanup(options) })
+	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   "replay-redirect",
+		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
+	})
+	request := &Request{
+		ID:           "replay-redirect",
+		Method:       HTTPMethodTypeHolder{MethodType: HTTPGet},
+		Path:         []string{"{{BaseURL}}/start"},
+		Redirects:    true,
+		MaxRedirects: 5,
+		Operators: operators.Operators{
+			Matchers: []*matchers.Matcher{{
+				Type:  matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
+				Words: []string{"match-me"},
+			}},
+		},
+	}
+	require.NoError(t, request.Compile(executerOpts))
+
+	input := contextargs.NewWithInput(context.Background(), target.URL)
+	require.NoError(t, request.ExecuteWithResults(input, output.InternalEvent{}, output.InternalEvent{}, func(*output.InternalWrappedEvent) {}))
+
+	mu.Lock()
+	defer mu.Unlock()
+	require.Equal(t, []string{target.URL + "/start"}, replayed)
+}
+
+func TestReplayProxyReplaysRaceRequest(t *testing.T) {
+	target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write([]byte("match-me"))
+	}))
+	defer target.Close()
+
+	var mu sync.Mutex
+	var replayed []replayedRequest
+	proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, _ := io.ReadAll(r.Body)
+		mu.Lock()
+		replayed = append(replayed, replayedRequest{method: r.Method, url: r.URL.String(), body: string(body)})
+		mu.Unlock()
+	}))
+	defer proxy.Close()
+
+	options := testutils.DefaultOptions.Copy()
+	options.SetExecutionID(t.Name())
+	options.ReplayProxy = proxy.URL
+	testutils.Init(options)
+	t.Cleanup(func() { testutils.Cleanup(options) })
+	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   "replay-race",
+		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
+	})
+	request := &Request{
+		ID:                 "replay-race",
+		Method:             HTTPMethodTypeHolder{MethodType: HTTPPost},
+		Path:               []string{"{{BaseURL}}/race"},
+		Body:               "race=1",
+		Race:               true,
+		RaceNumberRequests: 1,
+		Operators: operators.Operators{
+			Matchers: []*matchers.Matcher{{
+				Type:  matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
+				Words: []string{"match-me"},
+			}},
+		},
+	}
+	require.NoError(t, request.Compile(executerOpts))
+
+	input := contextargs.NewWithInput(context.Background(), target.URL)
+	require.NoError(t, request.ExecuteWithResults(input, output.InternalEvent{}, output.InternalEvent{}, func(*output.InternalWrappedEvent) {}))
+
+	mu.Lock()
+	defer mu.Unlock()
+	require.Equal(t, []replayedRequest{{method: http.MethodPost, url: target.URL + "/race", body: "race=1"}}, replayed)
+}
+
+func TestReplayRequestAddsNoHeaders(t *testing.T) {
+	var got http.Header
+	proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		got = r.Header.Clone()
+	}))
+	defer proxy.Close()
+
+	options := testutils.DefaultOptions.Copy()
+	options.ReplayProxy = proxy.URL
+	client, err := newReplayClient(options)
+	require.NoError(t, err)
+	request := &Request{replayClient: client, options: &protocols.ExecutorOptions{Options: options}}
+
+	request.replayRequest([]byte("GET /hit HTTP/1.1\r\nHost: target.example\r\nUser-Agent: nuclei\r\n\r\n"), "http://target.example/hit")
+	require.Empty(t, got.Get("Accept-Encoding"))
+}
+
+func TestReplayProxyReplaysGlobalMatcher(t *testing.T) {
+	target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write([]byte("match-me"))
+	}))
+	defer target.Close()
+
+	var mu sync.Mutex
+	var replayed []string
+	proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		mu.Lock()
+		replayed = append(replayed, r.URL.String())
+		mu.Unlock()
+	}))
+	defer proxy.Close()
+
+	options := testutils.DefaultOptions.Copy()
+	options.SetExecutionID(t.Name())
+	options.ReplayProxy = proxy.URL
+	testutils.Init(options)
+	t.Cleanup(func() { testutils.Cleanup(options) })
+	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   "replay-global",
+		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
+	})
+	global := &operators.Operators{Matchers: []*matchers.Matcher{{
+		Type:  matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
+		Words: []string{"match-me"},
+	}}}
+	require.NoError(t, global.Compile())
+	executerOpts.GlobalMatchers = globalmatchers.New()
+	executerOpts.GlobalMatchers.AddOperator(&globalmatchers.Item{
+		TemplateID: "global",
+		Operators:  []*operators.Operators{global},
+	})
+
+	request := &Request{
+		ID:     "replay-global",
+		Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
+		Path:   []string{"{{BaseURL}}/hit"},
+	}
+	require.NoError(t, request.Compile(executerOpts))
+
+	input := contextargs.NewWithInput(context.Background(), target.URL)
+	require.NoError(t, request.ExecuteWithResults(input, output.InternalEvent{}, output.InternalEvent{}, func(*output.InternalWrappedEvent) {}))
+
+	mu.Lock()
+	defer mu.Unlock()
+	require.Equal(t, []string{target.URL + "/hit"}, replayed)
+}
+
+func TestReplayProxyReplaysRaceRequestWithAuth(t *testing.T) {
+	target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write([]byte("match-me"))
+	}))
+	defer target.Close()
+
+	var mu sync.Mutex
+	var auth []string
+	proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		mu.Lock()
+		auth = append(auth, r.Header.Get("Authorization"))
+		mu.Unlock()
+	}))
+	defer proxy.Close()
+
+	options := testutils.DefaultOptions.Copy()
+	options.SetExecutionID(t.Name())
+	options.ReplayProxy = proxy.URL
+	testutils.Init(options)
+	t.Cleanup(func() { testutils.Cleanup(options) })
+	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   "replay-race-auth",
+		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
+	})
+	executerOpts.AuthProvider = staticReplayAuth{authx.NewBearerTokenAuthStrategy(&authx.Secret{Token: "session-token"})}
+
+	request := &Request{
+		ID:                 "replay-race-auth",
+		Method:             HTTPMethodTypeHolder{MethodType: HTTPPost},
+		Path:               []string{"{{BaseURL}}/race"},
+		Body:               "race=1",
+		Race:               true,
+		RaceNumberRequests: 1,
+		Operators: operators.Operators{
+			Matchers: []*matchers.Matcher{{
+				Type:  matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
+				Words: []string{"match-me"},
+			}},
+		},
+	}
+	require.NoError(t, request.Compile(executerOpts))
+
+	input := contextargs.NewWithInput(context.Background(), target.URL)
+	require.NoError(t, request.ExecuteWithResults(input, output.InternalEvent{}, output.InternalEvent{}, func(*output.InternalWrappedEvent) {}))
+
+	mu.Lock()
+	defer mu.Unlock()
+	require.Equal(t, []string{"Bearer session-token"}, auth)
+}
+
+type staticReplayAuth struct {
+	strategy authx.AuthStrategy
+}
+
+func (s staticReplayAuth) LookupAddr(string) []authx.AuthStrategy { return nil }
+func (s staticReplayAuth) LookupURL(*neturl.URL) []authx.AuthStrategy {
+	return []authx.AuthStrategy{s.strategy}
+}
+func (s staticReplayAuth) LookupURLX(*urlutil.URL) []authx.AuthStrategy {
+	return []authx.AuthStrategy{s.strategy}
+}
+func (s staticReplayAuth) GetTemplatePaths() []string { return nil }
+func (s staticReplayAuth) PreFetchSecrets() error     { return nil }

pkg/protocols/http/request.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/hex"
+	stderrors "errors"
 	"fmt"
 	"io"
 	"maps"
@@ -34,6 +35,8 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hostbackoff"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hostratelimit"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
@@ -48,7 +51,6 @@ import (
 	convUtil "github.com/projectdiscovery/utils/conversion"
 	"github.com/projectdiscovery/utils/errkit"
 	httpUtils "github.com/projectdiscovery/utils/http"
-	"github.com/projectdiscovery/utils/reader"
 	sliceutil "github.com/projectdiscovery/utils/slice"
 	stringsutil "github.com/projectdiscovery/utils/strings"
 	unitutils "github.com/projectdiscovery/utils/unit"
@@ -75,6 +77,91 @@ func (request *Request) Type() templateTypes.ProtocolType {
 	return templateTypes.HTTPProtocol
 }
 
+// hostBackoffKey is the identity shared by the pre-request wait and the
+// post-response observation. It keeps the scheme, or an explicit port, so
+// NormalizeHostPort does not record an https target under port 80.
+func hostBackoffKey(req *generatedRequest, input *contextargs.Context) string {
+	if req != nil {
+		if key := req.URL(); key != "" {
+			return key
+		}
+		if req.request != nil && req.request.Request != nil && req.request.Request.URL != nil {
+			return req.request.Request.URL.String()
+		}
+	}
+	if input != nil && input.MetaInput != nil {
+		return input.MetaInput.Input
+	}
+	return ""
+}
+
+// observeHostBackoff reports one request outcome to the per-host governor.
+func (request *Request) observeHostBackoff(host string, resp *http.Response, err error) {
+	if request.options == nil || request.options.HostBackoff == nil {
+		return
+	}
+	// A cancelled sibling, or a scan that is stopping, is not the host asking
+	// for less traffic. A real status on the response is still worth recording.
+	if resp == nil && err != nil && stderrors.Is(err, context.Canceled) {
+		return
+	}
+	statusCode := 0
+	var retryAfter time.Duration
+	if resp != nil {
+		statusCode = resp.StatusCode
+		retryAfter = hostbackoff.RetryAfter(resp.Header.Get("Retry-After"))
+	}
+	request.options.HostBackoff.Observe(hostratelimit.NormalizeHostPort(host), statusCode, retryAfter, err)
+}
+
+// waitHostBackoff pauses for the delay this host has already earned.
+func (request *Request) waitHostBackoff(ctx context.Context, host string) error {
+	if request.options == nil || request.options.HostBackoff == nil {
+		return nil
+	}
+	return request.options.HostBackoff.Wait(ctx, hostratelimit.NormalizeHostPort(host))
+}
+
+type httpRateLimitWaitObserver interface {
+	ObserveHTTPRateLimitWait(time.Duration)
+}
+
+type httpRequestTimingObserver interface {
+	ObserveHTTPRequest(time.Duration, int, error)
+}
+
+// rateLimitTake first honours any backoff the host earned by signalling it is
+// overloaded, so a blocking target is paced regardless of which limiter is in play.
+func (request *Request) rateLimitTake(ctx context.Context, hostname string) error {
+	if err := request.waitHostBackoff(ctx, hostname); err != nil {
+		return err
+	}
+	started := time.Now()
+	err := request.options.RateLimitTakeContext(ctx, hostname)
+	if observer, ok := request.options.Progress.(httpRateLimitWaitObserver); ok {
+		observer.ObserveHTTPRateLimitWait(time.Since(started))
+	}
+	return err
+}
+
+// stopWithHandler ends when parent ends or the handler stops, so a backoff
+// wait does not outlive stop-at-first-match.
+func stopWithHandler[T comparable](parent context.Context, handler *httputils.StopAtFirstMatchHandler[T]) (context.Context, context.CancelFunc) {
+	ctx, cancel := context.WithCancel(parent)
+	if handler == nil {
+		return ctx, cancel
+	}
+	if handler.Context().Err() != nil {
+		cancel()
+		return ctx, func() {}
+	}
+	stop := context.AfterFunc(handler.Context(), cancel)
+	return ctx, func() {
+		stop()
+		cancel()
+	}
+}
+
 // executeRaceRequest executes race condition request for a URL
 func (request *Request) executeRaceRequest(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
 	reqURL := input.MetaInput.Input
@@ -94,6 +181,11 @@ func (request *Request) executeRaceRequest(input *contextargs.Context, dynamicVa
 		return err
 	}
 	request.setCustomHeaders(requestForDump)
+	// Auth is applied again on each raced request. The dump is what replay
+	// sends, so it has to include the same headers.
+	if !request.SkipSecretFile {
+		requestForDump.ApplyAuth(request.options.AuthProvider)
+	}
 	dumpedRequest, err := dump(requestForDump, reqURL)
 	if err != nil {
 		return err
@@ -167,8 +259,20 @@ func (request *Request) executeRaceRequest(input *contextargs.Context, dynamicVa
 				// stop sending more requests condition is met
 				return
 			}
+			waitCtx, cancelWait := stopWithHandler(requestInput.Context(), spmHandler)
+			err := request.waitHostBackoff(waitCtx, hostBackoffKey(httpRequest, requestInput))
+			cancelWait()
+			if err != nil || spmHandler.FoundFirstMatch() || spmHandler.Cancelled() {
+				if err != nil {
+					select {
+					case <-spmHandler.Done():
+					case spmHandler.ResultChan <- err:
+					}
+				}
+				return
+			}
 
-			err := request.executeRequest(requestInput, httpRequest, previous, false, wrappedCallback, 0)
+			err = request.executeRequest(requestInput, httpRequest, previous, false, wrappedCallback, 0)
 			select {
 			case <-spmHandler.Done():
 				return
@@ -257,15 +361,12 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 					spmHandler.Release()
 					continue
 				}
-				// Extract hostname for per-host rate limiting (use full URL - normalization happens in rateLimitTake)
-				hostname := t.updatedInput.MetaInput.Input
-				if t.req != nil && t.req.URL() != "" {
-					hostname = t.req.URL()
-				} else if t.req != nil && t.req.request != nil && t.req.request.Request != nil && t.req.request.Request.URL != nil {
-					// Extract from request URL if available
-					hostname = t.req.request.Request.URL.String()
-				}
-				if err := request.options.RateLimitTakeContext(ctx, hostname); err != nil {
+				// Full URL, so https and an explicit port stay on the same key the observation uses.
+				// The handler context ends the wait when stop-at-first-match fires.
+				waitCtx, cancelWait := stopWithHandler(ctx, spmHandler)
+				err := request.rateLimitTake(waitCtx, hostBackoffKey(t.req, t.updatedInput))
+				cancelWait()
+				if err != nil {
 					select {
 					case <-spmHandler.Done():
 						spmHandler.Release()
@@ -284,7 +385,7 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 				}
 				hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators)
 				needsRequestEvent := hasInteractMatchers && request.NeedsRequestCondition()
-				err := request.executeRequest(t.updatedInput, t.req, make(map[string]interface{}), hasInteractMatchers, func(event *output.InternalWrappedEvent) {
+				err = request.executeRequest(t.updatedInput, t.req, make(map[string]interface{}), hasInteractMatchers, func(event *output.InternalWrappedEvent) {
 					if (t.hasInteractMarkers || needsRequestEvent) && request.options.Interactsh != nil {
 						requestData := &interactsh.RequestData{
 							MakeResultFunc: request.MakeResultEvent,
@@ -331,8 +432,12 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 		}
 
 		// resize check point - nop if there are no changes
-		if shouldFollowGlobal && spmHandler.Size() != request.options.Options.PayloadConcurrency {
-			if err := spmHandler.Resize(input.Context(), request.options.Options.PayloadConcurrency); err != nil {
+		desiredWorkers := request.options.Options.PayloadConcurrency
+		if protocolstate.IsLowOnMemory() {
+			desiredWorkers = protocolstate.GuardThreadsOrDefault(desiredWorkers)
+		}
+		if shouldFollowGlobal && spmHandler.Size() != desiredWorkers {
+			if err := spmHandler.Resize(input.Context(), desiredWorkers); err != nil {
 				close(tasks)
 				workersWg.Wait()
 				return err
@@ -505,7 +610,19 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
 				// skip if first match is found
 				return
 			}
-			err := request.executeRequest(requestInput, httpRequest, previous, false, wrappedCallback, 0)
+			waitCtx, cancelWait := stopWithHandler(requestInput.Context(), spmHandler)
+			err := request.waitHostBackoff(waitCtx, hostBackoffKey(httpRequest, requestInput))
+			cancelWait()
+			if err != nil || spmHandler.FoundFirstMatch() || spmHandler.Cancelled() {
+				if err != nil {
+					select {
+					case <-spmHandler.Done():
+					case spmHandler.ResultChan <- err:
+					}
+				}
+				return
+			}
+			err = request.executeRequest(requestInput, httpRequest, previous, false, wrappedCallback, 0)
 			select {
 			case <-spmHandler.Done():
 				return
@@ -579,20 +696,17 @@ executionLoop:
 			// a copy and using it to check for host errors etc
 			// but this should be replaced once templateCtx is refactored properly
 			updatedInput := contextargs.GetCopyIfHostOutdated(input, generatedHttpRequest.URL())
-
-			// Extract hostname for per-host rate limiting (use generated request URL - normalization happens in rateLimitTake)
-			hostname := input.MetaInput.Input
-			if generatedHttpRequest.URL() != "" {
-				// Use the generated URL directly - the normalization function will extract host:port correctly
-				hostname = generatedHttpRequest.URL()
-			}
-			if err := request.options.RateLimitTakeContext(ctx, hostname); err != nil {
-				return false, true, nil, err
-			}
-
 			if generatedHttpRequest.customCancelFunction != nil {
 				defer generatedHttpRequest.customCancelFunction()
 			}
+			if request.isUnresponsiveAddress(updatedInput) {
+				return false, true, nil, nil
+			}
+
+			// Full URL, so https and an explicit port stay on the same key the observation uses.
+			if err := request.rateLimitTake(ctx, hostBackoffKey(generatedHttpRequest, input)); err != nil {
+				return false, true, nil, err
+			}
 
 			hasInteractMarkers := interactsh.HasMarkers(data) || len(generatedHttpRequest.interactshURLs) > 0
 			if input.MetaInput.Input == "" {
@@ -887,6 +1001,25 @@ func extractorAppliesToRequest(part string, requestIndex int) bool {
 
 const drainReqSize = int64(8 * unitutils.Kilo)
 
+// reuseRequestBody reads the body once and installs GetBody copies. The
+// post-send dump reads one while the transport may still be writing another.
+// GET, HEAD, and chunked requests keep their existing content length.
+func reuseRequestBody(req *retryablehttp.Request) error {
+	body, err := io.ReadAll(req.Body)
+	if err != nil {
+		return err
+	}
+	keepLength := stringsutil.EqualFoldAny(req.Method, http.MethodGet, http.MethodHead) || req.Header.Get("Transfer-Encoding") == "chunked"
+	contentLength := req.ContentLength
+	if err = req.SetBody(body); err != nil {
+		return err
+	}
+	if keepLength {
+		req.ContentLength = contentLength
+	}
+	return nil
+}
+
 // executeRequest executes the actual generated request and returns error if occurred
 func (request *Request) executeRequest(input *contextargs.Context, generatedRequest *generatedRequest, previousEvent output.InternalEvent, hasInteractMatchers bool, processEvent protocols.OutputEventCallback, requestCount int) (err error) {
 	// Check if hosts keep erroring
@@ -903,6 +1036,10 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 		processEvent(event)
 	}
 
+	// Captured before auth, scheme correction, or the error path can rewrite
+	// the URL. The caller waited on this same key.
+	backoffKey := hostBackoffKey(generatedRequest, input)
+
 	request.setCustomHeaders(generatedRequest)
 	fuzzed := &generatedRequest.fuzzGeneratedRequest
 	cookieFuzzing := generatedRequest.request != nil && fuzzed.Component != nil && fuzzed.Component.Name() == component.RequestCookieComponent && len(fuzzed.FuzzedKeys) > 0
@@ -922,20 +1059,10 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 	// For race conditions we can't dump the request body at this point as it's already waiting the open-gate event, already handled with a similar code within the race function
 	if !generatedRequest.original.Race {
 
-		// change encoding type to content-length unless transfer-encoding header is manually set
-		if generatedRequest.request != nil && !stringsutil.EqualFoldAny(generatedRequest.request.Method, http.MethodGet, http.MethodHead) && generatedRequest.request.Body != nil && generatedRequest.request.Header.Get("Transfer-Encoding") != "chunked" {
-			var newReqBody *reader.ReusableReadCloser
-			newReqBody, ok := generatedRequest.request.Body.(*reader.ReusableReadCloser)
-			if !ok {
-				newReqBody, err = reader.NewReusableReadCloser(generatedRequest.request.Body)
-			}
-			if err == nil {
-				// update the request body with the reusable reader
-				generatedRequest.request.SetBodyReader(newReqBody)
-				// get content length
-				length, _ := io.Copy(io.Discard, newReqBody)
-				generatedRequest.request.ContentLength = length
-			} else {
+		// Buffer every body so the post-send dump reads its own copy. GET, HEAD,
+		// and chunked requests keep the content length they already had.
+		if generatedRequest.request != nil && generatedRequest.request.Body != nil {
+			if err = reuseRequestBody(generatedRequest.request); err != nil {
 				// log error and continue
 				gologger.Verbose().Msgf("[%v] Could not read request body while forcing transfer encoding: %s\n", request.options.TemplateID, err)
 				err = nil
@@ -986,6 +1113,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 
 	var formedURL string
 	var hostname string
+	var sent bool
 	timeStart := time.Now()
 	if generatedRequest.original.Pipeline {
 		// if request is a pipeline request, use the pipelined client
@@ -995,8 +1123,10 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 				hostname = parsed.Host
 			}
 			resp, err = generatedRequest.pipelinedClient.DoRaw(generatedRequest.rawRequest.Method, input.MetaInput.Input, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)))
+			sent = true
 		} else if generatedRequest.request != nil {
 			resp, err = generatedRequest.pipelinedClient.Dor(generatedRequest.request)
+			sent = true
 		}
 	} else if generatedRequest.original.Unsafe && generatedRequest.rawRequest != nil {
 		// if request is a unsafe request, use the rawhttp client
@@ -1040,6 +1170,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 			Body:    io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)),
 			Options: &options,
 		})
+		sent = true
 	} else {
 		//** For Normal requests **//
 		// Use the dial target (URL.Host) rather than the optional Host-header
@@ -1116,6 +1247,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 			}
 
 			resp, err = executingClient.Do(generatedRequest.request)
+			sent = true
 
 			// If we forced http->https from a previous detection and the corrected
 			// request failed (e.g. a false positive where the port actually speaks
@@ -1130,6 +1262,18 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 			}
 		}
 	}
+	// A project-file hit never sets sent, so a stored response cannot move the delay.
+	// Record before dump or the dialer lookup can return and drop the outcome.
+	if sent {
+		request.observeHostBackoff(backoffKey, resp, err)
+		if observer, ok := request.options.Progress.(httpRequestTimingObserver); ok {
+			statusCode := 0
+			if resp != nil {
+				statusCode = resp.StatusCode
+			}
+			observer.ObserveHTTPRequest(time.Since(timeStart), statusCode, err)
+		}
+	}
 	// use request url as matched url if empty
 	if formedURL == "" {
 		formedURL = input.MetaInput.Input
@@ -1235,6 +1379,17 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 	// we only intend to log/save the final redirected response
 	// i.e why we have to use sync.Once to ensure it's only done once
 	var errx error
+	// Race skips the dumps above because the body is already waiting on the
+	// gate. executeRaceRequest stored those bytes for DSL use.
+	if generatedRequest.original != nil && generatedRequest.original.Race && len(dumpedRequest) == 0 {
+		if raw, ok := previousEvent["request"].(string); ok {
+			dumpedRequest = []byte(raw)
+		}
+	}
+	// One executed request is replayed once, even when several responses in
+	// the redirect chain match.
+	replayed := false
+
 	onceFunc := sync.OnceFunc(func() {
 		// if nuclei-project is enabled store the response if not previously done
 		if request.options.ProjectFile != nil && !fromCache {
@@ -1385,8 +1540,10 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 			event.UsesInteractsh = true
 		}
 
+		globalMatched := false
 		if request.options.GlobalMatchers.HasMatchers() {
 			request.options.GlobalMatchers.Match(interimEvent, request.Match, request.Extract, isDebug, func(event output.InternalEvent, result *operators.Result) {
+				globalMatched = true
 				callback(eventcreator.CreateEventWithOperatorResults(request, event, result))
 			})
 		}
@@ -1397,6 +1554,12 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 
 		callback(event)
 
+		localMatched := event.OperatorsResult != nil && event.OperatorsResult.Matched
+		if !replayed && (localMatched || globalMatched) {
+			replayed = true
+			request.replayRequest(dumpedRequest, formedURL)
+		}
+
 		if request.options.FuzzStatsDB != nil && generatedRequest.fuzzGeneratedRequest.Request != nil {
 			request.options.FuzzStatsDB.RecordResultEvent(fuzzStats.FuzzingEvent{
 				URL:           input.MetaInput.Target(),

pkg/protocols/http/request_fuzz.go

@@ -190,15 +190,6 @@ func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest,
 	if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(request.options.ProtocolType.String(), input) {
 		return false, nil
 	}
-	// Extract hostname for per-host rate limiting: prefer the concrete fuzzed
-	// request URL (rules may change host/port), fall back to the input target
-	hostname := input.MetaInput.Input
-	if gr.Request != nil && gr.Request.Request != nil && gr.Request.Request.URL != nil {
-		hostname = gr.Request.Request.URL.String()
-	}
-	if err := request.options.RateLimitTakeContext(input.Context(), hostname); err != nil {
-		return false, err
-	}
 	req := &generatedRequest{
 		request:              gr.Request,
 		dynamicValues:        gr.DynamicValues,
@@ -206,6 +197,10 @@ func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest,
 		original:             request,
 		fuzzGeneratedRequest: gr,
 	}
+	// Same key executeRequest will observe, including a host or port the fuzz rule rewrote.
+	if err := request.rateLimitTake(input.Context(), hostBackoffKey(req, input)); err != nil {
+		return false, err
+	}
 	var gotMatches bool
 	requestErr := request.executeRequest(input, req, gr.DynamicValues, hasInteractMatchers, func(event *output.InternalWrappedEvent) {
 		for _, result := range event.Results {

pkg/protocols/http/request_test.go

@@ -21,9 +21,16 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors"
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
+	"github.com/projectdiscovery/nuclei/v3/pkg/progress"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hostbackoff"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/ratelimit"
+	"github.com/projectdiscovery/retryablehttp-go"
+	"github.com/projectdiscovery/utils/memguardian"
+	"io"
 )
 
 func executeDynamicHTTPTestRequest(t *testing.T, request *Request, target string) {
@@ -1294,6 +1301,291 @@ func TestExecuteParallelHTTP_StopAtFirstMatchCancelsQueuedRateLimitWaiters(t *te
 	require.Equal(t, int32(1), requests.Load())
 }
 
+func TestPayloadConcurrencyIsGuardedOnceUnderMemoryPressure(t *testing.T) {
+	options := testutils.DefaultOptions.Copy()
+	options.ExecutionId = t.Name()
+	options.RestrictLocalNetworkAccess = false
+	testutils.Init(options)
+	t.Cleanup(func() { testutils.Cleanup(options) })
+	require.NotNil(t, memguardian.DefaultMemGuardian)
+	previousWarning := memguardian.DefaultMemGuardian.Warning.Swap(true)
+	t.Cleanup(func() { memguardian.DefaultMemGuardian.Warning.Store(previousWarning) })
+
+	var active atomic.Int32
+	var maximum atomic.Int32
+	var total atomic.Int32
+	var matched atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
+		current := active.Add(1)
+		for previous := maximum.Load(); current > previous && !maximum.CompareAndSwap(previous, current); previous = maximum.Load() {
+		}
+		time.Sleep(40 * time.Millisecond)
+		active.Add(-1)
+		total.Add(1)
+		writer.WriteHeader(http.StatusOK)
+	}))
+	defer server.Close()
+
+	request := &Request{
+		ID:      "memory-guard-payload-concurrency",
+		Method:  HTTPMethodTypeHolder{MethodType: HTTPGet},
+		Path:    []string{"{{BaseURL}}/test?payload={{v}}"},
+		Threads: 20,
+		Payloads: map[string]interface{}{
+			"v": []string{"1", "2", "3", "4", "5", "6", "7", "8"},
+		},
+		Operators: operators.Operators{Matchers: []*matchers.Matcher{{
+			Type: matchers.MatcherTypeHolder{MatcherType: matchers.DSLMatcher},
+			DSL:  []string{"status_code == 200"},
+		}}},
+	}
+	executerOptions := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   request.ID,
+		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
+	})
+	executerOptions.RateLimiter = nil
+	require.NoError(t, request.Compile(executerOptions))
+	compiledThreads := request.Threads
+	require.NoError(t, request.ExecuteWithResults(
+		contextargs.NewWithInput(t.Context(), server.URL),
+		make(output.InternalEvent),
+		make(output.InternalEvent),
+		func(event *output.InternalWrappedEvent) {
+			if event.OperatorsResult != nil && event.OperatorsResult.Matched {
+				matched.Add(1)
+			}
+		},
+	))
+	require.Equal(t, int32(8), total.Load())
+	require.Equal(t, int32(8), matched.Load())
+	if compiledThreads != 20 || maximum.Load() <= 1 {
+		t.Fatalf("memory guard applied more than once: compiled threads=%d, maximum concurrent requests=%d", compiledThreads, maximum.Load())
+	}
+
+	memguardian.DefaultMemGuardian.Warning.Store(false)
+	maximum.Store(0)
+	total.Store(0)
+	matched.Store(0)
+	require.NoError(t, request.ExecuteWithResults(
+		contextargs.NewWithInput(t.Context(), server.URL),
+		make(output.InternalEvent),
+		make(output.InternalEvent),
+		func(event *output.InternalWrappedEvent) {
+			if event.OperatorsResult != nil && event.OperatorsResult.Matched {
+				matched.Add(1)
+			}
+		},
+	))
+	require.Equal(t, int32(8), total.Load())
+	require.Equal(t, int32(8), matched.Load())
+	require.Greater(t, maximum.Load(), int32(4))
+}
+
+func TestDefaultPayloadConcurrencyRespectsMemoryPressure(t *testing.T) {
+	options := testutils.DefaultOptions.Copy()
+	options.ExecutionId = t.Name()
+	options.RestrictLocalNetworkAccess = false
+	options.PayloadConcurrency = 25
+	testutils.Init(options)
+	t.Cleanup(func() { testutils.Cleanup(options) })
+	require.NotNil(t, memguardian.DefaultMemGuardian)
+	previousWarning := memguardian.DefaultMemGuardian.Warning.Swap(true)
+	t.Cleanup(func() { memguardian.DefaultMemGuardian.Warning.Store(previousWarning) })
+
+	var active atomic.Int32
+	var maximum atomic.Int32
+	var total atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
+		current := active.Add(1)
+		for previous := maximum.Load(); current > previous && !maximum.CompareAndSwap(previous, current); previous = maximum.Load() {
+		}
+		time.Sleep(40 * time.Millisecond)
+		active.Add(-1)
+		total.Add(1)
+		writer.WriteHeader(http.StatusOK)
+	}))
+	defer server.Close()
+
+	request := &Request{
+		ID:      "default-payload-concurrency-under-memory-pressure",
+		Method:  HTTPMethodTypeHolder{MethodType: HTTPGet},
+		Path:    []string{"{{BaseURL}}/test?payload={{v}}"},
+		Threads: options.PayloadConcurrency,
+		Payloads: map[string]interface{}{
+			"v": []string{"1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16"},
+		},
+		Operators: operators.Operators{Matchers: []*matchers.Matcher{{
+			Type: matchers.MatcherTypeHolder{MatcherType: matchers.DSLMatcher},
+			DSL:  []string{"status_code == 200"},
+		}}},
+	}
+	executerOptions := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   request.ID,
+		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
+	})
+	executerOptions.RateLimiter = nil
+	require.NoError(t, request.Compile(executerOptions))
+	require.NoError(t, request.ExecuteWithResults(
+		contextargs.NewWithInput(t.Context(), server.URL),
+		make(output.InternalEvent),
+		make(output.InternalEvent),
+		func(*output.InternalWrappedEvent) {},
+	))
+	require.Equal(t, int32(16), total.Load())
+	require.LessOrEqual(t, maximum.Load(), int32(protocolstate.GuardThreadsOrDefault(options.PayloadConcurrency)))
+	require.Greater(t, maximum.Load(), int32(1))
+}
+
+type timingProgressObserver struct {
+	progress.Progress
+	mu              sync.Mutex
+	requestDuration time.Duration
+	requestStatus   int
+	requestError    error
+	requestCount    int
+	rateWaitCount   int
+	rateWait        time.Duration
+}
+
+func (o *timingProgressObserver) ObserveHTTPRateLimitWait(duration time.Duration) {
+	o.mu.Lock()
+	defer o.mu.Unlock()
+	o.rateWaitCount++
+	o.rateWait = duration
+}
+
+func (o *timingProgressObserver) ObserveHTTPRequest(duration time.Duration, status int, err error) {
+	o.mu.Lock()
+	defer o.mu.Unlock()
+	o.requestDuration = duration
+	o.requestStatus = status
+	o.requestError = err
+	o.requestCount++
+}
+
+func TestHTTPExecutionObservesRequestTiming(t *testing.T) {
+	options := testutils.DefaultOptions.Copy()
+	options.ExecutionId = t.Name()
+	options.RestrictLocalNetworkAccess = false
+	testutils.Init(options)
+	t.Cleanup(func() { testutils.Cleanup(options) })
+
+	server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
+		time.Sleep(20 * time.Millisecond)
+		writer.WriteHeader(http.StatusTeapot)
+	}))
+	defer server.Close()
+
+	request := &Request{
+		ID:     "http-timing-observer",
+		Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
+		Path:   []string{"{{BaseURL}}/"},
+		Operators: operators.Operators{Matchers: []*matchers.Matcher{{
+			Type:   matchers.MatcherTypeHolder{MatcherType: matchers.StatusMatcher},
+			Status: []int{http.StatusTeapot},
+		}}},
+	}
+	executerOptions := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   request.ID,
+		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
+	})
+	observer := &timingProgressObserver{Progress: executerOptions.Progress}
+	executerOptions.Progress = observer
+	require.NoError(t, request.Compile(executerOptions))
+	require.NoError(t, request.ExecuteWithResults(
+		contextargs.NewWithInput(t.Context(), server.URL),
+		make(output.InternalEvent),
+		make(output.InternalEvent),
+		func(*output.InternalWrappedEvent) {},
+	))
+
+	observer.mu.Lock()
+	defer observer.mu.Unlock()
+	require.Equal(t, 1, observer.rateWaitCount)
+	require.Equal(t, 1, observer.requestCount)
+	require.Equal(t, http.StatusTeapot, observer.requestStatus)
+	require.NoError(t, observer.requestError)
+	require.GreaterOrEqual(t, observer.requestDuration, 20*time.Millisecond)
+}
+
+func TestHTTPRateLimitWaitExcludesHostBackoff(t *testing.T) {
+	backoff := hostbackoff.New(hostbackoff.Config{Step: 200 * time.Millisecond, Max: 200 * time.Millisecond})
+	observer := &timingProgressObserver{}
+	request := &Request{options: &protocols.ExecutorOptions{HostBackoff: backoff, Progress: observer}}
+	target := "https://example.com/path"
+	request.observeHostBackoff(target, &http.Response{StatusCode: http.StatusTooManyRequests, Header: http.Header{}}, nil)
+
+	started := time.Now()
+	require.NoError(t, request.rateLimitTake(t.Context(), target))
+	elapsed := time.Since(started)
+
+	observer.mu.Lock()
+	defer observer.mu.Unlock()
+	require.Equal(t, 1, observer.rateWaitCount)
+	require.GreaterOrEqual(t, elapsed, 150*time.Millisecond)
+	require.Less(t, observer.rateWait, elapsed/2)
+}
+
+func TestHTTPRateLimitWaitNotObservedWhenBackoffCanceled(t *testing.T) {
+	backoff := hostbackoff.New(hostbackoff.Config{Step: time.Second, Max: time.Second})
+	observer := &timingProgressObserver{}
+	request := &Request{options: &protocols.ExecutorOptions{HostBackoff: backoff, Progress: observer}}
+	target := "https://example.com/path"
+	request.observeHostBackoff(target, &http.Response{StatusCode: http.StatusTooManyRequests, Header: http.Header{}}, nil)
+	ctx, cancel := context.WithCancel(t.Context())
+	cancel()
+
+	require.ErrorIs(t, request.rateLimitTake(ctx, target), context.Canceled)
+	observer.mu.Lock()
+	defer observer.mu.Unlock()
+	require.Zero(t, observer.rateWaitCount)
+}
+
+func TestSerialHTTPUnresponsiveHostSkipsBeforeRateLimit(t *testing.T) {
+	options := testutils.DefaultOptions.Copy()
+	options.ExecutionId = t.Name()
+	options.RateLimit = 1
+	options.RateLimitDuration = 2 * time.Second
+	options.RestrictLocalNetworkAccess = false
+	testutils.Init(options)
+	t.Cleanup(func() { testutils.Cleanup(options) })
+
+	var requests atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
+		requests.Add(1)
+		writer.WriteHeader(http.StatusOK)
+	}))
+	defer server.Close()
+
+	request := &Request{
+		ID:     "serial-skip-before-rate-limit",
+		Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
+		Path:   []string{"{{BaseURL}}/"},
+	}
+	executerOptions := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   request.ID,
+		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
+	})
+	executerOptions.HostErrorsCache = &fakeHostErrorsCache{}
+	observer := &timingProgressObserver{Progress: executerOptions.Progress}
+	executerOptions.Progress = observer
+	executerOptions.RateLimiter = ratelimit.New(context.Background(), 1, 2*time.Second)
+	t.Cleanup(executerOptions.RateLimiter.Stop)
+	require.NoError(t, executerOptions.RateLimiter.TakeContext(t.Context()))
+	require.NoError(t, request.Compile(executerOptions))
+
+	ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
+	defer cancel()
+	started := time.Now()
+	err := request.ExecuteWithResults(contextargs.NewWithInput(ctx, server.URL), make(output.InternalEvent), make(output.InternalEvent), func(*output.InternalWrappedEvent) {})
+	require.NoError(t, err)
+	require.Less(t, time.Since(started), 250*time.Millisecond)
+	require.Zero(t, requests.Load())
+	observer.mu.Lock()
+	defer observer.mu.Unlock()
+	require.Zero(t, observer.rateWaitCount)
+}
+
 func TestExecuteParallelHTTP_SkipOnUnresponsiveFromCache(t *testing.T) {
 	options := testutils.DefaultOptions
 	testutils.Init(options)
@@ -1562,3 +1854,98 @@ func TestPerHostRateLimitHonorsRequestContextCancellation(t *testing.T) {
 	require.ErrorIs(t, err, context.Canceled)
 	require.Less(t, cancellationDelay, 250*time.Millisecond)
 }
+
+func TestDumpedRequestKeepsBodyServerDidNotRead(t *testing.T) {
+	// The server answers in full, then holds the connection without reading
+	// the body, so the transport is still writing it when the response arrives.
+	release := make(chan struct{})
+	target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Length", "2")
+		_, _ = w.Write([]byte("ok"))
+		w.(http.Flusher).Flush()
+		<-release
+	}))
+	defer target.Close()
+	defer close(release)
+
+	options := testutils.DefaultOptions.Copy()
+	options.SetExecutionID(t.Name())
+	testutils.Init(options)
+	t.Cleanup(func() { testutils.Cleanup(options) })
+	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   "dump-body",
+		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
+	})
+
+	body := strings.Repeat("A", 4<<20)
+	requests := []*Request{
+		{
+			ID:     "post",
+			Method: HTTPMethodTypeHolder{MethodType: HTTPPost},
+			Path:   []string{"{{BaseURL}}/"},
+			Body:   body,
+		},
+		{
+			ID:  "get",
+			Raw: []string{"GET / HTTP/1.1\nHost: {{Hostname}}\n\n" + body},
+		},
+		{
+			ID:  "chunked",
+			Raw: []string{"POST / HTTP/1.1\nHost: {{Hostname}}\nTransfer-Encoding: chunked\n\n" + body},
+		},
+	}
+	for _, request := range requests {
+		require.NoError(t, request.Compile(executerOpts))
+		// how much the transport has written when the response lands varies, so
+		// a few rounds make a lost body show up reliably
+		for range 3 {
+			var dumped string
+			input := contextargs.NewWithInput(context.Background(), target.URL)
+			err := request.ExecuteWithResults(input, output.InternalEvent{}, output.InternalEvent{}, func(event *output.InternalWrappedEvent) {
+				dumped, _ = event.InternalEvent["request"].(string)
+			})
+			require.NoError(t, err)
+			require.True(t, strings.HasSuffix(dumped, "\r\n\r\n"+body), "%s recorded request lost its body: %d of %d bytes", request.ID, len(dumped), len(body))
+		}
+	}
+}
+
+func TestReuseRequestBodyKeepsExcludedEncoding(t *testing.T) {
+	body := []byte("original")
+	for _, tc := range []struct {
+		name          string
+		method        string
+		header        string
+		contentLength int64
+		wantLength    int64
+	}{
+		{name: "post", method: http.MethodPost, contentLength: 1, wantLength: int64(len(body))},
+		{name: "get", method: http.MethodGet, contentLength: 3, wantLength: 3},
+		{name: "head", method: http.MethodHead, contentLength: 0, wantLength: 0},
+		{name: "chunked", method: http.MethodPost, header: "chunked", contentLength: -1, wantLength: -1},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			req, err := retryablehttp.NewRequest(tc.method, "https://example.com/", body)
+			require.NoError(t, err)
+			req.ContentLength = tc.contentLength
+			if tc.header != "" {
+				req.Header.Set("Transfer-Encoding", tc.header)
+			}
+
+			require.NoError(t, reuseRequestBody(req))
+			require.Equal(t, tc.wantLength, req.ContentLength)
+			if tc.header != "" {
+				require.Equal(t, tc.header, req.Header.Get("Transfer-Encoding"))
+			}
+
+			got, err := io.ReadAll(req.Body)
+			require.NoError(t, err)
+			require.Equal(t, body, got)
+			fresh, err := req.GetBody()
+			require.NoError(t, err)
+			got, err = io.ReadAll(fresh)
+			require.NoError(t, err)
+			require.Equal(t, body, got)
+		})
+	}
+}

pkg/protocols/http/utils.go

@@ -13,8 +13,16 @@ import (
 // dump creates a dump of the http request in form of a byte slice
 func dump(req *generatedRequest, reqURL string) ([]byte, error) {
 	if req.request != nil {
-		// Use a clone to avoid a race condition with the http transport
-		bin, err := req.request.Clone(req.request.Context()).Dump()
+		// Use a clone to avoid a race condition with the http transport. The
+		// clone shares the body, which the transport may still be sending, so
+		// dump a fresh copy of it where one is available.
+		clone := req.request.Clone(req.request.Context())
+		if clone.GetBody != nil {
+			if body, err := clone.GetBody(); err == nil {
+				clone.Body = body
+			}
+		}
+		bin, err := clone.Dump()
 		if err != nil {
 			return nil, errkit.Wrapf(err, "could not dump request: %v", req.request.String())
 		}

pkg/protocols/protocols.go

@@ -30,6 +30,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/projectfile"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/globalmatchers"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hostbackoff"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hostratelimit"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
@@ -115,6 +116,9 @@ type ExecutorOptions struct {
 	InteractshScope *interactsh.RequestScope
 	// HostErrorsCache is an optional cache for handling host errors
 	HostErrorsCache hosterrorscache.CacheInterface
+	// HostBackoff paces a host that signals it is being overloaded, so a scan
+	// slows down for it instead of being blocked outright. Nil disables it.
+	HostBackoff *hostbackoff.Governor
 	// Stop execution once first match is found (Assigned while parsing templates)
 	// Note: this is different from Options.StopAtFirstMatch (Assigned from CLI option)
 	StopAtFirstMatch bool
@@ -378,6 +382,7 @@ func (e *ExecutorOptions) Copy() *ExecutorOptions {
 		Interactsh:                   e.Interactsh,
 		InteractshScope:              e.InteractshScope,
 		HostErrorsCache:              e.HostErrorsCache,
+		HostBackoff:                  e.HostBackoff,
 		StopAtFirstMatch:             e.StopAtFirstMatch,
 		Variables:                    e.Variables,
 		Constants:                    e.Constants,
@@ -606,6 +611,7 @@ func (e *ExecutorOptions) ApplyNewEngineOptions(n *ExecutorOptions) {
 	e.Interactsh = n.Interactsh
 	e.InteractshScope = n.InteractshScope
 	e.HostErrorsCache = n.HostErrorsCache
+	e.HostBackoff = n.HostBackoff
 	e.InputHelper = n.InputHelper
 	e.FuzzParamsFrequency = n.FuzzParamsFrequency
 	e.FuzzStatsDB = n.FuzzStatsDB

pkg/tmplexec/flow/flow_executor_test.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/julienschmidt/httprouter"
+	"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk"
 	"github.com/projectdiscovery/nuclei/v3/pkg/loader/workflow"
@@ -21,7 +22,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
 	"github.com/projectdiscovery/nuclei/v3/pkg/scan"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
-	"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
 	"github.com/projectdiscovery/ratelimit"
 	"github.com/stretchr/testify/require"
 )
@@ -287,6 +287,10 @@ func newMultiPayloadServer() *httptest.Server {
 // survive across multiple pitchfork payload iterations (generator restarts).
 func TestFlowMultiPayloadIteration(t *testing.T) {
 	setup()
+	options := executerOpts.Options
+	previousConcurrency := options.PayloadConcurrency
+	options.PayloadConcurrency = 2
+	t.Cleanup(func() { options.PayloadConcurrency = previousConcurrency })
 	ts := newMultiPayloadServer()
 	defer ts.Close()
 
@@ -295,6 +299,7 @@ func TestFlowMultiPayloadIteration(t *testing.T) {
 
 	err = tmpl.Executer.Compile()
 	require.Nil(t, err, "could not compile template")
+	require.Zero(t, tmpl.RequestsHTTP[0].Threads, "request-condition payloads require ordered execution")
 
 	input := contextargs.NewWithInput(context.Background(), ts.URL)
 	ctx := scan.NewScanContext(context.Background(), input)
@@ -303,6 +308,24 @@ func TestFlowMultiPayloadIteration(t *testing.T) {
 	require.True(t, gotresults, "expected match with flow + multi payload iteration")
 }
 
+func TestIndependentMultiPayloadRequestsRemainParallel(t *testing.T) {
+	setup()
+	options := executerOpts.Options
+	previousConcurrency := options.PayloadConcurrency
+	options.PayloadConcurrency = 2
+	t.Cleanup(func() { options.PayloadConcurrency = previousConcurrency })
+
+	tmpl, err := templates.Parse("testcases/flow-multi-payload-iteration.yaml", nil, executerOpts)
+	require.NoError(t, err)
+	for _, matcher := range tmpl.RequestsHTTP[0].Matchers {
+		matcher.Part = "body"
+	}
+	require.False(t, tmpl.RequestsHTTP[0].NeedsRequestCondition())
+
+	require.NoError(t, tmpl.Executer.Compile())
+	require.Positive(t, tmpl.RequestsHTTP[0].Threads)
+}
+
 // TestFlowRequestConditionWithPayloads is the same as above but with
 // pitchfork payloads and header-based extraction, matching the exact
 // pattern from issue #5095.

pkg/types/types.go

@@ -103,6 +103,8 @@ type Options struct {
 	ListDslSignatures bool
 	// List of HTTP(s)/SOCKS5 proxy to use (comma separated or file input)
 	Proxy goflags.StringSlice
+	// ReplayProxy re-sends the requests that produced a finding through this proxy
+	ReplayProxy string
 	// AliveProxy is the alive proxy to use
 	AliveHttpProxy string
 	// AliveSocksProxy is the alive socks proxy to use
@@ -209,6 +211,12 @@ type Options struct {
 	// PreflightPortScan enables a preflight resolve + TCP portscan and filters targets
 	// before running templates. Disabled by default.
 	PreflightPortScan bool
+	// HostBackoff slows a host down when it signals it is being overloaded
+	// (429, a run of 403s, or transport failures) instead of scanning it at full
+	// rate until it blocks the scanner. Opt-in, since it changes scan pacing.
+	HostBackoff bool
+	// HostBackoffMax caps the per-host backoff delay.
+	HostBackoffMax time.Duration
 	// MaxURLsPerPattern caps how many targets sharing one structural shape
 	// (/user/1, /user/2, ...) are scanned. Zero, the default, scans every
 	// target: dropping one means never scanning it.
@@ -621,6 +629,7 @@ func (options *Options) Copy() *Options {
 		Resume:                         options.Resume,
 		Output:                         options.Output,
 		ProxyInternal:                  options.ProxyInternal,
+		ReplayProxy:                    options.ReplayProxy,
 		ListDslSignatures:              options.ListDslSignatures,
 		Proxy:                          options.Proxy,
 		AliveHttpProxy:                 options.AliveHttpProxy,