Browser agnostic

README.md

@@ -323,10 +323,7 @@ OPTIMIZATIONS:
    -stream                          stream mode - start elaborating without sorting the input
    -ss, -scan-strategy value        strategy to use while scanning(auto/host-spray/template-spray) (default auto)
    -irt, -input-read-timeout value  timeout on input read (default 3m0s)
-   -response-cache                  reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
    -nh, -no-httpx                   disable httpx probing for non-url input
-   -stp, -strict-probe              skip templates whose target service is unreachable: HTTP/headless on hosts httpx could not confirm and network templates on closed ports (lossless, no raw-input fallback)
-   -tf, -tech-filter                opt-in: fingerprint HTTP targets and skip product/macro-tagged templates that cannot match (off by default; enables scan-scoped GET cache; fail-open; no-op if no tech-bound templates)
    -preflight-portscan              run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
    -no-stdin                        disable stdin processing
 

README_CN.md

@@ -318,10 +318,7 @@ OPTIMIZATIONS:
    -stream                          stream mode - start elaborating without sorting the input
    -ss, -scan-strategy value        strategy to use while scanning(auto/host-spray/template-spray) (default auto)
    -irt, -input-read-timeout value  timeout on input read (default 3m0s)
-   -response-cache                  reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
    -nh, -no-httpx                   disable httpx probing for non-url input
-   -stp, -strict-probe              skip templates whose target service is unreachable: HTTP/headless on hosts httpx could not confirm and network templates on closed ports (lossless, no raw-input fallback)
-   -tf, -tech-filter                opt-in: fingerprint HTTP targets and skip product/macro-tagged templates that cannot match (off by default; enables scan-scoped GET cache; fail-open; no-op if no tech-bound templates)
    -preflight-portscan              run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
    -no-stdin                        disable stdin processing
 

README_ES.md

@@ -318,10 +318,7 @@ OPTIMIZATIONS:
    -stream                          stream mode - start elaborating without sorting the input
    -ss, -scan-strategy value        strategy to use while scanning(auto/host-spray/template-spray) (default auto)
    -irt, -input-read-timeout value  timeout on input read (default 3m0s)
-   -response-cache                  reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
    -nh, -no-httpx                   disable httpx probing for non-url input
-   -stp, -strict-probe              skip templates whose target service is unreachable: HTTP/headless on hosts httpx could not confirm and network templates on closed ports (lossless, no raw-input fallback)
-   -tf, -tech-filter                opt-in: fingerprint HTTP targets and skip product/macro-tagged templates that cannot match (off by default; enables scan-scoped GET cache; fail-open; no-op if no tech-bound templates)
    -preflight-portscan              run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
    -no-stdin                        disable stdin processing
 

README_FA.md

@@ -338,10 +338,7 @@ OPTIMIZATIONS:
    -stream                          stream mode - start elaborating without sorting the input
    -ss, -scan-strategy value        strategy to use while scanning(auto/host-spray/template-spray) (default auto)
    -irt, -input-read-timeout value  timeout on input read (default 3m0s)
-   -response-cache                  reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
    -nh, -no-httpx                   disable httpx probing for non-url input
-   -stp, -strict-probe              skip templates whose target service is unreachable: HTTP/headless on hosts httpx could not confirm and network templates on closed ports (lossless, no raw-input fallback)
-   -tf, -tech-filter                opt-in: fingerprint HTTP targets and skip product/macro-tagged templates that cannot match (off by default; enables scan-scoped GET cache; fail-open; no-op if no tech-bound templates)
    -preflight-portscan              run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
    -no-stdin                        disable stdin processing
 

README_ID.md

@@ -318,10 +318,7 @@ OPTIMIZATIONS:
    -stream                          stream mode - start elaborating without sorting the input
    -ss, -scan-strategy value        strategy to use while scanning(auto/host-spray/template-spray) (default auto)
    -irt, -input-read-timeout value  timeout on input read (default 3m0s)
-   -response-cache                  reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
    -nh, -no-httpx                   disable httpx probing for non-url input
-   -stp, -strict-probe              skip templates whose target service is unreachable: HTTP/headless on hosts httpx could not confirm and network templates on closed ports (lossless, no raw-input fallback)
-   -tf, -tech-filter                opt-in: fingerprint HTTP targets and skip product/macro-tagged templates that cannot match (off by default; enables scan-scoped GET cache; fail-open; no-op if no tech-bound templates)
    -preflight-portscan              run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
    -no-stdin                        disable stdin processing
 

README_JP.md

@@ -318,10 +318,7 @@ OPTIMIZATIONS:
    -stream                          stream mode - start elaborating without sorting the input
    -ss, -scan-strategy value        strategy to use while scanning(auto/host-spray/template-spray) (default auto)
    -irt, -input-read-timeout value  timeout on input read (default 3m0s)
-   -response-cache                  reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
    -nh, -no-httpx                   disable httpx probing for non-url input
-   -stp, -strict-probe              skip templates whose target service is unreachable: HTTP/headless on hosts httpx could not confirm and network templates on closed ports (lossless, no raw-input fallback)
-   -tf, -tech-filter                opt-in: fingerprint HTTP targets and skip product/macro-tagged templates that cannot match (off by default; enables scan-scoped GET cache; fail-open; no-op if no tech-bound templates)
    -preflight-portscan              run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
    -no-stdin                        disable stdin processing
 

README_KR.md

@@ -318,10 +318,7 @@ OPTIMIZATIONS:
    -stream                          stream mode - start elaborating without sorting the input
    -ss, -scan-strategy value        strategy to use while scanning(auto/host-spray/template-spray) (default auto)
    -irt, -input-read-timeout value  timeout on input read (default 3m0s)
-   -response-cache                  reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
    -nh, -no-httpx                   disable httpx probing for non-url input
-   -stp, -strict-probe              skip templates whose target service is unreachable: HTTP/headless on hosts httpx could not confirm and network templates on closed ports (lossless, no raw-input fallback)
-   -tf, -tech-filter                opt-in: fingerprint HTTP targets and skip product/macro-tagged templates that cannot match (off by default; enables scan-scoped GET cache; fail-open; no-op if no tech-bound templates)
    -preflight-portscan              run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
    -no-stdin                        disable stdin processing
 

README_PT-BR.md

@@ -318,10 +318,7 @@ OPTIMIZATIONS:
    -stream                          stream mode - start elaborating without sorting the input
    -ss, -scan-strategy value        strategy to use while scanning(auto/host-spray/template-spray) (default auto)
    -irt, -input-read-timeout value  timeout on input read (default 3m0s)
-   -response-cache                  reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
    -nh, -no-httpx                   disable httpx probing for non-url input
-   -stp, -strict-probe              skip templates whose target service is unreachable: HTTP/headless on hosts httpx could not confirm and network templates on closed ports (lossless, no raw-input fallback)
-   -tf, -tech-filter                opt-in: fingerprint HTTP targets and skip product/macro-tagged templates that cannot match (off by default; enables scan-scoped GET cache; fail-open; no-op if no tech-bound templates)
    -preflight-portscan              run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
    -no-stdin                        disable stdin processing
 

README_TR.md

@@ -318,10 +318,7 @@ OPTIMIZATIONS:
    -stream                          stream mode - start elaborating without sorting the input
    -ss, -scan-strategy value        strategy to use while scanning(auto/host-spray/template-spray) (default auto)
    -irt, -input-read-timeout value  timeout on input read (default 3m0s)
-   -response-cache                  reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
    -nh, -no-httpx                   disable httpx probing for non-url input
-   -stp, -strict-probe              skip templates whose target service is unreachable: HTTP/headless on hosts httpx could not confirm and network templates on closed ports (lossless, no raw-input fallback)
-   -tf, -tech-filter                opt-in: fingerprint HTTP targets and skip product/macro-tagged templates that cannot match (off by default; enables scan-scoped GET cache; fail-open; no-op if no tech-bound templates)
    -preflight-portscan              run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
    -no-stdin                        disable stdin processing
 

cmd/nuclei/main.go

@@ -466,10 +466,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 			scanstrategy.TemplateSpray.String(): goflags.EnumVariable(2),
 		}),
 		flagSet.DurationVarP(&options.InputReadTimeout, "input-read-timeout", "irt", time.Duration(3*time.Minute), "timeout on input read"),
-		flagSet.BoolVar(&options.ResponseCache, "response-cache", false, "reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)"),
 		flagSet.BoolVarP(&options.DisableHTTPProbe, "no-httpx", "nh", false, "disable httpx probing for non-url input"),
-		flagSet.BoolVarP(&options.StrictProbe, "strict-probe", "stp", false, "skip templates whose target service is unreachable: HTTP/headless on hosts httpx could not confirm and network templates on closed ports (lossless, no raw-input fallback)"),
-		flagSet.BoolVarP(&options.TechFilter, "tech-filter", "tf", false, "opt-in: fingerprint HTTP targets and skip product/macro-tagged templates that cannot match (off by default; enables scan-scoped GET cache; fail-open; no-op if no tech-bound templates)"),
 		flagSet.BoolVar(&options.PreflightPortScan, "preflight-portscan", false, "run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)"),
 		flagSet.BoolVar(&options.DisableStdin, "no-stdin", false, "disable stdin processing"),
 		flagSet.BoolVar(&options.EnableHTTPCache, "http-cache", false, "enable HTTP cache (RFC 9111) for HTTP requests"),

go.mod

@@ -102,7 +102,7 @@ require (
 	github.com/projectdiscovery/tlsx v1.4.3-0.20261020052910-acb80c2c7c75
 	github.com/projectdiscovery/uncover v1.2.1
 	github.com/projectdiscovery/useragent v0.0.109
-	github.com/projectdiscovery/utils v0.11.12-0.20261025172519-64b3c221cfe5
+	github.com/projectdiscovery/utils v0.11.12-0.20261026114542-c746c2a9b8ec
 	github.com/projectdiscovery/wappalyzergo v0.3.5
 	github.com/projectdiscovery/yamldoc-go v1.0.7
 	github.com/redis/go-redis/v9 v9.19.0

go.sum

@@ -999,8 +999,8 @@ github.com/projectdiscovery/uncover v1.2.1 h1:8U46T/96CLT7BPoXBgkTvWqB06lOyeTSLv
 github.com/projectdiscovery/uncover v1.2.1/go.mod h1:0p8onrWxfpXQEYs90ZDzTSpu1107gWmodX1NWqu/+z4=
 github.com/projectdiscovery/useragent v0.0.109 h1:b86BPdZvmgbJousa04jAugDRY4Y+rAvEamE0TFCPmeE=
 github.com/projectdiscovery/useragent v0.0.109/go.mod h1:sRyBqDAcD31RKYrO8fyXGowLd36fNwY1g0MNW9mp7zk=
-github.com/projectdiscovery/utils v0.11.12-0.20261025172519-64b3c221cfe5 h1:AOspf7ScbwZr8A2YnjCFd4ZJAMSFqzfEaIJIbJPuu10=
-github.com/projectdiscovery/utils v0.11.12-0.20261025172519-64b3c221cfe5/go.mod h1:TJgsyVjB02CavwZVFzUzPg6YOSJXQxLhL808JCiQKlU=
+github.com/projectdiscovery/utils v0.11.12-0.20261026114542-c746c2a9b8ec h1:yoAPKM61ofYFGaUUSqTCW4J0B4mHJoTg5FDv0ZrCDb4=
+github.com/projectdiscovery/utils v0.11.12-0.20261026114542-c746c2a9b8ec/go.mod h1:TJgsyVjB02CavwZVFzUzPg6YOSJXQxLhL808JCiQKlU=
 github.com/projectdiscovery/wappalyzergo v0.3.5 h1:UZradYF1GIcruSIV5wwo/Coph9/VnM17oajLD9eXR2U=
 github.com/projectdiscovery/wappalyzergo v0.3.5/go.mod h1:4E6+fh/ckaY3wo5tlM7lNYSkhuTYRHzu2YmGIRraXP0=
 github.com/projectdiscovery/yamldoc-go v1.0.7 h1:Odmjd1o4EkMff2F6QHi1dIYPIQPZUFy24YqmQCprmTk=

internal/runner/host_group.go

@@ -1,445 +0,0 @@
-package runner
-
-import (
-	"fmt"
-	"sort"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/projectdiscovery/httpx/common/httpx"
-	"github.com/projectdiscovery/nuclei/v3/pkg/input"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
-	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
-	"github.com/projectdiscovery/nuclei/v3/pkg/utils"
-	syncutil "github.com/projectdiscovery/utils/sync"
-)
-
-const hostGroupProbeWorkers = 100
-
-// hostGroup is a set of targets that share the same open-port / HTTP signature.
-// Used for savings estimates / logging; execution itself is a single spray pass
-// with a per-(template,target) filter derived from the same probe data.
-type hostGroup struct {
-	key          string
-	hosts        []*contextargs.MetaInput
-	openPorts    map[string]struct{}
-	httpOK       bool
-	explicitPort string // operator host:port; network dials this, not template defaults
-}
-
-type hostProbeResult struct {
-	input        *contextargs.MetaInput
-	openPorts    []string
-	httpOK       bool
-	explicitPort string
-}
-
-// hostReachabilityIndex maps each input to its probed reachability so the
-// engine can skip impossible template×target pairs in a single Execute pass.
-type hostReachabilityIndex struct {
-	byInput map[string]hostReachability
-}
-
-type hostReachability struct {
-	openPorts map[string]struct{}
-	httpOK    bool
-	// explicitPort is the operator-specified port from host:port input (if any).
-	// Network execution keeps this port (UseNetworkPort does not replace bare
-	// host:port), so reachability must not require the template's default port.
-	explicitPort string
-}
-
-// Allow reports whether template may produce a finding on input (lossless).
-func (idx *hostReachabilityIndex) Allow(t *templates.Template, mi *contextargs.MetaInput) bool {
-	if idx == nil || t == nil || mi == nil {
-		return true
-	}
-	h, ok := idx.byInput[mi.Input]
-	if !ok {
-		return true
-	}
-	return templateAllowedOnHost(t, h.httpOK, h.openPorts, h.explicitPort)
-}
-
-func templateAllowedOnHost(t *templates.Template, httpOK bool, openPorts map[string]struct{}, explicitPort string) bool {
-	if t.SelfContained || len(t.Workflows) > 0 || isUniversalTemplate(t) {
-		return true
-	}
-	if isWebTemplate(t) {
-		return httpOK
-	}
-	ports, ok := tcpNetworkOnlyPorts(t)
-	if !ok {
-		return true
-	}
-	// host:port inputs: dial the operator-chosen port at execution time, not the
-	// template default (see contextargs.UseNetworkPort). If that port is
-	// reachable, network templates may run against it.
-	if explicitPort != "" {
-		if _, open := openPorts[explicitPort]; open {
-			return true
-		}
-		return false
-	}
-	for _, p := range ports {
-		if _, open := openPorts[p]; open {
-			return true
-		}
-	}
-	return false
-}
-
-// estimateTemplateRequests mirrors core.getRequestCount for scheduling math.
-func estimateTemplateRequests(tpls []*templates.Template) int {
-	count := 0
-	for _, t := range tpls {
-		if len(t.Workflows) > 0 {
-			continue
-		}
-		if t.TotalRequests > 0 {
-			count += t.TotalRequests
-			continue
-		}
-		count++
-	}
-	return count
-}
-
-func hostGroupKey(openPorts []string, httpOK bool) string {
-	ports := append([]string(nil), openPorts...)
-	sort.Strings(ports)
-	http := "nohttp"
-	if httpOK {
-		http = "http"
-	}
-	if len(ports) == 0 {
-		return http + "|-"
-	}
-	return http + "|" + strings.Join(ports, ",")
-}
-
-func isWebTemplate(t *templates.Template) bool {
-	return len(t.RequestsHTTP)+len(t.RequestsWithHTTP)+len(t.RequestsHeadless)+len(t.RequestsWebsocket) > 0 &&
-		len(t.RequestsNetwork)+len(t.RequestsWithTCP)+len(t.RequestsDNS)+len(t.RequestsFile)+
-			len(t.RequestsSSL)+len(t.RequestsCode)+len(t.RequestsJavascript)+len(t.Workflows) == 0
-}
-
-func isUniversalTemplate(t *templates.Template) bool {
-	if t.SelfContained || len(t.Workflows) > 0 {
-		return true
-	}
-	if _, ok := tcpNetworkOnlyPorts(t); ok {
-		return false
-	}
-	if isWebTemplate(t) {
-		return false
-	}
-	return true
-}
-
-// templatesForHostGroup returns templates that can produce findings on this group.
-func templatesForHostGroup(all []*templates.Template, g hostGroup) []*templates.Template {
-	out := make([]*templates.Template, 0, len(all))
-	for _, t := range all {
-		if t.SelfContained || len(t.Workflows) > 0 || isUniversalTemplate(t) {
-			continue
-		}
-		if templateAllowedOnHost(t, g.httpOK, g.openPorts, g.explicitPort) {
-			out = append(out, t)
-		}
-	}
-	return out
-}
-
-func partitionTemplates(all []*templates.Template) (web, network, universal, selfContained []*templates.Template) {
-	for _, t := range all {
-		switch {
-		case t.SelfContained:
-			selfContained = append(selfContained, t)
-		case len(t.Workflows) > 0:
-			universal = append(universal, t)
-		case isWebTemplate(t):
-			web = append(web, t)
-		case isUniversalTemplate(t):
-			universal = append(universal, t)
-		default:
-			network = append(network, t)
-		}
-	}
-	return
-}
-
-// buildHostReachability probes each target once, then returns an index for
-// single-pass filtering plus host groups for logging/estimates.
-//
-// HTTP reachability prefers the already-populated InputHelper.InputsHTTP map
-// (from initializeTemplatesHTTPInput) so we do not re-run httpx.
-// When probePorts is false, only explicit input ports are recorded (HTTP-only mode).
-func (r *Runner) buildHostReachability(tpls []*templates.Template, httpHelper *input.Helper, probePorts bool) (*hostReachabilityIndex, []hostGroup, error) {
-	if r.inputProvider == nil {
-		return nil, nil, fmt.Errorf("no input provider")
-	}
-
-	var portsToProbe []string
-	if probePorts {
-		// Only concrete network-template ports. Do not pull 80/443 from HTTP
-		// templates via portsPopularityFromTemplates — httpOK already comes from
-		// scheme / InputsHTTP, and spraying web ports across every host inflates
-		// probe cost and flakes under parallel dial load.
-		seen := map[string]struct{}{}
-		for _, t := range tpls {
-			ps, ok := tcpNetworkOnlyPorts(t)
-			if !ok {
-				continue
-			}
-			for _, p := range ps {
-				if _, dup := seen[p]; dup {
-					continue
-				}
-				seen[p] = struct{}{}
-				portsToProbe = append(portsToProbe, p)
-			}
-		}
-		sort.Strings(portsToProbe)
-	}
-
-	var targets []*contextargs.MetaInput
-	r.inputProvider.Iterate(func(mi *contextargs.MetaInput) bool {
-		targets = append(targets, mi.Clone())
-		return true
-	})
-	if len(targets) == 0 {
-		return &hostReachabilityIndex{byInput: map[string]hostReachability{}}, nil, nil
-	}
-
-	// Only build an httpx client if we still need live HTTP probes.
-	needLiveHTTPProbe := false
-	for _, mi := range targets {
-		if strings.HasPrefix(mi.Input, "http://") || strings.HasPrefix(mi.Input, "https://") {
-			continue
-		}
-		if httpHelper != nil && httpHelper.InputsHTTPProbed {
-			continue // already probed (hit or miss) during initializeTemplatesHTTPInput
-		}
-		needLiveHTTPProbe = true
-		break
-	}
-
-	var httpClient *httpx.HTTPX
-	if needLiveHTTPProbe {
-		dialers := protocolstate.GetDialersWithId(r.options.ExecutionId)
-		if dialers != nil {
-			httpxOptions := httpx.DefaultOptions
-			if r.options.AliveHttpProxy != "" {
-				httpxOptions.Proxy = r.options.AliveHttpProxy
-			} else if r.options.AliveSocksProxy != "" {
-				httpxOptions.Proxy = r.options.AliveSocksProxy
-			}
-			httpxOptions.RetryMax = r.options.Retries
-			if r.options.Timeout > 0 {
-				httpxOptions.Timeout = time.Duration(r.options.Timeout) * time.Second
-			}
-			httpxOptions.NetworkPolicy = dialers.NetworkPolicy
-			if c, err := httpx.New(&httpxOptions); err == nil {
-				httpClient = c
-			}
-		}
-	}
-
-	workers := hostGroupProbeWorkers
-	// Fan out one task per host:port so closed-port timeouts do not serialize per host.
-	type probeJob struct {
-		hostIdx int
-		host    string
-		port    string
-	}
-	var jobs []probeJob
-	hostMeta := make([]struct {
-		mi           *contextargs.MetaInput
-		host         string
-		explicitPort string
-	}, len(targets))
-
-	for i, mi := range targets {
-		host, explicitPort := hostAndExplicitPort(mi.Target())
-		if host == "" {
-			host, explicitPort = hostAndExplicitPort(mi.Input)
-		}
-		hostMeta[i].mi = mi
-		hostMeta[i].host = host
-		hostMeta[i].explicitPort = explicitPort
-		if !probePorts {
-			continue
-		}
-		// Always probe the operator-specified port (may differ from template
-		// defaults). Do not spray unrelated template ports onto host:port inputs.
-		if explicitPort != "" && isNumericPort(explicitPort) {
-			jobs = append(jobs, probeJob{hostIdx: i, host: host, port: explicitPort})
-			continue
-		}
-		for _, p := range portsToProbe {
-			jobs = append(jobs, probeJob{hostIdx: i, host: host, port: p})
-		}
-	}
-
-	if len(jobs) < workers {
-		workers = len(jobs)
-	}
-	if workers < 1 {
-		workers = 1
-	}
-
-	openByHost := make([]map[string]struct{}, len(targets))
-	for i := range openByHost {
-		openByHost[i] = map[string]struct{}{}
-		// Only assume explicit ports are open when we are not probing them.
-		if !probePorts {
-			if ep := hostMeta[i].explicitPort; ep != "" && isNumericPort(ep) {
-				openByHost[i][ep] = struct{}{}
-			}
-		}
-	}
-
-	if len(jobs) > 0 {
-		swg, err := syncutil.New(syncutil.WithSize(workers))
-		if err != nil {
-			return nil, nil, err
-		}
-		var mu sync.Mutex
-		for _, job := range jobs {
-			swg.Add()
-			go func(job probeJob) {
-				defer swg.Done()
-				switch r.probe(job.host, job.port) {
-				case portOpen, portUnknown:
-					mu.Lock()
-					openByHost[job.hostIdx][job.port] = struct{}{}
-					mu.Unlock()
-				}
-			}(job)
-		}
-		swg.Wait()
-	}
-
-	results := make([]hostProbeResult, len(targets))
-	for i, meta := range hostMeta {
-		open := openByHost[i]
-		ports := make([]string, 0, len(open))
-		for p := range open {
-			ports = append(ports, p)
-		}
-		sort.Strings(ports)
-		results[i] = hostProbeResult{
-			input:        meta.mi,
-			openPorts:    ports,
-			httpOK:       resolveHTTPReachability(meta.mi, open, httpHelper, httpClient),
-			explicitPort: meta.explicitPort,
-		}
-	}
-
-	idx := &hostReachabilityIndex{byInput: make(map[string]hostReachability, len(results))}
-	grouped := map[string]*hostGroup{}
-	order := make([]string, 0)
-	for _, res := range results {
-		if res.input == nil {
-			continue
-		}
-		portsSet := make(map[string]struct{}, len(res.openPorts))
-		for _, p := range res.openPorts {
-			portsSet[p] = struct{}{}
-		}
-		idx.byInput[res.input.Input] = hostReachability{
-			openPorts:    portsSet,
-			httpOK:       res.httpOK,
-			explicitPort: res.explicitPort,
-		}
-
-		key := hostGroupKey(res.openPorts, res.httpOK)
-		if res.explicitPort != "" {
-			// Keep explicit host:port inputs in their own estimate bucket so
-			// network-template fan-out is not attributed to bare hosts.
-			key = key + "|ep:" + res.explicitPort
-		}
-		g, ok := grouped[key]
-		if !ok {
-			g = &hostGroup{
-				key:          key,
-				openPorts:    portsSet,
-				httpOK:       res.httpOK,
-				explicitPort: res.explicitPort,
-			}
-			grouped[key] = g
-			order = append(order, key)
-		}
-		g.hosts = append(g.hosts, res.input)
-	}
-
-	out := make([]hostGroup, 0, len(order))
-	for _, key := range order {
-		out = append(out, *grouped[key])
-	}
-	return idx, out, nil
-}
-
-func resolveHTTPReachability(mi *contextargs.MetaInput, open map[string]struct{}, helper *input.Helper, client *httpx.HTTPX) bool {
-	if mi == nil {
-		return false
-	}
-	if strings.HasPrefix(mi.Input, "http://") || strings.HasPrefix(mi.Input, "https://") {
-		return true
-	}
-	// Reuse httpx results from initializeTemplatesHTTPInput when available.
-	if helper != nil && helper.InputsHTTP != nil {
-		if probed, ok := helper.InputsHTTP.Get(mi.Input); ok && len(probed) > 0 {
-			return true
-		}
-		// Empty/skipped probe map (e.g. MultiFormat) must not force non-HTTP.
-		if !helper.InputsHTTPProbed {
-			// fall through to live probe / open ports
-		} else {
-			// Map was built for all inputs: absence means not HTTP.
-			return false
-		}
-	}
-	if client != nil {
-		return utils.ProbeURL(mi.Input, client) != ""
-	}
-	if _, ok := open["80"]; ok {
-		return true
-	}
-	if _, ok := open["443"]; ok {
-		return true
-	}
-	return false
-}
-
-// estimateGroupedExecutions returns baseline and filtered template×host counts.
-func estimateGroupedExecutions(all []*templates.Template, groups []hostGroup, hostCount int) (baseline, filtered int) {
-	baseline = estimateTemplateRequests(all) * hostCount
-	_, _, universal, selfContained := partitionTemplates(all)
-	once := append([]*templates.Template{}, selfContained...)
-	once = append(once, universal...)
-	filtered = estimateTemplateRequests(once) * hostCount
-	for _, g := range groups {
-		filtered += estimateTemplateRequests(templatesForHostGroup(all, g)) * len(g.hosts)
-	}
-	return baseline, filtered
-}
-
-// countReachabilityStats returns concrete network template count and distinct numeric ports to probe.
-func countReachabilityStats(tpls []*templates.Template) (concreteNetwork int, portsToProbe int) {
-	ports := map[string]struct{}{}
-	for _, t := range tpls {
-		ps, ok := tcpNetworkOnlyPorts(t)
-		if !ok {
-			continue
-		}
-		concreteNetwork++
-		for _, p := range ps {
-			ports[p] = struct{}{}
-		}
-	}
-	return concreteNetwork, len(ports)
-}

internal/runner/host_group_test.go

@@ -1,217 +0,0 @@
-package runner
-
-import (
-	"testing"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/input"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/network"
-	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
-	"github.com/stretchr/testify/require"
-)
-
-func TestHostGroupKey(t *testing.T) {
-	// Lexicographic sort: "443" < "80".
-	require.Equal(t, "http|443,80", hostGroupKey([]string{"443", "80"}, true))
-	require.Equal(t, "nohttp|22", hostGroupKey([]string{"22"}, false))
-	require.Equal(t, "nohttp|-", hostGroupKey(nil, false))
-}
-
-func TestTemplatesForHostGroup(t *testing.T) {
-	web := &templates.Template{
-		ID:           "web",
-		RequestsHTTP: []*http.Request{{}},
-	}
-	ftp := &templates.Template{
-		ID: "ftp",
-		RequestsNetwork: []*network.Request{{
-			Port: "21",
-		}},
-	}
-	ssh := &templates.Template{
-		ID: "ssh",
-		RequestsNetwork: []*network.Request{{
-			Port: "22",
-		}},
-	}
-	all := []*templates.Template{web, ftp, ssh}
-
-	webGroup := hostGroup{
-		key:       "http|80,443",
-		httpOK:    true,
-		openPorts: map[string]struct{}{"80": {}, "443": {}},
-	}
-	got := templatesForHostGroup(all, webGroup)
-	require.Len(t, got, 1)
-	require.Equal(t, "web", got[0].ID)
-
-	ftpGroup := hostGroup{
-		key:       "nohttp|21",
-		httpOK:    false,
-		openPorts: map[string]struct{}{"21": {}},
-	}
-	got = templatesForHostGroup(all, ftpGroup)
-	require.Len(t, got, 1)
-	require.Equal(t, "ftp", got[0].ID)
-
-	sshGroup := hostGroup{
-		key:       "nohttp|22",
-		httpOK:    false,
-		openPorts: map[string]struct{}{"22": {}},
-	}
-	got = templatesForHostGroup(all, sshGroup)
-	require.Len(t, got, 1)
-	require.Equal(t, "ssh", got[0].ID)
-}
-
-func TestPartitionTemplates(t *testing.T) {
-	web := &templates.Template{ID: "web", RequestsHTTP: []*http.Request{{}}}
-	ftp := &templates.Template{ID: "ftp", RequestsNetwork: []*network.Request{{Port: "21"}}}
-	self := &templates.Template{ID: "self", SelfContained: true}
-	dynNet := &templates.Template{
-		ID:              "dyn",
-		RequestsNetwork: []*network.Request{{Port: ""}},
-	}
-
-	w, n, u, s := partitionTemplates([]*templates.Template{web, ftp, self, dynNet})
-	require.Len(t, w, 1)
-	require.Equal(t, "web", w[0].ID)
-	require.Len(t, n, 1)
-	require.Equal(t, "ftp", n[0].ID)
-	require.Len(t, u, 1)
-	require.Equal(t, "dyn", u[0].ID)
-	require.Len(t, s, 1)
-	require.Equal(t, "self", s[0].ID)
-}
-
-func TestEstimateGroupedVsBaseline(t *testing.T) {
-	web := &templates.Template{ID: "web", RequestsHTTP: []*http.Request{{}}, TotalRequests: 1}
-	ftp := &templates.Template{ID: "ftp", RequestsNetwork: []*network.Request{{Port: "21"}}, TotalRequests: 1}
-	ssh := &templates.Template{ID: "ssh", RequestsNetwork: []*network.Request{{Port: "22"}}, TotalRequests: 1}
-	all := []*templates.Template{web, ftp, ssh}
-
-	groups := []hostGroup{
-		{key: "http|80", httpOK: true, openPorts: map[string]struct{}{"80": {}}, hosts: metaInputs("h1", "h2")},
-		{key: "nohttp|21", httpOK: false, openPorts: map[string]struct{}{"21": {}}, hosts: metaInputs("h3")},
-		{key: "nohttp|22", httpOK: false, openPorts: map[string]struct{}{"22": {}}, hosts: metaInputs("h4")},
-	}
-
-	baseline, filtered := estimateGroupedExecutions(all, groups, 4)
-	require.Equal(t, 12, baseline)
-	// web*2 + ftp*1 + ssh*1 = 4
-	require.Equal(t, 4, filtered)
-	require.Equal(t, 8, baseline-filtered)
-}
-
-func TestHostReachabilityAllow(t *testing.T) {
-	web := &templates.Template{ID: "web", RequestsHTTP: []*http.Request{{}}}
-	ftp := &templates.Template{ID: "ftp", RequestsNetwork: []*network.Request{{Port: "21"}}}
-	redis := &templates.Template{ID: "redis", RequestsNetwork: []*network.Request{{Port: "6379"}}}
-	idx := &hostReachabilityIndex{byInput: map[string]hostReachability{
-		"h1":              {httpOK: true, openPorts: map[string]struct{}{"80": {}}},
-		"h2":              {httpOK: false, openPorts: map[string]struct{}{"21": {}}},
-		"127.0.0.1:19637": {httpOK: false, openPorts: map[string]struct{}{"19637": {}}, explicitPort: "19637"},
-	}}
-	require.True(t, idx.Allow(web, &contextargs.MetaInput{Input: "h1"}))
-	require.False(t, idx.Allow(web, &contextargs.MetaInput{Input: "h2"}))
-	require.False(t, idx.Allow(ftp, &contextargs.MetaInput{Input: "h1"}))
-	require.True(t, idx.Allow(ftp, &contextargs.MetaInput{Input: "h2"}))
-	require.True(t, idx.Allow(web, &contextargs.MetaInput{Input: "unknown"})) // lossless
-	// Explicit host:port must allow network templates even when template default
-	// port (6379) differs from the operator-specified port (19637).
-	require.True(t, idx.Allow(redis, &contextargs.MetaInput{Input: "127.0.0.1:19637"}))
-	require.False(t, idx.Allow(web, &contextargs.MetaInput{Input: "127.0.0.1:19637"}))
-}
-
-func TestTemplateAllowedOnHostUniversal(t *testing.T) {
-	dnsish := &templates.Template{ID: "dns", RequestsDNS: nil} // no concrete network ports
-	// force universal via empty network dynamic port
-	dyn := &templates.Template{ID: "dyn", RequestsNetwork: []*network.Request{{Port: ""}}}
-	require.True(t, templateAllowedOnHost(dyn, false, map[string]struct{}{}, ""))
-	_ = dnsish
-	self := &templates.Template{ID: "self", SelfContained: true}
-	require.True(t, templateAllowedOnHost(self, false, nil, ""))
-}
-
-func TestTemplateAllowedOnHostExplicitPort(t *testing.T) {
-	redis := &templates.Template{ID: "redis", RequestsNetwork: []*network.Request{{Port: "6379"}}}
-	web := &templates.Template{ID: "web", RequestsHTTP: []*http.Request{{}}}
-	open := map[string]struct{}{"19637": {}}
-	require.True(t, templateAllowedOnHost(redis, false, open, "19637"))
-	require.False(t, templateAllowedOnHost(redis, false, open, "")) // bare host: need 6379
-	require.False(t, templateAllowedOnHost(web, false, open, "19637"))
-	// Closed explicit port: openPorts empty => network templates denied.
-	require.False(t, templateAllowedOnHost(redis, false, map[string]struct{}{}, "19637"))
-}
-
-func TestTemplatesForHostGroupExplicitPort(t *testing.T) {
-	redis := &templates.Template{ID: "redis", RequestsNetwork: []*network.Request{{Port: "6379"}}}
-	ftp := &templates.Template{ID: "ftp", RequestsNetwork: []*network.Request{{Port: "21"}}}
-	web := &templates.Template{ID: "web", RequestsHTTP: []*http.Request{{}}}
-	g := hostGroup{
-		httpOK:       false,
-		openPorts:    map[string]struct{}{"19637": {}},
-		explicitPort: "19637",
-	}
-	got := templatesForHostGroup([]*templates.Template{redis, ftp, web}, g)
-	require.Len(t, got, 2)
-	ids := map[string]bool{got[0].ID: true, got[1].ID: true}
-	require.True(t, ids["redis"] && ids["ftp"])
-}
-
-func TestCountReachabilityStats(t *testing.T) {
-	web := &templates.Template{ID: "web", RequestsHTTP: []*http.Request{{}}}
-	redis := &templates.Template{ID: "redis", RequestsNetwork: []*network.Request{{Port: "6379"}}}
-	pg := &templates.Template{ID: "pg", RequestsNetwork: []*network.Request{{Port: "5432,5432"}}}
-	dyn := &templates.Template{ID: "dyn", RequestsNetwork: []*network.Request{{Port: ""}}}
-	concrete, ports := countReachabilityStats([]*templates.Template{web, redis, pg, dyn})
-	require.Equal(t, 2, concrete)
-	require.Equal(t, 2, ports) // 6379 + 5432
-}
-
-func TestResolveHTTPReachability(t *testing.T) {
-	require.True(t, resolveHTTPReachability(&contextargs.MetaInput{Input: "http://web"}, nil, nil, nil))
-	require.True(t, resolveHTTPReachability(&contextargs.MetaInput{Input: "https://web"}, nil, nil, nil))
-	require.False(t, resolveHTTPReachability(&contextargs.MetaInput{Input: "redis1"}, map[string]struct{}{}, nil, nil))
-	require.True(t, resolveHTTPReachability(&contextargs.MetaInput{Input: "host"}, map[string]struct{}{"80": {}}, nil, nil))
-	require.True(t, resolveHTTPReachability(&contextargs.MetaInput{Input: "host"}, map[string]struct{}{"443": {}}, nil, nil))
-}
-
-func TestResolveHTTPReachabilitySkippedProbeFailOpen(t *testing.T) {
-	// MultiFormat leaves an empty InputsHTTP map without probing — must not
-	// force non-HTTP when open ports still suggest HTTP.
-	helper := &input.Helper{InputsHTTPProbed: false}
-	require.True(t, resolveHTTPReachability(&contextargs.MetaInput{Input: "host"}, map[string]struct{}{"80": {}}, helper, nil))
-}
-
-func TestTemplatesForHostGroupSkipsUniversal(t *testing.T) {
-	web := &templates.Template{ID: "web", RequestsHTTP: []*http.Request{{}}}
-	univ := &templates.Template{ID: "univ", RequestsNetwork: []*network.Request{{Port: ""}}}
-	g := hostGroup{httpOK: true, openPorts: map[string]struct{}{"80": {}}}
-	got := templatesForHostGroup([]*templates.Template{web, univ}, g)
-	require.Len(t, got, 1)
-	require.Equal(t, "web", got[0].ID)
-}
-
-func TestEstimateGroupedIncludesUniversalsOnce(t *testing.T) {
-	web := &templates.Template{ID: "web", RequestsHTTP: []*http.Request{{}}, TotalRequests: 1}
-	univ := &templates.Template{ID: "univ", RequestsNetwork: []*network.Request{{Port: ""}}, TotalRequests: 1}
-	groups := []hostGroup{
-		{httpOK: true, openPorts: map[string]struct{}{"80": {}}, hosts: metaInputs("h1")},
-	}
-	baseline, filtered := estimateGroupedExecutions([]*templates.Template{web, univ}, groups, 1)
-	require.Equal(t, 2, baseline)
-	// universal counted once for all hosts + web for http group
-	require.Equal(t, 2, filtered)
-}
-
-func metaInputs(hosts ...string) []*contextargs.MetaInput {
-	out := make([]*contextargs.MetaInput, 0, len(hosts))
-	for _, h := range hosts {
-		mi := contextargs.NewMetaInput()
-		mi.Input = h
-		out = append(out, mi)
-	}
-	return out
-}

internal/runner/plan_scope.go

@@ -1,104 +0,0 @@
-package runner
-
-import (
-	"sync"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
-	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
-)
-
-// planScope narrows the templates run on each target by the scan plan filters
-// (strict-probe reachability, tech filter), on top of the per-target profiles
-// in base. It is a protocols.TargetScope, so the engine skips, counts and
-// narrows clusters for these pairs the same way it does for profiles.
-type planScope struct {
-	base  protocols.TargetScope
-	allow func(*templates.Template, *contextargs.MetaInput) bool
-	// byPath maps a template path to its template. A path shared by several
-	// templates, or empty, is left out, so such templates are never filtered.
-	byPath map[string]*templates.Template
-
-	mu         sync.RWMutex
-	selections map[planKey]*planSelection
-}
-
-// planKey identifies one selection: the target and the profile selection it
-// narrows. Profile selections are comparable by contract.
-type planKey struct {
-	input string
-	ip    string
-	base  protocols.TemplateSelection
-}
-
-// planSelection is the selection of one target.
-type planSelection struct {
-	base  protocols.TemplateSelection
-	scope *planScope
-	input *contextargs.MetaInput
-}
-
-var _ protocols.TargetScope = (*planScope)(nil)
-
-func newPlanScope(base protocols.TargetScope, allow func(*templates.Template, *contextargs.MetaInput) bool, tpls []*templates.Template) *planScope {
-	byPath := make(map[string]*templates.Template, len(tpls))
-	shared := make(map[string]struct{})
-	for _, t := range tpls {
-		if t == nil || t.Path == "" {
-			continue
-		}
-		if _, ok := byPath[t.Path]; ok {
-			shared[t.Path] = struct{}{}
-			continue
-		}
-		byPath[t.Path] = t
-	}
-	for path := range shared {
-		delete(byPath, path)
-	}
-	return &planScope{
-		base:       base,
-		allow:      allow,
-		byPath:     byPath,
-		selections: make(map[planKey]*planSelection),
-	}
-}
-
-// For returns the selection for input. It is the same value for the same
-// input, so the engine can group targets by selection.
-func (s *planScope) For(input *contextargs.MetaInput) protocols.TemplateSelection {
-	var base protocols.TemplateSelection
-	if s.base != nil {
-		base = s.base.For(input)
-	}
-	if input == nil {
-		return base
-	}
-	key := planKey{input: input.Input, ip: input.CustomIP, base: base}
-	s.mu.RLock()
-	sel, ok := s.selections[key]
-	s.mu.RUnlock()
-	if ok {
-		return sel
-	}
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	if sel, ok := s.selections[key]; ok {
-		return sel
-	}
-	sel = &planSelection{base: base, scope: s, input: input.Clone()}
-	s.selections[key] = sel
-	return sel
-}
-
-// Allows reports whether the profiles and the plan filters select the template.
-func (p *planSelection) Allows(templatePath string) bool {
-	if p.base != nil && !p.base.Allows(templatePath) {
-		return false
-	}
-	t, ok := p.scope.byPath[templatePath]
-	if !ok {
-		return true
-	}
-	return p.scope.allow(t, p.input)
-}

internal/runner/plan_scope_test.go

@@ -1,52 +0,0 @@
-package runner
-
-import (
-	"testing"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
-	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
-	"github.com/stretchr/testify/require"
-)
-
-// profileScope selects by path, like per-target profiles do.
-type profileScope struct{ sel *profileSelection }
-
-func (p profileScope) For(*contextargs.MetaInput) protocols.TemplateSelection { return p.sel }
-
-type profileSelection struct{ allowed map[string]bool }
-
-func (p *profileSelection) Allows(path string) bool { return p.allowed[path] }
-
-func TestPlanScope(t *testing.T) {
-	web := &templates.Template{ID: "web", Path: "/t/web.yaml"}
-	net := &templates.Template{ID: "net", Path: "/t/net.yaml"}
-	dupA := &templates.Template{ID: "dup-a", Path: "/t/dup.yaml"}
-	dupB := &templates.Template{ID: "dup-b", Path: "/t/dup.yaml"}
-	// the filter drops the network template on host-a only
-	allow := func(tpl *templates.Template, mi *contextargs.MetaInput) bool {
-		return mi.Input != "host-a" || tpl.ID == "web"
-	}
-	input := func(raw string) *contextargs.MetaInput {
-		mi := contextargs.NewMetaInput()
-		mi.Input = raw
-		return mi
-	}
-	a, b := input("host-a"), input("host-b")
-
-	scope := newPlanScope(nil, allow, []*templates.Template{web, net, dupA, dupB})
-	selA := scope.For(a)
-	require.Same(t, selA, scope.For(input("host-a")), "one selection per target, so the engine can group targets")
-	require.True(t, web.SelectedBy(selA))
-	require.False(t, net.SelectedBy(selA))
-	require.True(t, dupA.SelectedBy(selA), "a path shared by several templates is never filtered")
-	require.True(t, net.SelectedBy(scope.For(b)))
-	require.True(t, (&templates.Template{Path: "/t/unknown.yaml"}).SelectedBy(selA), "unknown templates are never filtered")
-
-	// The plan narrows the profiles, never widens them.
-	profiles := profileScope{sel: &profileSelection{allowed: map[string]bool{"/t/net.yaml": true}}}
-	scope = newPlanScope(profiles, allow, []*templates.Template{web, net})
-	require.False(t, web.SelectedBy(scope.For(b)), "a template the profile leaves out stays out")
-	require.True(t, net.SelectedBy(scope.For(b)))
-	require.False(t, net.SelectedBy(scope.For(a)))
-}

internal/runner/reachability.go

@@ -1,199 +0,0 @@
-package runner
-
-import (
-	"context"
-	"net"
-	"net/url"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/projectdiscovery/httpx/common/httpx"
-	"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/network"
-	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
-	"github.com/projectdiscovery/nuclei/v3/pkg/utils"
-	sliceutil "github.com/projectdiscovery/utils/slice"
-)
-
-type probeResult int
-
-const (
-	portClosed  probeResult = iota // connection refused / host unreachable
-	portOpen                       // connect succeeded
-	portUnknown                    // timeout / filtered — treat as possibly open
-)
-
-const reachabilityProbeTimeout = 750 * time.Millisecond
-
-// strictProbeEnabled reports whether the lossless reachability prune
-// applies to this run. It is limited to standard target inputs; request-shaped
-// (DAST / multi-format) inputs are never pruned, because probing a request
-// template as a live service is meaningless and could drop coverage.
-func (r *Runner) strictProbeEnabled() bool {
-	return r.options.StrictProbe &&
-		r.inputProvider != nil &&
-		r.inputProvider.InputType() != provider.MultiFormatInputProvider
-}
-
-// noHTTPServiceReachable reports whether NO target actually speaks HTTP(S).
-//
-// When it returns true, web templates (HTTP/headless/websocket) can be excluded
-// losslessly: a port that does not serve HTTP has no HTTP application, so no
-// HTTP template can produce a real finding there. Unlike a bare TCP-open check,
-// this catches the open-but-non-HTTP case (e.g. redis on 6379): the port is
-// open, but it is not an HTTP service, so HTTP templates are pure waste.
-//
-// It reuses nuclei's own httpx probe (the same one used for input routing), so
-// the determination is consistent with the engine and respects the network
-// policy (the probe dials only through the policy-configured httpx client).
-// Conservative: an explicit http/https target, any scheme that responds, or the
-// inability to probe safely all keep web templates in place.
-// Skipped when -no-httpx is set so we do not probe or prune against user intent.
-func (r *Runner) noHTTPServiceReachable() bool {
-	if r.inputProvider == nil || r.options.DisableHTTPProbe {
-		return false
-	}
-	dialers := protocolstate.GetDialersWithId(r.options.ExecutionId)
-	if dialers == nil {
-		return false // cannot probe within policy — keep web templates
-	}
-	httpxOptions := httpx.DefaultOptions
-	if r.options.AliveHttpProxy != "" {
-		httpxOptions.Proxy = r.options.AliveHttpProxy
-	} else if r.options.AliveSocksProxy != "" {
-		httpxOptions.Proxy = r.options.AliveSocksProxy
-	}
-	httpxOptions.RetryMax = r.options.Retries
-	if r.options.Timeout > 0 {
-		httpxOptions.Timeout = time.Duration(r.options.Timeout) * time.Second
-	}
-	httpxOptions.NetworkPolicy = dialers.NetworkPolicy
-	client, err := httpx.New(&httpxOptions)
-	if err != nil {
-		return false // cannot probe — keep web templates
-	}
-
-	anyHTTP := false
-	r.inputProvider.Iterate(func(mi *contextargs.MetaInput) bool {
-		if strings.HasPrefix(mi.Input, "http://") || strings.HasPrefix(mi.Input, "https://") {
-			anyHTTP = true
-			return false
-		}
-		if utils.ProbeURL(mi.Input, client) != "" {
-			anyHTTP = true
-			return false
-		}
-		return true
-	})
-	return !anyHTTP
-}
-
-// tcpNetworkOnlyPorts returns the concrete numeric TCP ports of a single-protocol
-// network template, or ok=false if the template is ineligible for port pruning.
-func tcpNetworkOnlyPorts(t *templates.Template) ([]string, bool) {
-	if len(t.RequestsNetwork)+len(t.RequestsWithTCP) == 0 {
-		return nil, false
-	}
-	// must be network-only (no other protocol request could still be reachable)
-	otherReqs := len(t.RequestsHTTP) + len(t.RequestsWithHTTP) + len(t.RequestsHeadless) +
-		len(t.RequestsDNS) + len(t.RequestsFile) + len(t.RequestsSSL) +
-		len(t.RequestsCode) + len(t.RequestsJavascript) + len(t.Workflows)
-	if otherReqs > 0 {
-		return nil, false
-	}
-	reqs := make([]*network.Request, 0, len(t.RequestsNetwork)+len(t.RequestsWithTCP))
-	reqs = append(reqs, t.RequestsNetwork...)
-	reqs = append(reqs, t.RequestsWithTCP...)
-
-	var ports []string
-	for _, req := range reqs {
-		for _, addr := range req.Address {
-			if strings.Contains(strings.ToLower(addr), "udp://") {
-				return nil, false // UDP: a TCP probe cannot prove unreachability
-			}
-		}
-		pp := splitPorts(req.Port)
-		if len(pp) == 0 {
-			return nil, false // empty/dynamic port (uses input port) — keep
-		}
-		for _, p := range pp {
-			if !isNumericPort(p) {
-				return nil, false // service name or template var — keep
-			}
-			ports = append(ports, p)
-		}
-	}
-	if len(ports) == 0 {
-		return nil, false
-	}
-	return sliceutil.Dedupe(ports), true
-}
-
-func isNumericPort(p string) bool {
-	n, err := strconv.Atoi(p)
-	return err == nil && n > 0 && n < 65536
-}
-
-// hostAndExplicitPort splits a target into host and explicit port (port empty if
-// the target is a bare host).
-func hostAndExplicitPort(input string) (string, string) {
-	if strings.Contains(input, "://") {
-		if u, err := url.Parse(input); err == nil && u.Hostname() != "" {
-			return u.Hostname(), u.Port()
-		}
-	}
-	if host, port, err := net.SplitHostPort(input); err == nil {
-		return host, port
-	}
-	return input, ""
-}
-
-// probe classifies reachability of host:port WITHIN the scan's network policy.
-// Security first: it never dials a target the policy forbids and never bypasses
-// the sandbox — all probing goes through the execution's fastdialer, which
-// enforces the same DenyList (‑lna, exclude-targets, private/metadata ranges,
-// DNS-rebind protection) as the scan itself.
-//
-// It reconciles security with losslessness by mapping every uncertain case to
-// portUnknown ("keep, don't dial"): a policy-denied host, a missing dialer, or a
-// timeout all leave templates in place. Pruning happens only on a definitive,
-// policy-permitted, refused connection.
-func (r *Runner) probe(host, port string) probeResult {
-	if !protocolstate.IsHostAllowed(r.options.ExecutionId, host) {
-		// Policy denies this host: do not dial (security), do not prune (lossless).
-		return portUnknown
-	}
-	dialers := protocolstate.GetDialersWithId(r.options.ExecutionId)
-	if dialers == nil || dialers.Fastdialer == nil {
-		// No policy-enforcing dialer available: never fall back to a raw dial.
-		return portUnknown
-	}
-	return classifyDial(dialers.Fastdialer.Dial, net.JoinHostPort(host, port), reachabilityProbeTimeout)
-}
-
-type dialFunc func(ctx context.Context, network, addr string) (net.Conn, error)
-
-// classifyDial performs one connect via the supplied (policy-enforcing) dialer.
-func classifyDial(dial dialFunc, addr string, timeout time.Duration) probeResult {
-	ctx, cancel := context.WithTimeout(context.Background(), timeout)
-	defer cancel()
-	conn, err := dial(ctx, "tcp", addr)
-	if err == nil {
-		_ = conn.Close()
-		return portOpen
-	}
-	if ne, ok := err.(net.Error); ok && ne.Timeout() {
-		return portUnknown
-	}
-	msg := strings.ToLower(err.Error())
-	// Only definitive negatives prune. DNS blips / temporary dialer errors must
-	// stay unknown so mixed-fleet probing under load remains lossless.
-	if strings.Contains(msg, "refused") || strings.Contains(msg, "unreachable") ||
-		strings.Contains(msg, "no route to host") || strings.Contains(msg, "network is unreachable") {
-		return portClosed
-	}
-	return portUnknown
-}

internal/runner/reachability_test.go

@@ -1,133 +0,0 @@
-package runner
-
-import (
-	"context"
-	"net"
-	"testing"
-	"time"
-
-	httpproto "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http"
-	jsproto "github.com/projectdiscovery/nuclei/v3/pkg/protocols/javascript"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/network"
-	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
-)
-
-func TestHostAndExplicitPort(t *testing.T) {
-	cases := []struct{ in, host, port string }{
-		{"127.0.0.1", "127.0.0.1", ""},
-		{"127.0.0.1:6379", "127.0.0.1", "6379"},
-		{"http://example.com:8080", "example.com", "8080"},
-		{"https://example.com", "example.com", ""},
-	}
-	for _, c := range cases {
-		h, p := hostAndExplicitPort(c.in)
-		if h != c.host || p != c.port {
-			t.Errorf("hostAndExplicitPort(%q) = (%q,%q), want (%q,%q)", c.in, h, p, c.host, c.port)
-		}
-	}
-}
-
-func TestIsNumericPort(t *testing.T) {
-	for _, p := range []string{"80", "6379", "65535", "1"} {
-		if !isNumericPort(p) {
-			t.Errorf("isNumericPort(%q) = false, want true", p)
-		}
-	}
-	for _, p := range []string{"ftp", "{{Port}}", "0", "70000", "", "80a"} {
-		if isNumericPort(p) {
-			t.Errorf("isNumericPort(%q) = true, want false", p)
-		}
-	}
-}
-
-// TestTCPNetworkOnlyPorts locks in the losslessness guards: only single-protocol
-// network templates with concrete numeric TCP ports are eligible for pruning.
-func TestTCPNetworkOnlyPorts(t *testing.T) {
-	netTmpl := func(addr []string, port string) *templates.Template {
-		return &templates.Template{RequestsNetwork: []*network.Request{{Address: addr, Port: port}}}
-	}
-
-	t.Run("concrete tcp port eligible", func(t *testing.T) {
-		ports, ok := tcpNetworkOnlyPorts(netTmpl([]string{"{{Hostname}}"}, "6379,6380"))
-		if !ok || len(ports) != 2 {
-			t.Fatalf("want ok with 2 ports, got ok=%v ports=%v", ok, ports)
-		}
-	})
-	t.Run("tls address still tcp", func(t *testing.T) {
-		if _, ok := tcpNetworkOnlyPorts(netTmpl([]string{"tls://{{Hostname}}"}, "6379")); !ok {
-			t.Fatal("tls:// should remain eligible (still TCP)")
-		}
-	})
-	t.Run("udp address rejected", func(t *testing.T) {
-		if _, ok := tcpNetworkOnlyPorts(netTmpl([]string{"udp://{{Hostname}}"}, "161")); ok {
-			t.Fatal("udp:// must be ineligible — a TCP probe cannot prove UDP unreachability")
-		}
-	})
-	t.Run("service-name port rejected", func(t *testing.T) {
-		if _, ok := tcpNetworkOnlyPorts(netTmpl([]string{"{{Hostname}}"}, "ftp")); ok {
-			t.Fatal("service-name port must be ineligible")
-		}
-	})
-	t.Run("dynamic port rejected", func(t *testing.T) {
-		if _, ok := tcpNetworkOnlyPorts(netTmpl([]string{"{{Hostname}}"}, "{{Port}}")); ok {
-			t.Fatal("template-var port must be ineligible")
-		}
-	})
-	t.Run("empty port rejected", func(t *testing.T) {
-		if _, ok := tcpNetworkOnlyPorts(netTmpl([]string{"{{Hostname}}"}, "")); ok {
-			t.Fatal("empty port (uses input port) must be ineligible")
-		}
-	})
-	t.Run("mixed protocol rejected", func(t *testing.T) {
-		mixed := netTmpl([]string{"{{Hostname}}"}, "6379")
-		mixed.RequestsHTTP = []*httpproto.Request{{}}
-		if _, ok := tcpNetworkOnlyPorts(mixed); ok {
-			t.Fatal("template with an HTTP request must be ineligible")
-		}
-	})
-	t.Run("javascript protocol rejected", func(t *testing.T) {
-		js := &templates.Template{RequestsJavascript: []*jsproto.Request{{}}}
-		if _, ok := tcpNetworkOnlyPorts(js); ok {
-			t.Fatal("javascript template must be ineligible (opaque dial protocol)")
-		}
-	})
-}
-
-func TestClassifyDial(t *testing.T) {
-	ln, err := net.Listen("tcp", "127.0.0.1:0")
-	if err != nil {
-		t.Skip("cannot listen:", err)
-	}
-	defer func() { _ = ln.Close() }()
-	d := &net.Dialer{}
-	if got := classifyDial(d.DialContext, ln.Addr().String(), reachabilityProbeTimeout); got != portOpen {
-		t.Errorf("classifyDial(open) = %v, want portOpen", got)
-	}
-	// an unused low port on loopback should refuse quickly
-	if got := classifyDial(d.DialContext, "127.0.0.1:1", reachabilityProbeTimeout); got != portClosed {
-		t.Errorf("classifyDial(closed) = %v, want portClosed", got)
-	}
-}
-
-func TestClassifyDialDNSFailureIsUnknown(t *testing.T) {
-	// NXDOMAIN / resolution failure must not prune (lossless under load).
-	dial := func(ctx context.Context, network, addr string) (net.Conn, error) {
-		return nil, &net.OpError{Op: "dial", Net: network, Err: &net.DNSError{Err: "no such host", Name: "no-such-host.invalid", IsNotFound: true}}
-	}
-	got := classifyDial(dial, "no-such-host.invalid:6379", reachabilityProbeTimeout)
-	if got != portUnknown {
-		t.Fatalf("classifyDial(dns-fail) = %v, want portUnknown", got)
-	}
-}
-
-func TestClassifyDialTimeoutIsUnknown(t *testing.T) {
-	// A dropped connect runs into the timeout and must not prune.
-	dial := func(ctx context.Context, network, addr string) (net.Conn, error) {
-		<-ctx.Done()
-		return nil, &net.OpError{Op: "dial", Net: network, Err: ctx.Err()}
-	}
-	got := classifyDial(dial, "192.0.2.1:65535", 50*time.Millisecond)
-	if got != portUnknown {
-		t.Fatalf("classifyDial(timeout) = %v, want portUnknown", got)
-	}
-}

internal/runner/runner.go

@@ -45,8 +45,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
 	"github.com/projectdiscovery/nuclei/v3/pkg/core"
-	scanplan "github.com/projectdiscovery/nuclei/v3/pkg/core/plan"
-	"github.com/projectdiscovery/nuclei/v3/pkg/core/techfilter"
 	"github.com/projectdiscovery/nuclei/v3/pkg/external/customtemplates"
 	fuzzStats "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/stats"
 	"github.com/projectdiscovery/nuclei/v3/pkg/input"
@@ -70,18 +68,14 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
 	httpProtocol "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
-	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
-	"github.com/projectdiscovery/nuclei/v3/pkg/types/scanstrategy"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils/stats"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml"
 	"github.com/projectdiscovery/retryablehttp-go"
 	ptrutil "github.com/projectdiscovery/utils/ptr"
-	stringsutil "github.com/projectdiscovery/utils/strings"
 )
 
 var (
@@ -695,12 +689,6 @@ func (r *Runner) RunEnumeration() error {
 		DoNotCache:          r.options.DoNotCacheTemplates,
 		Logger:              r.Logger,
 	}
-	// Attach scan-scoped HTTP response cache only for opt-in -tf, and only
-	// before templates compile so request executers share the same pointer.
-	// Default scans leave this nil (no behavioral change).
-	if r.options.UsesResponseCache() {
-		executorOpts.HTTPResponseCache = httprespcache.New()
-	}
 
 	if config.DefaultConfig.IsDebugArgEnabled(config.DebugExportURLPattern) {
 		// Go StdLib style experimental/debug feature switch
@@ -758,16 +746,6 @@ func (r *Runner) RunEnumeration() error {
 
 	// If using input-file flags, only load http fuzzing based templates.
 	loaderConfig := loader.NewConfig(r.options, r.catalog, executorOpts)
-
-	// Lossless reachability prune: if no web port is reachable on any target,
-	// exclude web-protocol templates at load time — they cannot connect, so
-	// cannot produce a finding. Reuses the standard protocol-type exclusion on
-	// the loader config only, so shared options stay unchanged across runs.
-	if r.strictProbeEnabled() && r.noHTTPServiceReachable() {
-		loaderConfig.ExcludeProtocols = append(slices.Clone(loaderConfig.ExcludeProtocols),
-			templateTypes.HTTPProtocol, templateTypes.HeadlessProtocol, templateTypes.WebsocketProtocol)
-		r.Logger.Info().Msgf("reachability prune: no HTTP service on any target; excluding web-protocol templates (lossless)")
-	}
 	profiles := r.targetProfiles()
 	if profiles != nil {
 		if r.options.AutomaticScan {
@@ -879,15 +857,6 @@ func (r *Runner) RunEnumeration() error {
 			return errors.Wrap(err, "could not probe http input")
 		}
 		executorOpts.InputHelper.InputsHTTP = inputHelpers
-		executorOpts.InputHelper.InputsHTTPProbed = r.inputProvider.InputType() != provider.MultiFormatInputProvider
-		// Under strict-probe, skip web templates per-input for targets that
-		// probed as non-HTTP (lossless), instead of falling back to raw URL.
-		executorOpts.InputHelper.StrictProbe = r.strictProbeEnabled()
-		if r.strictProbeEnabled() {
-			r.Logger.Info().Msgf("Strict probe enabled: hosts without httpx confirmation are skipped for HTTP/headless templates")
-		}
-	} else if r.strictProbeEnabled() && r.options.DisableHTTPProbe {
-		r.Logger.Warning().Msgf("strict-probe has no effect with -no-httpx (httpx probing is disabled)")
 	}
 
 	inputCount := int(r.inputProvider.Count())
@@ -1062,157 +1031,15 @@ func (r *Runner) executeTemplatesInput(store *loader.Store, engine *core.Engine)
 		return nil, errors.New("no templates provided for scan")
 	}
 
+	// pass input provider to engine
+	// TODO: this should be not necessary after r.hmapInputProvider is removed + refactored
 	if r.inputProvider == nil {
 		return nil, errors.New("no input provider found")
 	}
-
-	return r.executeTemplatesWithScanPlan(engine, finalTemplates)
-}
-
-// executeTemplatesWithScanPlan chooses spray strategy and optionally attaches a
-// reachability / tech filter, then runs a single Execute pass. Wall clock must
-// match or beat baseline spray; probing is skipped when its cost would dominate.
-func (r *Runner) executeTemplatesWithScanPlan(engine *core.Engine, finalTemplates []*templates.Template) (*atomic.Bool, error) {
-	defer r.attachScanPlan(engine, finalTemplates)()
 	results := engine.ExecuteScanWithOpts(context.Background(), finalTemplates, r.inputProvider, r.options.DisableClustering)
 	return results, nil
 }
 
-// attachScanPlan applies the scan plan to engine for one scan. It returns a
-// func that undoes the per-scan state once the scan is done.
-func (r *Runner) attachScanPlan(engine *core.Engine, finalTemplates []*templates.Template) func() {
-	var undo []func()
-	cleanup := func() {
-		for i := len(undo) - 1; i >= 0; i-- {
-			undo[i]()
-		}
-	}
-	hostCount := int(r.inputProvider.Count())
-	baselineReqs := estimateTemplateRequests(finalTemplates) * hostCount
-	concreteNet, portsToProbe := countReachabilityStats(finalTemplates)
-	techBound := techfilter.CountTechBound(finalTemplates)
-
-	planIn := scanplan.Input{
-		Hosts:                    hostCount,
-		Templates:                len(finalTemplates),
-		Requests:                 baselineReqs,
-		BulkSize:                 r.options.BulkSize,
-		Stream:                   r.options.Stream,
-		StrictProbe:              r.strictProbeEnabled(),
-		PortsToProbe:             portsToProbe,
-		ConcreteNetworkTemplates: concreteNet,
-		TechFilter:               r.options.TechFilter,
-		TechBoundTemplates:       techBound,
-	}
-	p := scanplan.Decide(planIn)
-
-	// Resolve -ss auto before Execute so the engine placeholder is unused.
-	if stringsutil.EqualFoldAny(r.options.ScanStrategy, scanstrategy.Auto.String(), "") {
-		r.options.ScanStrategy = p.Strategy
-	}
-
-	var httpHelper *input.Helper
-	if opts := engine.ExecuterOptions(); opts != nil {
-		httpHelper = opts.InputHelper
-	}
-
-	var reachFilter func(*templates.Template, *contextargs.MetaInput) bool
-	var techFilterFn func(*templates.Template, *contextargs.MetaInput) bool
-	var techIdx *techReachabilityIndex
-	filteredReqs := baselineReqs
-
-	if p.BuildReachability {
-		idx, groups, err := r.buildHostReachability(finalTemplates, httpHelper, p.ProbePorts)
-		if err != nil {
-			r.Logger.Warning().Msgf("could not build host reachability index, continuing unfiltered: %s", err)
-		} else {
-			baseline, filtered := estimateGroupedExecutions(finalTemplates, groups, hostCount)
-			filteredReqs = filtered
-			saved := baseline - filtered
-			pct := 0.0
-			if baseline > 0 {
-				pct = 100 * float64(saved) / float64(baseline)
-			}
-			for _, g := range groups {
-				r.Logger.Info().Msgf("host-group %s: hosts=%d templates=%d", g.key, len(g.hosts), len(templatesForHostGroup(finalTemplates, g)))
-			}
-			r.Logger.Info().Msgf(
-				"host-groups: groups=%d baseline_exec=%d filtered_exec=%d saved=%d (%.1f%%) mode=single-pass",
-				len(groups), baseline, filtered, saved, pct,
-			)
-			if p.UseReachabilityFilter {
-				reachFilter = idx.Allow
-			}
-		}
-	}
-
-	if p.UseTechFilter {
-		var err error
-		var cache *httprespcache.Cache
-		if opts := engine.ExecuterOptions(); opts != nil {
-			if opts.HTTPResponseCache == nil {
-				opts.HTTPResponseCache = httprespcache.New()
-			}
-			cache = opts.HTTPResponseCache
-		}
-		techIdx, err = r.buildTechReachability(finalTemplates, httpHelper, cache)
-		if err != nil {
-			r.Logger.Warning().Msgf("could not build tech filter index, continuing without tech-filter: %s", err)
-			if cache != nil {
-				cache.Disable()
-			}
-			techIdx = nil
-		} else {
-			base, techFiltered := estimateTechFilteredExecutions(finalTemplates, techIdx, hostCount)
-			// When stacked with reachability, take the more aggressive (lower) estimate
-			// as a lower bound for progress; actual AND filter may skip more.
-			if techFiltered < filteredReqs {
-				filteredReqs = techFiltered
-			}
-			saved := base - techFiltered
-			pct := 0.0
-			if base > 0 {
-				pct = 100 * float64(saved) / float64(base)
-			}
-			r.Logger.Info().Msgf(
-				"tech-filter: baseline_exec=%d filtered_exec=%d saved=%d (%.1f%%) bound_templates=%d",
-				base, techFiltered, saved, pct, techBound,
-			)
-			techFilterFn = techIdx.Allow
-		}
-	} else if opts := engine.ExecuterOptions(); opts != nil && opts.HTTPResponseCache != nil && !r.options.ResponseCache {
-		// -tf set but planner skipped (e.g. no tech-bound templates): do not
-		// alter request paths via an idle response cache. Keep it when the user
-		// asked for -response-cache explicitly.
-		opts.HTTPResponseCache.Disable()
-	}
-
-	p.ApplyFiltered(filteredReqs)
-	if opts := engine.ExecuterOptions(); opts != nil {
-		// The filters narrow the templates per target through the same
-		// TargetScope as per-target profiles, on top of them. The engine then
-		// skips the pairs, counts progress and narrows clusters as it does for
-		// profiles.
-		if filter := composeTemplateFilters(reachFilter, techFilterFn); filter != nil {
-			profiles := opts.TargetScope
-			opts.TargetScope = newPlanScope(profiles, filter, finalTemplates)
-			undo = append(undo, func() { opts.TargetScope = profiles })
-		}
-		if opts.HTTPResponseCache != nil {
-			undo = append(undo, func() {
-				hits, misses, stores := opts.HTTPResponseCache.Stats()
-				r.Logger.Info().Msgf("http-response-cache: hits=%d misses=%d stores=%d", hits, misses, stores)
-				opts.HTTPResponseCache = nil
-			})
-		}
-	}
-
-	r.Logger.Info().Msgf("scan-plan: strategy=%s reach=%v tech=%v expected_req=%d reason=%s",
-		r.options.ScanStrategy, p.UseReachabilityFilter, p.UseTechFilter, p.ExpectedRequests, p.Reason)
-
-	return cleanup
-}
-
 // displayExecutionInfo prints parser stats, version info, and scan counts.
 func (r *Runner) displayExecutionInfo(store *loader.Store) {
 	// Display parser stats for templates loaded into the store.

internal/runner/sdk_scan_plan.go

@@ -1,40 +0,0 @@
-package runner
-
-import (
-	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v3/pkg/core"
-	"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache"
-	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
-	"github.com/projectdiscovery/nuclei/v3/pkg/types"
-)
-
-// AttachScanPlan applies -strict-probe and -tech-filter to an SDK engine for
-// one scan, the same way the CLI runner does. It returns a func that undoes the
-// per-scan state; call it once the scan is done. When neither option is set it
-// does nothing, so default SDK scans keep their behavior.
-func AttachScanPlan(opts *types.Options, inputProvider provider.InputProvider, logger *gologger.Logger, engine *core.Engine, tpls []*templates.Template) func() {
-	if opts == nil || engine == nil || inputProvider == nil || (!opts.StrictProbe && !opts.TechFilter) {
-		return func() {}
-	}
-	if logger == nil {
-		logger = gologger.DefaultLogger
-	}
-	// The SDK engine is reused across scans, so keep the options and the
-	// response cache it owns as they were before this scan.
-	strategy := opts.ScanStrategy
-	executerOpts := engine.ExecuterOptions()
-	var responseCache *httprespcache.Cache
-	if executerOpts != nil {
-		responseCache = executerOpts.HTTPResponseCache
-	}
-	r := &Runner{options: opts, inputProvider: inputProvider, Logger: logger}
-	cleanup := r.attachScanPlan(engine, tpls)
-	return func() {
-		cleanup()
-		opts.ScanStrategy = strategy
-		if executerOpts != nil {
-			executerOpts.HTTPResponseCache = responseCache
-		}
-	}
-}

internal/runner/sdk_scan_plan_test.go

@@ -1,44 +0,0 @@
-package runner
-
-import (
-	"testing"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/core"
-	"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache"
-	"github.com/projectdiscovery/nuclei/v3/pkg/types"
-	"github.com/stretchr/testify/require"
-)
-
-// Without -strict-probe or -tech-filter the SDK hook changes nothing.
-func TestAttachScanPlanNoop(t *testing.T) {
-	opts := types.DefaultOptions()
-	opts.ScanStrategy = "auto"
-	engine := core.New(opts)
-	engine.SetExecuterOptions(&protocols.ExecutorOptions{Options: opts})
-	ip := provider.NewSimpleInputProviderWithUrls(opts.ExecutionId, "http://127.0.0.1:1")
-
-	AttachScanPlan(opts, ip, nil, engine, nil)()
-	require.Equal(t, "auto", opts.ScanStrategy)
-	require.Nil(t, engine.ExecuterOptions().TargetScope)
-}
-
-// The SDK engine is reused across scans, so the hook must leave its options
-// and its response cache as they were before the scan.
-func TestAttachScanPlanRestoresEngineState(t *testing.T) {
-	opts := types.DefaultOptions()
-	opts.ScanStrategy = "auto"
-	opts.TechFilter = true
-	cache := httprespcache.New()
-	engine := core.New(opts)
-	engine.SetExecuterOptions(&protocols.ExecutorOptions{Options: opts, HTTPResponseCache: cache})
-	ip := provider.NewSimpleInputProviderWithUrls(opts.ExecutionId, "http://127.0.0.1:1")
-
-	cleanup := AttachScanPlan(opts, ip, nil, engine, nil)
-	require.NotEqual(t, "auto", opts.ScanStrategy, "the plan resolves the auto strategy for the scan")
-	cleanup()
-	require.Equal(t, "auto", opts.ScanStrategy)
-	require.Same(t, cache, engine.ExecuterOptions().HTTPResponseCache)
-	require.Nil(t, engine.ExecuterOptions().TargetScope)
-}

internal/runner/tech_filter.go

@@ -1,283 +0,0 @@
-package runner
-
-import (
-	"io"
-	"net/http"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v3/pkg/core/techfilter"
-	"github.com/projectdiscovery/nuclei/v3/pkg/input"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
-	httpProtocol "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache"
-	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
-	"github.com/projectdiscovery/retryablehttp-go"
-	syncutil "github.com/projectdiscovery/utils/sync"
-	unitutils "github.com/projectdiscovery/utils/unit"
-	wappalyzer "github.com/projectdiscovery/wappalyzergo"
-)
-
-const (
-	techFilterWorkers      = 50
-	techFilterMaxBody      = 1 * unitutils.Mega
-	techFilterHTTPTimeout  = 5 * time.Second
-	techFilterMaxRedirects = 5
-)
-
-// techReachabilityIndex filters template×target pairs by product/macro tags.
-type techReachabilityIndex struct {
-	byInput map[string]techfilter.HostProfile // keyed by normalizeInputKey
-
-	// boundTags memoizes template ID -> product tags. It is filled lazily from
-	// the template execution goroutines, so every access needs the mutex.
-	tagsMu    sync.RWMutex
-	boundTags map[string][]string
-}
-
-func normalizeInputKey(raw string) string {
-	return strings.TrimRight(strings.TrimSpace(raw), "/")
-}
-
-// Allow is fail-open: missing profile or unbound template => true.
-func (idx *techReachabilityIndex) Allow(t *templates.Template, mi *contextargs.MetaInput) bool {
-	if idx == nil || t == nil || mi == nil {
-		return true
-	}
-	profile, ok := idx.lookup(mi.Input)
-	if !ok {
-		return true
-	}
-	return techfilter.AllowTags(profile, idx.tagsFor(t))
-}
-
-func (idx *techReachabilityIndex) tagsFor(t *templates.Template) []string {
-	if t == nil {
-		return nil
-	}
-	idx.tagsMu.RLock()
-	tags, ok := idx.boundTags[t.ID]
-	idx.tagsMu.RUnlock()
-	if ok {
-		return tags
-	}
-
-	tags = techfilter.TemplateProductTags(t)
-
-	idx.tagsMu.Lock()
-	defer idx.tagsMu.Unlock()
-	if idx.boundTags == nil {
-		idx.boundTags = make(map[string][]string)
-	}
-	// Another goroutine may have won the race; keep one slice per template ID.
-	if existing, ok := idx.boundTags[t.ID]; ok {
-		return existing
-	}
-	idx.boundTags[t.ID] = tags
-	return tags
-}
-
-func (idx *techReachabilityIndex) lookup(raw string) (techfilter.HostProfile, bool) {
-	if idx == nil || idx.byInput == nil {
-		return techfilter.HostProfile{}, false
-	}
-	if p, ok := idx.byInput[normalizeInputKey(raw)]; ok {
-		return p, true
-	}
-	if p, ok := idx.byInput[raw]; ok {
-		return p, true
-	}
-	return techfilter.HostProfile{}, false
-}
-
-// buildTechReachability fingerprints HTTP(S) targets and builds a tag index.
-// When cache is non-nil, fingerprint GET responses seed it so matching template
-// GETs pay no extra RTT.
-func (r *Runner) buildTechReachability(tpls []*templates.Template, httpHelper *input.Helper, cache *httprespcache.Cache) (*techReachabilityIndex, error) {
-	_ = tpls
-	wapp, err := wappalyzer.New()
-	if err != nil {
-		return nil, err
-	}
-	// Redirects must be followed. A site that answers "/" with a 302 to its real
-	// entry point (WordPress -> /wp-admin/install.php, any app -> /login) returns
-	// an empty body, so the probe would detect nothing and every template bound
-	// to the undetected product would be pruned. Same-host only, so the probe
-	// never leaves the target authority.
-	httpclient, err := httpclientpool.Get(r.options, &httpclientpool.Configuration{
-		DisableCookie:         true,
-		ResponseHeaderTimeout: techFilterHTTPTimeout,
-		RedirectFlow:          httpclientpool.FollowSameHostRedirect,
-		MaxRedirects:          techFilterMaxRedirects,
-	}, "")
-	if err != nil {
-		return nil, err
-	}
-
-	idx := &techReachabilityIndex{
-		byInput:   make(map[string]techfilter.HostProfile),
-		boundTags: make(map[string][]string),
-	}
-	var mu sync.Mutex
-	sg, err := syncutil.New(syncutil.WithSize(techFilterWorkers))
-	if err != nil {
-		return nil, err
-	}
-
-	fingerprinted := 0
-	eligible := 0
-	r.inputProvider.Iterate(func(mi *contextargs.MetaInput) bool {
-		if mi == nil || mi.Input == "" {
-			return true
-		}
-		fpURL := fingerprintURLForInput(mi.Input, httpHelper)
-		if fpURL == "" {
-			return true
-		}
-		eligible++
-		sg.Add()
-		go func(inputKey, url string) {
-			defer sg.Done()
-			profile := fingerprintTarget(wapp, httpclient, url, cache)
-			key := normalizeInputKey(inputKey)
-			mu.Lock()
-			idx.byInput[key] = profile
-			if profile.HasTags() {
-				fingerprinted++
-			}
-			mu.Unlock()
-		}(mi.Input, fpURL)
-		return true
-	})
-	sg.Wait()
-
-	gologger.Info().Msgf("tech-filter: fingerprinted=%d/%d hosts with tags (eligible=%d)", fingerprinted, len(idx.byInput), eligible)
-	return idx, nil
-}
-
-// fingerprintURLForInput returns an absolute http(s) URL to fingerprint, or "".
-// Bare hosts reuse InputsHTTP probed URLs when available.
-func fingerprintURLForInput(raw string, helper *input.Helper) string {
-	if looksLikeHTTPTarget(raw) {
-		return raw
-	}
-	if helper != nil && helper.InputsHTTP != nil {
-		if probed, ok := helper.InputsHTTP.Get(raw); ok && len(probed) > 0 {
-			return string(probed)
-		}
-	}
-	return ""
-}
-
-func looksLikeHTTPTarget(raw string) bool {
-	lower := strings.ToLower(raw)
-	return strings.HasPrefix(lower, "http://") || strings.HasPrefix(lower, "https://")
-}
-
-func fingerprintTarget(wapp *wappalyzer.Wappalyze, client *retryablehttp.Client, rawURL string, cache *httprespcache.Cache) techfilter.HostProfile {
-	// Prefer a trailing-slash form that matches typical {{BaseURL}}/ template URLs.
-	seedURL := rawURL
-	if !strings.HasSuffix(seedURL, "/") {
-		seedURL = seedURL + "/"
-	}
-	req, err := retryablehttp.NewRequest(http.MethodGet, seedURL, nil)
-	if err != nil {
-		return techfilter.HostProfile{}
-	}
-	// Same user agent the templates will send, so the response seeded below is
-	// reachable from the cache instead of being stored under a key nothing hits.
-	req.Header.Set("User-Agent", httpProtocol.StableUserAgent())
-	resp, err := client.Do(req)
-	if err != nil {
-		return techfilter.HostProfile{}
-	}
-	defer func() { _ = resp.Body.Close() }()
-	body, err := io.ReadAll(io.LimitReader(resp.Body, techFilterMaxBody))
-	if err != nil {
-		return techfilter.HostProfile{}
-	}
-	if cache != nil {
-		seedFingerprintResponse(cache, req, resp, body, seedURL)
-	}
-	info := wapp.FingerprintWithInfo(resp.Header, body)
-	products := make(map[string][]string, len(info))
-	for name, app := range info {
-		products[name] = append([]string(nil), app.Categories...)
-	}
-	return techfilter.ProfileFromFingerprint(products)
-}
-
-// seedFingerprintResponse stores the probe response under the URL it actually
-// belongs to, so templates can reuse the RTT.
-//
-// The probe follows redirects, so resp may describe a different URL than the one
-// requested. Seeding it under the requested URL would hand every template that
-// asks for that URL the redirect target's status, headers and body instead of
-// the redirect itself, which silently changes what they match on.
-func seedFingerprintResponse(cache *httprespcache.Cache, req *retryablehttp.Request, resp *http.Response, body []byte, seedURL string) {
-	finalURL := seedURL
-	if resp != nil && resp.Request != nil && resp.Request.URL != nil {
-		finalURL = resp.Request.URL.String()
-	}
-	if finalURL == seedURL {
-		cache.SeedHTTP(req.Request, resp, body, techFilterMaxBody)
-		return
-	}
-	finalReq := req.Request.Clone(req.Context())
-	finalReq.URL = resp.Request.URL
-	finalReq.Host = ""
-	cache.SeedHTTP(finalReq, resp, body, techFilterMaxBody)
-}
-
-// estimateTechFilteredExecutions counts template×host pairs that survive the tech filter.
-func estimateTechFilteredExecutions(tpls []*templates.Template, idx *techReachabilityIndex, hostCount int) (baseline, filtered int) {
-	baseline = estimateTemplateRequests(tpls) * hostCount
-	if idx == nil || len(idx.byInput) == 0 {
-		return baseline, baseline
-	}
-	filtered = 0
-	for _, t := range tpls {
-		reqs := t.TotalRequests
-		if reqs <= 0 {
-			reqs = 1
-		}
-		if len(t.Workflows) > 0 {
-			continue
-		}
-		tags := idx.tagsFor(t)
-		for _, profile := range idx.byInput {
-			if techfilter.AllowTags(profile, tags) {
-				filtered += reqs
-			}
-		}
-		missing := hostCount - len(idx.byInput)
-		if missing > 0 {
-			filtered += reqs * missing
-		}
-	}
-	return baseline, filtered
-}
-
-// composeTemplateFilters ANDs optional filters; nil filters are ignored.
-func composeTemplateFilters(filters ...func(*templates.Template, *contextargs.MetaInput) bool) func(*templates.Template, *contextargs.MetaInput) bool {
-	var active []func(*templates.Template, *contextargs.MetaInput) bool
-	for _, f := range filters {
-		if f != nil {
-			active = append(active, f)
-		}
-	}
-	if len(active) == 0 {
-		return nil
-	}
-	return func(t *templates.Template, mi *contextargs.MetaInput) bool {
-		for _, f := range active {
-			if !f(t, mi) {
-				return false
-			}
-		}
-		return true
-	}
-}

internal/runner/tech_filter_race_test.go

@@ -1,33 +0,0 @@
-package runner
-
-import (
-	"sync"
-	"testing"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/core/techfilter"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
-	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
-)
-
-// tagsFor is filled lazily from the template execution goroutines. Run it under
-// -race to catch the concurrent map access that used to crash whole scans.
-func TestTechReachabilityIndexConcurrentTagsFor(t *testing.T) {
-	idx := &techReachabilityIndex{
-		byInput:   map[string]techfilter.HostProfile{},
-		boundTags: map[string][]string{},
-	}
-
-	var wg sync.WaitGroup
-	for i := 0; i < 64; i++ {
-		wg.Add(1)
-		go func(i int) {
-			defer wg.Done()
-			for j := 0; j < 64; j++ {
-				tpl := &templates.Template{ID: string(rune('a' + (i+j)%26))}
-				_ = idx.tagsFor(tpl)
-				_ = idx.Allow(tpl, &contextargs.MetaInput{Input: "http://x/"})
-			}
-		}(i)
-	}
-	wg.Wait()
-}

internal/runner/tech_filter_seed_test.go

@@ -1,70 +0,0 @@
-package runner
-
-import (
-	"net/http"
-	"net/url"
-	"testing"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache"
-	"github.com/projectdiscovery/retryablehttp-go"
-)
-
-func mustReq(t *testing.T, raw string) *retryablehttp.Request {
-	t.Helper()
-	req, err := retryablehttp.NewRequest(http.MethodGet, raw, nil)
-	if err != nil {
-		t.Fatalf("request: %v", err)
-	}
-	return req
-}
-
-func respFor(raw string, status int) *http.Response {
-	u, _ := url.Parse(raw)
-	return &http.Response{
-		StatusCode: status,
-		Status:     http.StatusText(status),
-		Header:     http.Header{"Server": []string{"probe"}},
-		Request:    &http.Request{Method: http.MethodGet, URL: u},
-	}
-}
-
-// The probe follows redirects. The body it comes back with belongs to the
-// redirect target, so it must never be stored under the requested URL: a
-// template asking for "/" must still see the redirect, not the target page.
-func TestSeedFingerprintResponseAfterRedirect(t *testing.T) {
-	const seedURL = "http://example.test/"
-	const finalURL = "http://example.test/wp-admin/install.php"
-
-	c := httprespcache.New()
-	req := mustReq(t, seedURL)
-	req.Header.Set("User-Agent", "probe-ua")
-	seedFingerprintResponse(c, req, respFor(finalURL, 200), []byte("install page"), seedURL)
-
-	lookup := req.Request.Clone(req.Context())
-	if got := c.Get(httprespcache.KeyFromRequest(lookup), "template", lookup); got != nil {
-		t.Fatal("the redirect target must not be cached under the requested URL")
-	}
-
-	finalLookup := req.Request.Clone(req.Context())
-	finalLookup.URL, _ = url.Parse(finalURL)
-	finalLookup.Host = ""
-	if got := c.Get(httprespcache.KeyFromRequest(finalLookup), "template", finalLookup); got == nil {
-		t.Fatal("the response must be cached under the URL it belongs to")
-	}
-}
-
-// Without a redirect the response does describe the requested URL, so seeding
-// it there is what saves the round trip.
-func TestSeedFingerprintResponseWithoutRedirect(t *testing.T) {
-	const seedURL = "http://example.test/"
-
-	c := httprespcache.New()
-	req := mustReq(t, seedURL)
-	req.Header.Set("User-Agent", "probe-ua")
-	seedFingerprintResponse(c, req, respFor(seedURL, 200), []byte("home"), seedURL)
-
-	lookup := req.Request.Clone(req.Context())
-	if got := c.Get(httprespcache.KeyFromRequest(lookup), "template", lookup); got == nil {
-		t.Fatal("a non-redirected response must be cached under the requested URL")
-	}
-}

internal/runner/tech_filter_test.go

@@ -1,21 +0,0 @@
-package runner
-
-import (
-	"testing"
-
-	"github.com/projectdiscovery/hmap/store/hybrid"
-	"github.com/projectdiscovery/nuclei/v3/pkg/input"
-	"github.com/stretchr/testify/require"
-)
-
-func TestFingerprintURLForInput(t *testing.T) {
-	require.Equal(t, "http://a/", fingerprintURLForInput("http://a/", nil))
-	require.Equal(t, "", fingerprintURLForInput("example.com", nil))
-
-	hm, err := hybrid.New(hybrid.DefaultDiskOptions)
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = hm.Close() })
-	require.NoError(t, hm.Set("example.com", []byte("https://example.com")))
-	helper := &input.Helper{InputsHTTP: hm, InputsHTTPProbed: true}
-	require.Equal(t, "https://example.com", fingerprintURLForInput("example.com", helper))
-}

internal/tests/integration/testdata/protocols/headless/file-upload.yaml

@@ -23,6 +23,11 @@ headless:
       args:
         by: x
         xpath: /html/body/form/input[2]
+    # the upload response echoes the file, rendered as text in a pre element
+    - action: waitvisible
+      args:
+        by: xpath
+        xpath: //pre[contains(., "Basic File Upload")]
     matchers:
       - type: word
         words:

lib/config.go

@@ -173,16 +173,6 @@ func WithResponseReadSize(responseReadSize int) NucleiSDKOptions {
 	}
 }
 
-// EnableResponseCache reuses identical safe GET/HEAD responses within one scan.
-// It is off by default: a template that depends on state or timing could see a
-// different result.
-func EnableResponseCache() NucleiSDKOptions {
-	return func(e *NucleiEngine) error {
-		e.opts.ResponseCache = true
-		return nil
-	}
-}
-
 // WithGlobalRateLimit sets global rate (i.e all hosts combined) limit options
 // Deprecated: will be removed in favour of WithGlobalRateLimitCtx in next release
 func WithGlobalRateLimit(maxTokens int, duration time.Duration) NucleiSDKOptions {
@@ -766,31 +756,3 @@ func WithTemporaryDirectory(parentDir string) NucleiSDKOptions {
 		return nil
 	}
 }
-
-// WithPerHostRateLimit applies the rate limit per host instead of globally.
-func WithPerHostRateLimit(enabled bool) NucleiSDKOptions {
-	return func(e *NucleiEngine) error {
-		e.opts.PerHostRateLimit = enabled
-		return nil
-	}
-}
-
-// EnableStrictProbe skips templates whose target service is unreachable: web
-// templates on hosts that do not speak HTTP and network templates on closed
-// ports. It is the SDK form of -strict-probe.
-func EnableStrictProbe() NucleiSDKOptions {
-	return func(e *NucleiEngine) error {
-		e.opts.StrictProbe = true
-		return nil
-	}
-}
-
-// EnableTechFilter fingerprints HTTP targets and skips product-bound templates
-// that cannot match them. It also turns on the response cache, which the
-// fingerprint requests seed. It is the SDK form of -tech-filter.
-func EnableTechFilter() NucleiSDKOptions {
-	return func(e *NucleiEngine) error {
-		e.opts.TechFilter = true
-		return nil
-	}
-}

lib/multi.go

@@ -5,7 +5,6 @@ import (
 	"time"
 
 	"github.com/logrusorgru/aurora/v4"
-	"github.com/projectdiscovery/nuclei/v3/internal/runner"
 	"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
 	"github.com/projectdiscovery/nuclei/v3/pkg/core"
@@ -13,7 +12,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/loader/workflow"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils"
 	"github.com/projectdiscovery/utils/errkit"
@@ -62,11 +60,6 @@ func createEphemeralObjects(ctx context.Context, base *NucleiEngine, opts *types
 		opts.RateLimitDuration = time.Second
 	}
 	u.executerOpts.RateLimiter = utils.GetRateLimiter(ctx, opts.RateLimit, opts.RateLimitDuration)
-	// Each execution gets its own response cache, created before its
-	// templates compile.
-	if opts.UsesResponseCache() && u.executerOpts.HTTPResponseCache == nil {
-		u.executerOpts.HTTPResponseCache = httprespcache.New()
-	}
 	return u, nil
 }
 
@@ -247,7 +240,6 @@ func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOptsCtx(ctx context.Context, t
 	engine := core.New(tmpEngine.opts)
 	engine.SetExecuterOptions(unsafeOpts.executerOpts)
 
-	defer runner.AttachScanPlan(tmpEngine.opts, inputProvider, tmpEngine.opts.Logger, engine, store.Templates())()
 	_ = engine.ExecuteScanWithOpts(ctx, store.Templates(), inputProvider, false)
 
 	engine.WorkPool().Wait()

lib/scan_plan_test.go

@@ -1,120 +0,0 @@
-package nuclei_test
-
-import (
-	"context"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"path/filepath"
-	"sync"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	nuclei "github.com/projectdiscovery/nuclei/v3/lib"
-	"github.com/projectdiscovery/nuclei/v3/pkg/output"
-	"github.com/stretchr/testify/require"
-)
-
-func TestScanPlanOptions(t *testing.T) {
-	ne, err := nuclei.NewNucleiEngineCtx(context.Background(),
-		nuclei.DisableUpdateCheck(),
-		nuclei.EnableStrictProbe(),
-		nuclei.EnableTechFilter(),
-		nuclei.EnableResponseCache(),
-	)
-	require.NoError(t, err)
-	defer ne.Close()
-
-	opts := ne.Options()
-	require.True(t, opts.StrictProbe)
-	require.True(t, opts.TechFilter)
-	require.True(t, opts.ResponseCache)
-}
-
-// sdkCacheTemplates writes two templates that both GET /same. The second also
-// GETs /other, so request clustering cannot merge them into one request.
-func sdkCacheTemplates(t *testing.T) []string {
-	t.Helper()
-	dir := t.TempDir()
-	a := `id: sdk-cache-a
-info:
-  name: SDK Cache A
-  author: test
-  severity: info
-http:
-  - method: GET
-    path:
-      - "{{BaseURL}}/same"
-    matchers:
-      - type: word
-        words:
-          - "cached-body"
-`
-	b := `id: sdk-cache-b
-info:
-  name: SDK Cache B
-  author: test
-  severity: info
-http:
-  - method: GET
-    path:
-      - "{{BaseURL}}/same"
-      - "{{BaseURL}}/other"
-    matchers:
-      - type: word
-        words:
-          - "cached-body"
-`
-	pa, pb := filepath.Join(dir, "a.yaml"), filepath.Join(dir, "b.yaml")
-	require.NoError(t, os.WriteFile(pa, []byte(a), 0o600))
-	require.NoError(t, os.WriteFile(pb, []byte(b), 0o600))
-	return []string{pa, pb}
-}
-
-// With the SDK options for the scan plan set, both templates still run and
-// match, and the response cache sends /same once. Without them it is sent
-// twice.
-func TestSDKScanPlanAndResponseCache(t *testing.T) {
-	var sameHits atomic.Int32
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/same" {
-			sameHits.Add(1)
-			time.Sleep(200 * time.Millisecond)
-		}
-		_, _ = w.Write([]byte("cached-body"))
-	}))
-	defer ts.Close()
-	tpls := sdkCacheTemplates(t)
-
-	run := func(extra ...nuclei.NucleiSDKOptions) map[string]bool {
-		sameHits.Store(0)
-		opts := append([]nuclei.NucleiSDKOptions{
-			nuclei.DisableUpdateCheck(),
-			nuclei.WithTemplatesOrWorkflows(nuclei.TemplateSources{Templates: tpls}),
-		}, extra...)
-		ne, err := nuclei.NewNucleiEngineCtx(context.Background(), opts...)
-		require.NoError(t, err)
-		defer ne.Close()
-		ne.Options().RestrictLocalNetworkAccess = false
-		ne.Options().Retries = 0
-		ne.LoadTargets([]string{ts.URL}, false)
-
-		var mu sync.Mutex
-		matched := map[string]bool{}
-		require.NoError(t, ne.ExecuteCallbackWithCtx(context.Background(), func(event *output.ResultEvent) {
-			mu.Lock()
-			defer mu.Unlock()
-			matched[event.TemplateID] = true
-		}))
-		return matched
-	}
-
-	matched := run(nuclei.EnableResponseCache(), nuclei.EnableStrictProbe(), nuclei.EnableTechFilter())
-	require.Equal(t, map[string]bool{"sdk-cache-a": true, "sdk-cache-b": true}, matched)
-	require.Equal(t, int32(1), sameHits.Load(), "the response cache must send /same once")
-
-	matched = run()
-	require.Equal(t, map[string]bool{"sdk-cache-a": true, "sdk-cache-b": true}, matched)
-	require.Equal(t, int32(2), sameHits.Load(), "without the cache /same is sent by each template")
-}

lib/sdk.go

@@ -9,7 +9,6 @@ import (
 	"sync"
 
 	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v3/internal/runner"
 	"github.com/projectdiscovery/nuclei/v3/pkg/authprovider"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog"
 	metadataindex "github.com/projectdiscovery/nuclei/v3/pkg/catalog/index"
@@ -332,10 +331,6 @@ func (e *NucleiEngine) ExecuteCallbackWithCtx(ctx context.Context, callback ...f
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		// A response cached by an earlier scan of this engine is never served
-		// to this one.
-		e.executerOpts.HTTPResponseCache.Reset()
-		defer runner.AttachScanPlan(e.opts, e.inputProvider, e.opts.Logger, e.engine, templatesAndWorkflows)()
 		_ = e.engine.ExecuteScanWithOpts(ctx, templatesAndWorkflows, e.inputProvider, false)
 	}()
 

lib/sdk_private.go

@@ -33,7 +33,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	nucleiUtils "github.com/projectdiscovery/nuclei/v3/pkg/utils"
@@ -354,12 +353,6 @@ func (e *NucleiEngine) init(ctx context.Context) error {
 		e.executerOpts.Options.Parser = op
 	}
 
-	// Templates compile against e.executerOpts, so the response cache must
-	// exist before they load. Each scan resets it.
-	if e.opts.UsesResponseCache() && e.executerOpts.HTTPResponseCache == nil {
-		e.executerOpts.HTTPResponseCache = httprespcache.New()
-	}
-
 	e.engine = core.New(e.opts)
 	e.engine.SetExecuterOptions(e.executerOpts)
 

pkg/core/execute_options.go

@@ -86,7 +86,8 @@ func (e *Engine) ExecuteScanWithOpts(ctx context.Context, templatesList []*templ
 	}
 
 	if stringsutil.EqualFoldAny(e.options.ScanStrategy, scanstrategy.Auto.String(), "") {
-		// Last-resort fallback: runner scan planner normally resolves auto.
+		// TODO: this is only a placeholder, auto scan strategy should choose scan strategy
+		// based on no of hosts , templates , stream and other optimization parameters
 		e.options.ScanStrategy = scanstrategy.TemplateSpray.String()
 	}
 

pkg/core/plan/plan.go

@@ -1,165 +0,0 @@
-package plan
-
-import (
-	"fmt"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/types/scanstrategy"
-)
-
-// Input is the scan shape the planner uses to pick strategy and reachability.
-type Input struct {
-	Hosts       int
-	Templates   int
-	Requests    int // template×host request estimate (baseline)
-	BulkSize    int
-	Stream      bool
-	StrictProbe bool
-
-	// Reachability / probe budget (known before or after probing).
-	PortsToProbe             int
-	ConcreteNetworkTemplates int
-	// FilteredRequests, when > 0, is the post-filter request estimate the plan reports.
-	FilteredRequests int
-	// GroupCount is distinct host signatures after probing (0 = unknown).
-	GroupCount int
-
-	// TechFilter enables product/macro tag filtering after cheap HTTP fingerprinting.
-	TechFilter bool
-	// TechBoundTemplates is how many templates carry product/macro tags.
-	TechBoundTemplates int
-}
-
-// Plan is the execution plan for a single scan pass.
-type Plan struct {
-	Strategy              string
-	BuildReachability     bool
-	ProbePorts            bool
-	UseReachabilityFilter bool
-	UseTechFilter         bool
-	ExpectedRequests      int64
-	Reason                string
-}
-
-// Decide chooses spray strategy and whether to spend budget on reachability probing.
-//
-// Hard bar: never choose work that is expected to regress wall clock vs baseline spray.
-// When probe cost would dominate and savings are uncertain, reachability is skipped.
-func Decide(in Input) Plan {
-	p := Plan{
-		Strategy:         chooseStrategy(in),
-		ExpectedRequests: int64(max(in.Requests, 0)),
-	}
-
-	reachReason := "reachability=off"
-	if in.StrictProbe {
-		reach := decideReachability(in)
-		p.BuildReachability = reach.build
-		p.ProbePorts = reach.probePorts
-		p.UseReachabilityFilter = reach.useFilter
-		reachReason = reach.reason
-	}
-
-	tech := decideTechFilter(in)
-	p.UseTechFilter = tech.use
-	p.Reason = strategyReason(in, p.Strategy) + "; " + reachReason + "; " + tech.reason
-
-	if in.FilteredRequests > 0 {
-		p.ExpectedRequests = int64(in.FilteredRequests)
-	}
-	return p
-}
-
-func chooseStrategy(in Input) string {
-	if in.Stream || in.Hosts <= 0 || in.Templates <= 0 {
-		return scanstrategy.TemplateSpray.String()
-	}
-	bulk := in.BulkSize
-	if bulk <= 0 {
-		bulk = 25
-	}
-	// Host-spray only when the shape strongly favors per-host locality.
-	// Conservative default is template-spray (historical auto) so wall clock
-	// does not regress on small / mixed benches.
-	if in.Hosts <= bulk && in.Hosts <= 10 && in.Templates >= 10*in.Hosts && in.Requests >= 5000 {
-		return scanstrategy.HostSpray.String()
-	}
-	return scanstrategy.TemplateSpray.String()
-}
-
-func strategyReason(in Input, strategy string) string {
-	return fmt.Sprintf("strategy=%s hosts=%d templates=%d", strategy, in.Hosts, in.Templates)
-}
-
-type reachabilityDecision struct {
-	build      bool
-	probePorts bool
-	useFilter  bool
-	reason     string
-}
-
-func decideReachability(in Input) reachabilityDecision {
-	if in.Hosts <= 0 || in.Templates <= 0 {
-		return reachabilityDecision{reason: "reachability=skipped-empty"}
-	}
-
-	probeCost := in.PortsToProbe * in.Hosts
-	baseline := in.Requests
-	if baseline <= 0 {
-		baseline = in.Templates * in.Hosts
-	}
-
-	// No concrete network ports to probe: HTTP-only index is cheap (reuse httpx map).
-	if in.PortsToProbe == 0 || in.ConcreteNetworkTemplates == 0 {
-		return reachabilityDecision{
-			build:      true,
-			probePorts: false,
-			useFilter:  true,
-			reason:     "reachability=http-only",
-		}
-	}
-
-	// Probe budget: if dialing every port×host costs more than the scan itself
-	// and we have no signal of a mixed fleet, skip probing to match baseline time.
-	if probeCost > baseline && in.GroupCount <= 1 {
-		// GroupCount==0 (unknown) or 1 (homogeneous): do not pay for probes.
-		if probeCost > max(baseline*2, 500) {
-			return reachabilityDecision{
-				reason: fmt.Sprintf("reachability=skipped-probe-budget cost=%d baseline=%d", probeCost, baseline),
-			}
-		}
-	}
-
-	// After probing, homogeneous fleets still benefit from a cheap filter (no extra dials
-	// during spray for closed-port templates that global prune may have kept).
-	return reachabilityDecision{
-		build:      true,
-		probePorts: true,
-		useFilter:  true,
-		reason:     fmt.Sprintf("reachability=full probe_cost=%d", probeCost),
-	}
-}
-
-type techFilterDecision struct {
-	use    bool
-	reason string
-}
-
-func decideTechFilter(in Input) techFilterDecision {
-	// Opt-in only: never enable from strategy/reachability alone.
-	if !in.TechFilter {
-		return techFilterDecision{reason: "tech-filter=off"}
-	}
-	// Skip fingerprinting when nothing is tech-bound — otherwise we add
-	// probe latency with zero savings (wall-clock regression).
-	if in.TechBoundTemplates <= 0 {
-		return techFilterDecision{reason: "tech-filter=skipped-no-bound-templates"}
-	}
-	return techFilterDecision{use: true, reason: fmt.Sprintf("tech-filter=on bound=%d", in.TechBoundTemplates)}
-}
-
-// ApplyFiltered updates ExpectedRequests once post-probe estimates are known.
-func (p *Plan) ApplyFiltered(filtered int) {
-	if filtered > 0 {
-		p.ExpectedRequests = int64(filtered)
-	}
-}

pkg/core/plan/plan_test.go

@@ -1,205 +0,0 @@
-package plan
-
-import (
-	"testing"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/types/scanstrategy"
-	"github.com/stretchr/testify/require"
-)
-
-func TestDecideStrategyHostSpray(t *testing.T) {
-	p := Decide(Input{
-		Hosts:     5,
-		Templates: 200,
-		Requests:  10000,
-		BulkSize:  25,
-	})
-	require.Equal(t, scanstrategy.HostSpray.String(), p.Strategy)
-	require.Contains(t, p.Reason, "strategy=host-spray")
-}
-
-func TestDecideStrategyTemplateSpraySmallBench(t *testing.T) {
-	p := Decide(Input{
-		Hosts:     5,
-		Templates: 100,
-		Requests:  500,
-		BulkSize:  25,
-	})
-	require.Equal(t, scanstrategy.TemplateSpray.String(), p.Strategy)
-}
-
-func TestDecideStrategyTemplateSprayManyHosts(t *testing.T) {
-	p := Decide(Input{
-		Hosts:     1000,
-		Templates: 50,
-		Requests:  50000,
-		BulkSize:  25,
-	})
-	require.Equal(t, scanstrategy.TemplateSpray.String(), p.Strategy)
-}
-
-func TestDecideStrategyStreamDefaultsTemplateSpray(t *testing.T) {
-	p := Decide(Input{
-		Hosts:     5,
-		Templates: 100,
-		Stream:    true,
-		BulkSize:  25,
-	})
-	require.Equal(t, scanstrategy.TemplateSpray.String(), p.Strategy)
-}
-
-func TestDecideReachabilityOffWithoutStrictProbe(t *testing.T) {
-	p := Decide(Input{
-		Hosts:        5,
-		Templates:    60,
-		Requests:     300,
-		StrictProbe:  false,
-		PortsToProbe: 3,
-	})
-	require.False(t, p.BuildReachability)
-	require.False(t, p.UseReachabilityFilter)
-	require.Contains(t, p.Reason, "reachability=off")
-}
-
-func TestDecideReachabilityHTTPOnly(t *testing.T) {
-	p := Decide(Input{
-		Hosts:                    5,
-		Templates:                60,
-		Requests:                 300,
-		StrictProbe:              true,
-		PortsToProbe:             0,
-		ConcreteNetworkTemplates: 0,
-	})
-	require.True(t, p.BuildReachability)
-	require.False(t, p.ProbePorts)
-	require.True(t, p.UseReachabilityFilter)
-	require.Contains(t, p.Reason, "reachability=http-only")
-}
-
-func TestDecideReachabilityProbeBudgetSkip(t *testing.T) {
-	p := Decide(Input{
-		Hosts:                    100,
-		Templates:                10,
-		Requests:                 1000,
-		StrictProbe:              true,
-		PortsToProbe:             50, // cost = 5000 > 2*baseline
-		ConcreteNetworkTemplates: 5,
-		GroupCount:               0,
-	})
-	require.False(t, p.BuildReachability)
-	require.False(t, p.UseReachabilityFilter)
-	require.Contains(t, p.Reason, "reachability=skipped-probe-budget")
-}
-
-func TestDecideReachabilityProbeWithinBudget(t *testing.T) {
-	p := Decide(Input{
-		Hosts:                    100,
-		Templates:                10,
-		Requests:                 1000,
-		StrictProbe:              true,
-		PortsToProbe:             15, // cost = 1500: above baseline, within 2*baseline
-		ConcreteNetworkTemplates: 5,
-		GroupCount:               0,
-	})
-	require.True(t, p.BuildReachability)
-	require.True(t, p.ProbePorts)
-	require.True(t, p.UseReachabilityFilter)
-	require.Contains(t, p.Reason, "reachability=full")
-}
-
-func TestDecideReachabilityFull(t *testing.T) {
-	p := Decide(Input{
-		Hosts:                    5,
-		Templates:                60,
-		Requests:                 300,
-		StrictProbe:              true,
-		PortsToProbe:             3, // cost = 15 << baseline
-		ConcreteNetworkTemplates: 30,
-	})
-	require.True(t, p.BuildReachability)
-	require.True(t, p.ProbePorts)
-	require.True(t, p.UseReachabilityFilter)
-	require.Contains(t, p.Reason, "reachability=full")
-}
-
-func TestDecideReachabilityEmptyInput(t *testing.T) {
-	p := Decide(Input{StrictProbe: true, Hosts: 0, Templates: 10})
-	require.False(t, p.BuildReachability)
-	require.Contains(t, p.Reason, "reachability=skipped-empty")
-}
-
-func TestDecideReachabilityConcreteZeroUsesHTTPOnly(t *testing.T) {
-	p := Decide(Input{
-		Hosts:                    20,
-		Templates:                100,
-		Requests:                 2000,
-		StrictProbe:              true,
-		PortsToProbe:             5,
-		ConcreteNetworkTemplates: 0, // web-only set
-	})
-	require.True(t, p.BuildReachability)
-	require.False(t, p.ProbePorts)
-	require.True(t, p.UseReachabilityFilter)
-}
-
-func TestDecideHostSprayRequiresLargeRequestBudget(t *testing.T) {
-	// hosts small + many templates but requests below threshold → template-spray
-	p := Decide(Input{
-		Hosts:     5,
-		Templates: 200,
-		Requests:  4999,
-		BulkSize:  25,
-	})
-	require.Equal(t, scanstrategy.TemplateSpray.String(), p.Strategy)
-}
-
-func TestDecideApplyFilteredIgnoredWhenZero(t *testing.T) {
-	p := Decide(Input{
-		Hosts:            5,
-		Templates:        10,
-		Requests:         50,
-		StrictProbe:      false,
-		FilteredRequests: 0,
-	})
-	require.Equal(t, int64(50), p.ExpectedRequests)
-	p.ApplyFiltered(0)
-	require.Equal(t, int64(50), p.ExpectedRequests)
-	p.ApplyFiltered(12)
-	require.Equal(t, int64(12), p.ExpectedRequests)
-}
-
-func TestDecideTechFilterOffByDefault(t *testing.T) {
-	p := Decide(Input{
-		Hosts:              5,
-		Templates:          100,
-		Requests:           500,
-		TechFilter:         false,
-		TechBoundTemplates: 40,
-	})
-	require.False(t, p.UseTechFilter)
-	require.Contains(t, p.Reason, "tech-filter=off")
-}
-
-func TestDecideTechFilterSkippedWhenNoBound(t *testing.T) {
-	p := Decide(Input{
-		Hosts:              5,
-		Templates:          100,
-		Requests:           500,
-		TechFilter:         true,
-		TechBoundTemplates: 0,
-	})
-	require.False(t, p.UseTechFilter)
-	require.Contains(t, p.Reason, "tech-filter=skipped-no-bound-templates")
-}
-
-func TestDecideTechFilterOn(t *testing.T) {
-	p := Decide(Input{
-		Hosts:              5,
-		Templates:          100,
-		Requests:           500,
-		TechFilter:         true,
-		TechBoundTemplates: 40,
-	})
-	require.True(t, p.UseTechFilter)
-	require.Contains(t, p.Reason, "tech-filter=on")
-}

pkg/core/techfilter/techfilter.go

@@ -1,343 +0,0 @@
-// Package techfilter maps Wappalyzer products/categories to template tags and
-// decides whether a template can produce a finding on a fingerprinted host.
-//
-// Fail-open rules (coverage over aggressiveness):
-//   - templates with no product/macro tags always run
-//   - hosts with no fingerprints always allow every template
-//   - CDN/WAF-only fingerprints do not suppress product templates
-//   - non-HTTP templates are not filtered here (reachability owns that)
-package techfilter
-
-import (
-	"encoding/json"
-	"strings"
-	"sync"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
-	wappalyzer "github.com/projectdiscovery/wappalyzergo"
-)
-
-// genericTags are common nuclei tags that do not bind a template to a product.
-var genericTags = map[string]struct{}{
-	"cve": {}, "cnvd": {}, "edb": {}, "kev": {}, "vkev": {},
-	"misconfig": {}, "exposure": {}, "config": {}, "disclosure": {},
-	"tech": {}, "detect": {}, "fingerprint": {}, "favicon": {}, "waf": {},
-	"http": {}, "network": {}, "dns": {}, "ssl": {}, "tls": {},
-	"websocket": {}, "headless": {}, "file": {}, "code": {}, "javascript": {},
-	"workflow": {}, "intrusive": {}, "dos": {}, "fuzz": {}, "token-spray": {},
-	"osint": {}, "panel": {}, "login": {}, "auth": {}, "unauth": {},
-	"default-login": {}, "vuln": {}, "rce": {}, "lfi": {}, "sqli": {},
-	"xss": {}, "ssrf": {}, "xxe": {}, "ssti": {}, "idor": {},
-	"critical": {}, "high": {}, "medium": {}, "low": {}, "info": {},
-	"unknown": {}, "generic": {}, "misc": {}, "bench": {},
-	// Meta / source / taxonomy tags that are not products.
-	"oast": {}, "takeover": {}, "deserialization": {}, "packetstorm": {},
-	"seclists": {}, "hackerone": {}, "huntr": {}, "wp-plugin": {}, "wp-theme": {},
-	"wordpress-plugin": {}, "wordpress-theme": {}, "authenticated": {},
-	"unauthenticated": {}, "bypass": {}, "injection": {}, "traversal": {},
-	"redirect": {}, "crlf": {}, "csrf": {}, "cors": {},
-}
-
-// weakFingerprintTags alone must not suppress product-bound templates
-// (e.g. cloudflare/cdn-only fingerprints).
-var weakFingerprintTags = map[string]struct{}{
-	"cdn": {}, "waf": {}, "misc": {}, "security": {}, "caching": {},
-}
-
-// productAliases normalize common nuclei / wappalyzer spellings to one token.
-var productAliases = map[string]string{
-	"wp":                 "wordpress",
-	"httpd":              "apache",
-	"apache-http-server": "apache",
-}
-
-// categoryMacros maps Wappalyzer category names to short macro tags.
-var categoryMacros = map[string]string{
-	"cms":                   "cms",
-	"ecommerce":             "ecommerce",
-	"web servers":           "webserver",
-	"web-servers":           "webserver",
-	"web frameworks":        "web-framework",
-	"web-frameworks":        "web-framework",
-	"javascript frameworks": "js-framework",
-	"javascript-frameworks": "js-framework",
-	"javascript libraries":  "js-library",
-	"javascript-libraries":  "js-library",
-	"cdn":                   "cdn",
-	"databases":             "database",
-	"programming languages": "language",
-	"programming-languages": "language",
-	"paas":                  "paas",
-	"saas":                  "saas",
-	"security":              "security",
-	"caching":               "caching",
-	"miscellaneous":         "misc",
-}
-
-// HostProfile is the set of product + macro tags inferred for one target.
-type HostProfile struct {
-	Tags map[string]struct{} // empty / nil => unknown => fail-open
-
-	// Cats are the Wappalyzer category ids of every detected product. They
-	// record what the fingerprint actually has evidence about, which is what
-	// makes it safe to rule a template out. See AllowTags.
-	Cats map[int]struct{}
-}
-
-// HasTags reports whether fingerprinting produced at least one tag.
-func (h HostProfile) HasTags() bool {
-	return len(h.Tags) > 0
-}
-
-// HasSubstantiveTags reports whether the profile has a strong category macro
-// (cms, webserver, …). CDN/WAF-only fingerprints are not substantive.
-func (h HostProfile) HasSubstantiveTags() bool {
-	for tag := range h.Tags {
-		if isStrongMacro(tag) {
-			return true
-		}
-	}
-	return false
-}
-
-func isStrongMacro(tag string) bool {
-	if _, weak := weakFingerprintTags[tag]; weak {
-		return false
-	}
-	for _, m := range categoryMacros {
-		if tag == m {
-			return true
-		}
-	}
-	return false
-}
-
-// NormalizeProduct turns a Wappalyzer / nuclei product name into a tag token.
-func NormalizeProduct(name string) string {
-	name = strings.TrimSpace(strings.ToLower(name))
-	if name == "" {
-		return ""
-	}
-	if i := strings.IndexByte(name, ':'); i > 0 {
-		name = name[:i]
-	}
-	name = strings.ReplaceAll(name, "_", "-")
-	name = strings.Join(strings.Fields(name), "-")
-	if alias, ok := productAliases[name]; ok {
-		return alias
-	}
-	return name
-}
-
-// NormalizeCategory turns a Wappalyzer category into a macro tag.
-func NormalizeCategory(cat string) string {
-	key := strings.ToLower(strings.TrimSpace(cat))
-	if key == "" {
-		return ""
-	}
-	if m, ok := categoryMacros[key]; ok {
-		return m
-	}
-	return strings.ReplaceAll(key, " ", "-")
-}
-
-// IsGenericTag reports whether tag is not product/macro binding.
-func IsGenericTag(tag string) bool {
-	tag = strings.ToLower(strings.TrimSpace(tag))
-	if tag == "" {
-		return true
-	}
-	if _, ok := genericTags[tag]; ok {
-		return true
-	}
-	// cve2021 / cve-2021 style year tags bind no product.
-	if strings.HasPrefix(tag, "cve") {
-		rest := strings.TrimPrefix(tag, "cve")
-		rest = strings.TrimPrefix(rest, "-")
-		if rest != "" && isAllDigits(rest) {
-			return true
-		}
-	}
-	return false
-}
-
-func isAllDigits(s string) bool {
-	if s == "" {
-		return false
-	}
-	for _, r := range s {
-		if r < '0' || r > '9' {
-			return false
-		}
-	}
-	return true
-}
-
-// ProfileFromFingerprint builds a host profile from products and their categories.
-// products maps product name -> category names from Wappalyzer.
-func ProfileFromFingerprint(products map[string][]string) HostProfile {
-	tags := make(map[string]struct{})
-	observed := make(map[int]struct{})
-	byProduct := productCategories()
-	for product, cats := range products {
-		if p := NormalizeProduct(product); p != "" {
-			tags[p] = struct{}{}
-			for _, id := range byProduct[p] {
-				observed[id] = struct{}{}
-			}
-		}
-		for _, c := range cats {
-			if m := NormalizeCategory(c); m != "" {
-				tags[m] = struct{}{}
-			}
-		}
-	}
-	return HostProfile{Tags: tags, Cats: observed}
-}
-
-// TemplateProductTags returns product/macro tags that bind a template to a stack.
-// Empty means the template is generic and must always be allowed.
-func TemplateProductTags(t *templates.Template) []string {
-	if t == nil {
-		return nil
-	}
-	raw := t.Info.Tags.ToSlice()
-	out := make([]string, 0, len(raw))
-	seen := make(map[string]struct{}, len(raw))
-	known := knownFingerprintTokens()
-	for _, tag := range raw {
-		tag = NormalizeProduct(tag)
-		if tag == "" || IsGenericTag(tag) {
-			continue
-		}
-		// A tag binds only when fingerprinting can actually emit it. Anything
-		// else would never match a profile and would prune the template on
-		// every host, so it must leave the template unbound.
-		if _, ok := known[tag]; !ok {
-			continue
-		}
-		if _, ok := seen[tag]; ok {
-			continue
-		}
-		seen[tag] = struct{}{}
-		out = append(out, tag)
-	}
-	return out
-}
-
-// knownFingerprintTokens is every token a host profile can contain: the
-// normalized name of every Wappalyzer app plus every category macro.
-//
-// This is an allowlist on purpose. nuclei-templates carries thousands of tags
-// that name no product at all (discovery, wpscan, top-200, auth-bypass, ...).
-// Treating an unrecognized tag as a product binds the template to something no
-// fingerprint can ever report, which silently drops it on every host. Unknown
-// tags must therefore bind nothing.
-var knownFingerprintTokens = sync.OnceValue(func() map[string]struct{} {
-	var data struct {
-		Apps map[string]json.RawMessage `json:"apps"`
-	}
-	tokens := make(map[string]struct{}, 8192)
-	if err := json.Unmarshal([]byte(wappalyzer.GetFingerprints()), &data); err == nil {
-		for name := range data.Apps {
-			if p := NormalizeProduct(name); p != "" {
-				tokens[p] = struct{}{}
-			}
-		}
-	}
-	for _, macro := range categoryMacros {
-		tokens[macro] = struct{}{}
-	}
-	for tag := range weakFingerprintTags {
-		tokens[tag] = struct{}{}
-	}
-	return tokens
-})
-
-// IsTechBound reports whether the template is constrained to a product/macro.
-func IsTechBound(t *templates.Template) bool {
-	return len(TemplateProductTags(t)) > 0
-}
-
-// Allow reports whether template may run on a host with the given profile.
-// Fail-open when the host was not fingerprinted or the template is unbound.
-func Allow(profile HostProfile, t *templates.Template) bool {
-	return AllowTags(profile, TemplateProductTags(t))
-}
-
-// AllowTags is Allow with precomputed product tags (avoids per-pair allocation).
-func AllowTags(profile HostProfile, bound []string) bool {
-	if len(bound) == 0 {
-		return true
-	}
-	if !profile.HasTags() || !profile.HasSubstantiveTags() {
-		return true
-	}
-	for _, tag := range bound {
-		if _, ok := profile.Tags[tag]; ok {
-			return true
-		}
-	}
-
-	// No positive match. Ruling the template out now would assume the
-	// fingerprint is complete, and it never is: it is one page of one host, and
-	// a stack is routinely invisible from it (WordPress behind nginx is the
-	// ordinary case, not the exception). Absence of evidence is not evidence of
-	// absence, so a template is dropped only when the fingerprint positively
-	// contradicts it: something else was identified in the same category, and
-	// the template's own product was not. If any bound product sits in a
-	// category the fingerprint says nothing about, the template runs.
-	byProduct := productCategories()
-	for _, tag := range bound {
-		cats, ok := byProduct[tag]
-		if !ok || len(cats) == 0 {
-			return true // category macro or unknown product: no contradiction possible
-		}
-		contradicted := false
-		for _, id := range cats {
-			if _, seen := profile.Cats[id]; seen {
-				contradicted = true
-				break
-			}
-		}
-		if !contradicted {
-			return true
-		}
-	}
-	return false
-}
-
-// productCategories maps a normalized product token to its Wappalyzer category
-// ids. It is the evidence index: a category id present in a host profile means
-// fingerprinting actually identified something of that kind on that host.
-var productCategories = sync.OnceValue(func() map[string][]int {
-	var data struct {
-		Apps map[string]struct {
-			Cats []int `json:"cats"`
-		} `json:"apps"`
-	}
-	out := make(map[string][]int, 8192)
-	if err := json.Unmarshal([]byte(wappalyzer.GetFingerprints()), &data); err != nil {
-		return out
-	}
-	for name, app := range data.Apps {
-		p := NormalizeProduct(name)
-		if p == "" || len(app.Cats) == 0 {
-			continue
-		}
-		// Aliases collapse several apps onto one token; union their categories.
-		out[p] = append(out[p], app.Cats...)
-	}
-	return out
-})
-
-// CountTechBound returns how many templates carry product/macro tags.
-func CountTechBound(tpls []*templates.Template) int {
-	n := 0
-	for _, t := range tpls {
-		if IsTechBound(t) {
-			n++
-		}
-	}
-	return n
-}

pkg/core/techfilter/techfilter_test.go

@@ -1,208 +0,0 @@
-package techfilter
-
-import (
-	"testing"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/model"
-	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/stringslice"
-	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
-)
-
-func TestNormalizeProductAndCategory(t *testing.T) {
-	if got := NormalizeProduct("Nginx:1.25"); got != "nginx" {
-		t.Fatalf("NormalizeProduct = %q", got)
-	}
-	if got := NormalizeProduct("wp"); got != "wordpress" {
-		t.Fatalf("alias = %q", got)
-	}
-	if got := NormalizeCategory("Web servers"); got != "webserver" {
-		t.Fatalf("category = %q", got)
-	}
-}
-
-func TestProfileFromFingerprint(t *testing.T) {
-	p := ProfileFromFingerprint(map[string][]string{
-		"Nginx":     {"Web servers"},
-		"WordPress": {"CMS"},
-	})
-	for _, want := range []string{"nginx", "webserver", "wordpress", "cms"} {
-		if _, ok := p.Tags[want]; !ok {
-			t.Fatalf("missing tag %q in %#v", want, p.Tags)
-		}
-	}
-}
-
-func tagged(tags ...string) *templates.Template {
-	return &templates.Template{
-		Info: model.Info{Tags: stringslice.StringSlice{Value: tags}},
-	}
-}
-
-func TestAllowFailOpenAndMatch(t *testing.T) {
-	nginx := ProfileFromFingerprint(map[string][]string{"Nginx": {"Web servers"}})
-	empty := HostProfile{}
-
-	generic := tagged("misconfig", "http")
-	wp := tagged("cve", "wordpress")
-	ngx := tagged("nginx", "misconfig")
-	cms := tagged("cms")
-
-	if !Allow(nginx, generic) {
-		t.Fatal("generic must always run")
-	}
-	if !Allow(empty, wp) {
-		t.Fatal("unknown host must fail-open for bound templates")
-	}
-	// An nginx fingerprint says nothing about the CMS behind it, and WordPress
-	// behind nginx is an ordinary stack. Absence of a CMS in the profile is not
-	// evidence that there is none.
-	if !Allow(nginx, wp) {
-		t.Fatal("nginx says nothing about a CMS, so a wordpress template must run")
-	}
-	if !Allow(nginx, ngx) {
-		t.Fatal("nginx template must run on nginx host")
-	}
-	if !Allow(nginx, cms) {
-		t.Fatal("nginx says nothing about a CMS, so a cms-bound template must run")
-	}
-	wpHost := ProfileFromFingerprint(map[string][]string{"WordPress": {"CMS"}})
-	if !Allow(wpHost, cms) {
-		t.Fatal("cms macro must match wordpress host")
-	}
-	if !Allow(wpHost, wp) {
-		t.Fatal("wordpress product must match")
-	}
-}
-
-func TestMetaTagsAreNotTechBound(t *testing.T) {
-	cases := [][]string{
-		{"cve", "cve2021", "oast", "rce"},
-		{"takeover", "dns"},
-		{"hackerone", "packetstorm", "seclists"},
-		{"wp-plugin", "wordpress-plugin"},
-	}
-	for _, tags := range cases {
-		tpl := tagged(tags...)
-		if IsTechBound(tpl) {
-			t.Fatalf("tags %v must not be tech-bound, got %v", tags, TemplateProductTags(tpl))
-		}
-		cdnOnly := ProfileFromFingerprint(map[string][]string{"Cloudflare": {"CDN"}})
-		if !Allow(cdnOnly, tpl) {
-			t.Fatalf("unbound meta tags %v must run on any host", tags)
-		}
-	}
-}
-
-func TestCDNOnlyFingerprintFailOpen(t *testing.T) {
-	cdnOnly := ProfileFromFingerprint(map[string][]string{"Cloudflare": {"CDN"}})
-	if cdnOnly.HasSubstantiveTags() {
-		t.Fatalf("cdn-only profile should not be substantive: %#v", cdnOnly.Tags)
-	}
-	wp := tagged("wordpress")
-	if !Allow(cdnOnly, wp) {
-		t.Fatal("CDN-only fingerprint must fail-open for product templates")
-	}
-}
-
-func TestCountTechBound(t *testing.T) {
-	n := CountTechBound([]*templates.Template{
-		tagged("misconfig"),
-		tagged("wordpress"),
-		tagged("nginx", "http"),
-		tagged("cve2023", "oast"),
-	})
-	if n != 2 {
-		t.Fatalf("CountTechBound = %d", n)
-	}
-}
-
-// A tag only binds when fingerprinting can emit it. nuclei-templates carries
-// thousands of tags that name no product; binding to one of those prunes the
-// template on every host, because no profile can ever contain it.
-func TestOnlyKnownFingerprintTokensBind(t *testing.T) {
-	nonProducts := [][]string{
-		{"misconfig", "headers", "generic", "vuln"}, // http-missing-security-headers
-		{"tech", "discovery"},                       // tech-detect
-		{"waf", "tech", "misc", "discovery"},        // waf-detect
-		{"wpscan"},
-		{"top-100"}, {"top-200"},
-		{"auth-bypass"}, {"time-based-sqli"},
-		{"install"}, {"files"}, {"file-upload"},
-		{"phishing"}, {"osint-social"},
-		{"debug"}, {"devops"}, {"token"}, {"tokens"}, {"fpd"}, {"oss"},
-	}
-	nginx := ProfileFromFingerprint(map[string][]string{"Nginx": {"Web servers"}})
-	for _, tags := range nonProducts {
-		tpl := tagged(tags...)
-		if got := TemplateProductTags(tpl); len(got) != 0 {
-			t.Fatalf("tags %v must not bind, got %v", tags, got)
-		}
-		if !Allow(nginx, tpl) {
-			t.Fatalf("unbound template %v must run on every host", tags)
-		}
-	}
-}
-
-// The inversion must not disable the filter: real products still bind, and a
-// template for another stack is still pruned.
-func TestKnownProductsStillBind(t *testing.T) {
-	for _, tags := range [][]string{{"wordpress"}, {"apache"}, {"joomla"}, {"cms"}} {
-		if got := TemplateProductTags(tagged(tags...)); len(got) == 0 {
-			t.Fatalf("tags %v name a product and must bind", tags)
-		}
-	}
-
-	wp := ProfileFromFingerprint(map[string][]string{"WordPress": {"CMS"}})
-	if !Allow(wp, tagged("wordpress", "rce")) {
-		t.Fatal("a wordpress template must run on a wordpress profile")
-	}
-}
-
-// A template is dropped only when the fingerprint contradicts it: it identified
-// something else of the same kind and not the template's own product.
-func TestPruneOnlyOnPositiveContradiction(t *testing.T) {
-	nginx := ProfileFromFingerprint(map[string][]string{"Nginx": {"Web servers"}})
-	wpHost := ProfileFromFingerprint(map[string][]string{
-		"WordPress": {"CMS"}, "Nginx": {"Web servers"},
-	})
-
-	// Contradiction: a web server was identified and it is not apache.
-	if Allow(nginx, tagged("apache")) {
-		t.Fatal("nginx was identified, so an apache template is contradicted")
-	}
-	// Contradiction: a CMS was identified and it is not joomla.
-	if Allow(wpHost, tagged("joomla")) {
-		t.Fatal("wordpress was identified, so a joomla template is contradicted")
-	}
-	// No evidence: nothing of that kind was identified at all.
-	for _, tags := range [][]string{{"wordpress"}, {"joomla"}, {"cms"}, {"mysql"}} {
-		if !Allow(nginx, tagged(tags...)) {
-			t.Fatalf("nginx gives no evidence about %v, so it must run", tags)
-		}
-	}
-	// One uncontradicted product is enough to keep a multi-product template.
-	if !Allow(nginx, tagged("apache", "wordpress")) {
-		t.Fatal("wordpress is uncontradicted, so the template must run")
-	}
-	// A positive match always wins.
-	if !Allow(wpHost, tagged("wordpress")) {
-		t.Fatal("a positive match must run")
-	}
-}
-
-func TestKnownFingerprintTokensVocabulary(t *testing.T) {
-	tokens := knownFingerprintTokens()
-	if len(tokens) < 5000 {
-		t.Fatalf("expected the wappalyzer vocabulary, got %d tokens", len(tokens))
-	}
-	for _, want := range []string{"wordpress", "nginx", "apache", "cms", "webserver"} {
-		if _, ok := tokens[want]; !ok {
-			t.Fatalf("token %q missing from the vocabulary", want)
-		}
-	}
-	for _, bad := range []string{"discovery", "headers", "wpscan", "top-200"} {
-		if _, ok := tokens[bad]; ok {
-			t.Fatalf("token %q is not a product and must not be in the vocabulary", bad)
-		}
-	}
-}

pkg/input/transform.go

@@ -16,14 +16,6 @@ import (
 // Helper is a structure for helping with input transformation
 type Helper struct {
 	InputsHTTP *hybrid.HybridMap
-	// InputsHTTPProbed is true when InputsHTTP was filled by httpx probing.
-	// False means probing was skipped (e.g. MultiFormat) and map absence must
-	// not be treated as "not HTTP".
-	InputsHTTPProbed bool
-	// StrictProbe, when set, makes web-template input resolution skip inputs that
-	// were probed and found NOT to be HTTP services (instead of falling back to
-	// a raw URL). Lossless: a non-HTTP port has no HTTP application to find.
-	StrictProbe bool
 }
 
 // NewHelper returns a new input helper instance
@@ -148,12 +140,6 @@ func (h *Helper) convertInputToType(input string, inputType inputType, defaultPo
 			if probed, ok := h.InputsHTTP.Get(input); ok {
 				return string(probed)
 			}
-			// Strict reachability: this input was probed and is not an HTTP
-			// service, so skip it for web templates rather than falling back to
-			// a raw URL. Lossless — a non-HTTP port has no HTTP app to match.
-			if h.StrictProbe {
-				return ""
-			}
 		}
 		// try to parse it as absolute url and return
 		if absUrl, err := urlutil.ParseAbsoluteURL(input, false); err == nil {

pkg/input/transform_test.go

@@ -91,32 +91,3 @@ func TestConvertInputToType(t *testing.T) {
 		require.Equal(t, test.result, result, "could not get correct result %+v", test)
 	}
 }
-
-func TestStrictProbeSkipsUnconfirmedHTTPInput(t *testing.T) {
-	hm, err := hybrid.New(hybrid.DefaultDiskOptions)
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = hm.Close() })
-
-	_ = hm.Set("ok.example", []byte("https://ok.example"))
-
-	helper := &Helper{InputsHTTP: hm, StrictProbe: true}
-
-	// confirmed host -> resolved URL
-	require.Equal(t, "https://ok.example", helper.convertInputToType("ok.example", typeURL, ""))
-	// probed but non-HTTP hosts -> skipped (no raw fallback)
-	require.Equal(t, "", helper.convertInputToType("127.0.0.1:3306", typeURL, ""))
-	require.Equal(t, "", helper.convertInputToType("zombie.example:6379", typeURL, ""))
-	// already-URL inputs are not gated by probe results
-	require.Equal(t, "https://direct.example", helper.convertInputToType("https://direct.example", typeURL, ""))
-	// non-HTTP protocols still transform normally
-	require.Equal(t, "127.0.0.1:3306", helper.convertInputToType("127.0.0.1:3306", typeHostWithOptionalPort, ""))
-}
-
-func TestStrictProbeDisabledKeepsRawFallback(t *testing.T) {
-	hm, err := hybrid.New(hybrid.DefaultDiskOptions)
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = hm.Close() })
-
-	helper := &Helper{InputsHTTP: hm, StrictProbe: false}
-	require.Equal(t, "127.0.0.1:3306", helper.convertInputToType("127.0.0.1:3306", typeURL, ""))
-}

pkg/protocols/common/protocolstate/memoizer.go

@@ -4,15 +4,11 @@ import (
 	"github.com/projectdiscovery/utils/memoize"
 )
 
-// memoizerMaxSize is the process-wide @memo helper cache capacity.
-// Keys are xxhash uint64s of the arg string; values are the helper results.
-const memoizerMaxSize = 10_000
-
 var Memoizer *memoize.Memoizer
 
 func init() {
 	var err error
-	Memoizer, err = memoize.New(memoize.WithMaxSize(memoizerMaxSize))
+	Memoizer, err = memoize.New(memoize.WithMaxSize(1500))
 	if err != nil {
 		panic(err)
 	}

pkg/protocols/headless/engine/engine.go

@@ -1,6 +1,7 @@
 package engine
 
 import (
+	"context"
 	"fmt"
 	"net/http"
 	"os"
@@ -12,7 +13,9 @@ import (
 	"github.com/go-rod/rod/lib/launcher/flags"
 	"github.com/pkg/errors"
 
+	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	"github.com/projectdiscovery/utils/chromeshell"
 	fileutil "github.com/projectdiscovery/utils/file"
 	osutils "github.com/projectdiscovery/utils/os"
 )
@@ -38,7 +41,7 @@ func New(options *types.Options) (*Browser, error) {
 	var proxyServer *headlessProxy
 	var err error
 
-	chromeLauncher := launcher.New()
+	var chromeLauncher *launcher.Launcher
 
 	if options.CDPEndpoint == "" {
 		dataStore, err = os.MkdirTemp("", "nuclei-*")
@@ -46,58 +49,51 @@ func New(options *types.Options) (*Browser, error) {
 			return nil, errors.Wrap(err, "could not create temporary directory")
 		}
 
-		chromeLauncher = chromeLauncher.
-			Leakless(false).
-			Set("disable-crash-reporter").
-			Set("disable-gpu").
-			Set("disable-notifications").
-			Set("hide-scrollbars").
-			Set("ignore-certificate-errors").
-			Set("ignore-ssl-errors").
-			Set("incognito").
-			Set("mute-audio").
-			Set("window-size", fmt.Sprintf("%d,%d", 1080, 1920)).
-			Delete("use-mock-keychain").
-			UserDataDir(dataStore)
+		newChromeLauncher := func(browserPath string) *launcher.Launcher {
+			configured := launcher.New().
+				Leakless(false).
+				Set("disable-crash-reporter").
+				Set("disable-gpu").
+				Set("disable-notifications").
+				Set("hide-scrollbars").
+				Set("ignore-certificate-errors").
+				Set("ignore-ssl-errors").
+				Set("incognito").
+				Set("mute-audio").
+				Set("window-size", fmt.Sprintf("%d,%d", 1080, 1920)).
+				Delete("use-mock-keychain").
+				UserDataDir(dataStore)
 
-		if MustDisableSandbox() {
-			chromeLauncher = chromeLauncher.NoSandbox(true)
+			if browserPath != "" {
+				configured.Bin(browserPath)
+			}
+			if MustDisableSandbox() {
+				configured.NoSandbox(true)
+			}
+			configured.Headless(!options.ShowBrowser)
+			configured.
+				Proxy(proxyServer.Addr()).
+				Set("proxy-bypass-list", "<-loopback>")
+			for k, v := range options.ParseHeadlessOptionalArguments() {
+				configured.Set(flags.Flag(k), v)
+			}
+			return configured
 		}
 
-		executablePath, err := os.Executable()
+		browserPath, isChromeShell, err := browserPath(options)
 		if err != nil {
 			return nil, err
 		}
-
-		// if musl is used, most likely we are on alpine linux which is not supported by go-rod, so we fallback to default chrome
-		useMusl, _ := fileutil.UseMusl(executablePath)
-		if options.UseInstalledChrome || useMusl {
-			if chromePath, hasChrome := launcher.LookPath(); hasChrome {
-				chromeLauncher.Bin(chromePath)
-			} else {
-				return nil, errors.New("the chrome browser is not installed")
-			}
-		}
-
-		if options.ShowBrowser {
-			chromeLauncher = chromeLauncher.Headless(false)
-		} else {
-			chromeLauncher = chromeLauncher.Headless(true)
-		}
 		proxyServer, err = newHeadlessProxy(options)
 		if err != nil {
 			return nil, err
 		}
-		chromeLauncher = chromeLauncher.
-			Proxy(proxyServer.Addr()).
-			Set("proxy-bypass-list", "<-loopback>")
-
-		for k, v := range options.ParseHeadlessOptionalArguments() {
-			chromeLauncher.Set(flags.Flag(k), v)
-		}
-
-		launcherURL, err = chromeLauncher.Launch()
+		launcherURL, err = launchBrowser(browserPath, isChromeShell, func(path string) (string, error) {
+			chromeLauncher = newChromeLauncher(path)
+			return chromeLauncher.Launch()
+		})
 		if err != nil {
+			_ = proxyServer.Close()
 			return nil, err
 		}
 	} else {
@@ -143,6 +139,60 @@ func New(options *types.Options) (*Browser, error) {
 	return engine, nil
 }
 
+// browserPath returns the browser binary to launch, empty for the rod default,
+// and whether that binary is chrome-headless-shell.
+func browserPath(options *types.Options) (string, bool, error) {
+	executablePath, err := os.Executable()
+	if err != nil {
+		return "", false, err
+	}
+
+	// if musl is used, most likely we are on alpine linux which is not supported by go-rod, so we fallback to default chrome
+	useMusl, _ := fileutil.UseMusl(executablePath)
+	if options.UseInstalledChrome || useMusl {
+		if chromePath, hasChrome := launcher.LookPath(); hasChrome {
+			return chromePath, false, nil
+		}
+		return "", false, errors.New("the chrome browser is not installed")
+	}
+	if options.ShowBrowser || !chromeshell.Supported() {
+		return "", false, nil
+	}
+
+	// Prefer chrome-headless-shell for local headless templates; skip it when
+	// headed since the shell binary cannot show a UI.
+	ctx, cancel := context.WithTimeout(context.Background(), chromeshell.DefaultDownloadTimeout)
+	defer cancel()
+	shellPath, err := chromeshell.EnsureWithOptions(ctx, chromeshell.Options{Proxy: downloadProxy(options)})
+	if err != nil {
+		gologger.Warning().Msgf("Could not prepare chrome-headless-shell, using the default browser: %s\n", err)
+		return "", false, nil
+	}
+	return shellPath, true, nil
+}
+
+// downloadProxy returns the scan proxy for the chrome-headless-shell download,
+// or empty to use the environment proxy.
+func downloadProxy(options *types.Options) string {
+	if options.AliveHttpProxy != "" {
+		return options.AliveHttpProxy
+	}
+	return options.AliveSocksProxy
+}
+
+// launchBrowser launches browserPath and, only when it is chrome-headless-shell,
+// retries once with the default browser. An installed Chrome that fails to
+// launch is returned as an error, as before.
+func launchBrowser(browserPath string, isChromeShell bool, launch func(string) (string, error)) (string, error) {
+	launcherURL, err := launch(browserPath)
+	if err == nil || !isChromeShell {
+		return launcherURL, err
+	}
+
+	gologger.Warning().Msgf("Could not launch chrome-headless-shell, using the default browser: %s\n", err)
+	return launch("")
+}
+
 // MustDisableSandbox determines if the current os and user needs sandbox mode disabled
 func MustDisableSandbox() bool {
 	// linux with root user needs "--no-sandbox" option

pkg/protocols/headless/engine/engine_test.go

@@ -0,0 +1,67 @@
+package engine
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLaunchBrowserFallsBackFromChromeShell(t *testing.T) {
+	var paths []string
+	got, err := launchBrowser("/cache/chrome-headless-shell", true, func(path string) (string, error) {
+		paths = append(paths, path)
+		if path != "" {
+			return "", errors.New("shell launch failed")
+		}
+		return "ws://default-browser", nil
+	})
+
+	require.NoError(t, err)
+	require.Equal(t, "ws://default-browser", got)
+	require.Equal(t, []string{"/cache/chrome-headless-shell", ""}, paths)
+}
+
+func TestLaunchBrowserDoesNotRetryDefaultBrowser(t *testing.T) {
+	var calls int
+	expected := errors.New("default launch failed")
+	_, err := launchBrowser("", false, func(string) (string, error) {
+		calls++
+		return "", expected
+	})
+
+	require.ErrorIs(t, err, expected)
+	require.Equal(t, 1, calls)
+}
+
+func TestLaunchBrowserDoesNotRetryInstalledChrome(t *testing.T) {
+	var paths []string
+	expected := errors.New("installed chrome launch failed")
+	_, err := launchBrowser("/usr/bin/google-chrome", false, func(path string) (string, error) {
+		paths = append(paths, path)
+		return "", expected
+	})
+
+	require.ErrorIs(t, err, expected)
+	require.Equal(t, []string{"/usr/bin/google-chrome"}, paths)
+}
+
+// The browser download goes through the scan proxy, or the environment proxy
+// when there is none.
+func TestDownloadProxy(t *testing.T) {
+	cases := []struct {
+		name string
+		opts *types.Options
+		want string
+	}{
+		{"no proxy uses the environment", &types.Options{}, ""},
+		{"http proxy", &types.Options{AliveHttpProxy: "http://127.0.0.1:8080"}, "http://127.0.0.1:8080"},
+		{"socks proxy", &types.Options{AliveSocksProxy: "socks5://127.0.0.1:1080"}, "socks5://127.0.0.1:1080"},
+	}
+	for _, c := range cases {
+		if got := downloadProxy(c.opts); got != c.want {
+			t.Errorf("%s: downloadProxy()=%q want %q", c.name, got, c.want)
+		}
+	}
+}

pkg/protocols/headless/engine/page_test.go

@@ -117,8 +117,10 @@ func TestRunReportsNavigationResponseWhenActionsFail(t *testing.T) {
 
 	var got *http.Response
 	actions := []*Action{{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}}
-	_, _, err = instance.Run(input, actions, nil, &Options{Timeout: 20 * time.Second, Options: opts, OnNavigationResponse: func(resp *http.Response) { got = resp }})
-	require.Error(t, err, "the browser fails a navigation to an empty error page")
+	// Full Chromium fails a navigation to an empty error page, while
+	// chrome-headless-shell renders it, so only the reported response is
+	// checked: it must arrive whether or not the actions fail.
+	_, _, _ = instance.Run(input, actions, nil, &Options{Timeout: 20 * time.Second, Options: opts, OnNavigationResponse: func(resp *http.Response) { got = resp }})
 	require.NotNil(t, got)
 	require.Equal(t, http.StatusTooManyRequests, got.StatusCode)
 	require.Equal(t, "7", got.Header.Get("Retry-After"))

pkg/protocols/http/build_request.go

@@ -7,7 +7,6 @@ import (
 	"net/http"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/projectdiscovery/useragent"
@@ -581,7 +580,8 @@ func (r *requestGenerator) fillRequest(req *retryablehttp.Request, values map[st
 	}
 
 	if !r.request.Unsafe {
-		httputil.SetHeader(req, "User-Agent", r.userAgent())
+		userAgent := useragent.PickRandom()
+		httputil.SetHeader(req, "User-Agent", userAgent.Raw)
 	}
 
 	// Only set these headers on non-raw requests
@@ -626,27 +626,3 @@ func (r *requestGenerator) renderText(text string, values map[string]interface{}
 
 	return result.Text, nil
 }
-
-// StableUserAgent is one Chrome user agent, chosen once per process. A random
-// user agent per request makes otherwise identical requests differ, so the
-// scan-scoped response cache can never reuse them. Whenever that cache may be
-// active we send one constant user agent instead, which also makes scans
-// reproducible. Anything that seeds the cache sends the same one, so a seeded
-// response is reachable by the templates that follow.
-var StableUserAgent = sync.OnceValue(func() string {
-	if agents, err := useragent.PickWithFilters(1, useragent.Chrome); err == nil && len(agents) > 0 {
-		return agents[0].Raw
-	}
-	return useragent.PickRandom().Raw
-})
-
-// userAgent returns the user agent for one request.
-func (r *requestGenerator) userAgent() string {
-	if r.request == nil || r.request.options == nil || r.request.options.Options == nil {
-		return useragent.PickRandom().Raw
-	}
-	if r.request.options.Options.UsesResponseCache() {
-		return StableUserAgent()
-	}
-	return useragent.PickRandom().Raw
-}

pkg/protocols/http/httprespcache/cache.go

@@ -1,503 +0,0 @@
-// Package httprespcache is a scan-scoped in-memory cache for safe HTTP GET/HEAD
-// responses. It lets tech fingerprinting and templates share the same RTT.
-package httprespcache
-
-import (
-	"bytes"
-	"io"
-	"net/http"
-	"net/http/cookiejar"
-	"net/url"
-	"sort"
-	"strings"
-	"sync"
-	"sync/atomic"
-
-	"github.com/cespare/xxhash"
-)
-
-// representationHeaderNames are folded into the cache key so two requests
-// that differ only in content-negotiation / UA identity cannot share a body.
-var representationHeaderNames = []string{
-	"accept",
-	"accept-encoding",
-	"accept-language",
-	"user-agent",
-}
-
-const (
-	// Entry cap sized above a typical per-host unique GET set; the byte cap below
-	// is the real memory bound, so a larger entry cap cannot grow memory.
-	defaultMaxEntries = 16384
-	defaultMaxBytes   = 64 << 20 // 64 MiB of retained responses (key, headers and body)
-	// defaultMaxSeen bounds the (owner, key) record: about 16 bytes per pair.
-	defaultMaxSeen = 1 << 20
-)
-
-// Cache stores response snapshots keyed by METHOD + URL + representation headers.
-type Cache struct {
-	mu         sync.RWMutex
-	entries    map[string]*Entry
-	disabled   atomic.Bool
-	hits       atomic.Uint64
-	misses     atomic.Uint64
-	stores     atomic.Uint64
-	skipped    atomic.Uint64
-	maxEntries int
-	maxBytes   int64
-	curBytes   int64
-
-	// seen records every (owner, key) pair that has been looked up or stored,
-	// as a hash. A pair already seen is never served from the cache: a template
-	// that repeats a request means it (state changes, race checks,
-	// request-and-compare), so the repeat must reach the server even when another
-	// template stored a response in between. When the record is full the cache
-	// stops serving hits instead of forgetting pairs.
-	seen    map[uint64]struct{}
-	maxSeen int
-
-	// flights holds the requests now on the wire, by key. Identical requests
-	// that arrive meanwhile wait for the leader instead of sending their own.
-	flights map[string]*flight
-}
-
-// flight is one request on the wire that other callers may wait for.
-type flight struct {
-	owner string
-	done  chan struct{}
-	// ent is the entry the leader stored, or nil when it stored nothing
-	// (error, uncacheable response, size limit). Set before done is closed.
-	ent *Entry
-}
-
-// Entry is a reusable response snapshot.
-type Entry struct {
-	// Owner identifies who stored the entry (a template ID). It is never served
-	// back to the same owner: a template that repeats its own request means it
-	// (state changes, race checks, request-and-compare), so that repeat must
-	// reach the server.
-	Owner         string
-	StatusCode    int
-	Status        string
-	Proto         string
-	ProtoMajor    int
-	ProtoMinor    int
-	Header        http.Header
-	Body          []byte
-	ContentLength int64
-	// size is what the entry is charged against maxBytes: key, headers and body.
-	size int64
-}
-
-// New returns an empty cache with default memory bounds.
-func New() *Cache {
-	return &Cache{
-		entries:    make(map[string]*Entry),
-		maxEntries: defaultMaxEntries,
-		maxBytes:   defaultMaxBytes,
-		seen:       make(map[uint64]struct{}),
-		maxSeen:    defaultMaxSeen,
-		flights:    make(map[string]*flight),
-	}
-}
-
-// Disable stops all Get/Set operations (used when -tf is set but the planner
-// skips tech filtering, so default scan semantics stay unchanged).
-func (c *Cache) Disable() {
-	if c != nil {
-		c.disabled.Store(true)
-	}
-}
-
-// Enabled reports whether the cache is accepting Get/Set.
-func (c *Cache) Enabled() bool {
-	return c != nil && !c.disabled.Load()
-}
-
-// Key builds a cache key for a method and absolute URL (no header identity).
-func Key(method, rawURL string) string {
-	method = strings.ToUpper(strings.TrimSpace(method))
-	rawURL = strings.TrimSpace(rawURL)
-	return method + " " + rawURL
-}
-
-// KeyFromRequest builds a key from an http.Request, including representation
-// headers (UA / Accept*) so distinct content-negotiation contexts do not collide.
-func KeyFromRequest(req *http.Request) string {
-	if req == nil || req.URL == nil {
-		return ""
-	}
-	base := Key(req.Method, req.URL.String())
-	parts := make([]string, 0, len(representationHeaderNames))
-	for _, name := range representationHeaderNames {
-		vals := req.Header.Values(name)
-		if len(vals) == 0 {
-			continue
-		}
-		parts = append(parts, name+":"+strings.Join(vals, ","))
-	}
-	if len(parts) == 0 {
-		return base
-	}
-	sort.Strings(parts)
-	return base + " |" + strings.Join(parts, "|")
-}
-
-// CacheableRequest reports whether the request is safe to cache/serve.
-// Requests with auth, cookies, Host override, Cache-Control/Pragma, or any
-// non-allowlisted header are excluded so they cannot reuse another context.
-func CacheableRequest(req *http.Request) bool {
-	if req == nil || req.URL == nil {
-		return false
-	}
-	method := strings.ToUpper(req.Method)
-	if method != http.MethodGet && method != http.MethodHead {
-		return false
-	}
-	// A GET/HEAD carrying a real payload is excluded: the body is part of the
-	// request identity but not of the cache key. An empty body is not.
-	// Every GET/HEAD built here goes through retryablehttp, which computes
-	// ContentLength from the buffered body, so ContentLength == 0 proves the
-	// body carries no bytes even when the reader itself is non-nil.
-	if req.Body != nil && req.Body != http.NoBody && req.ContentLength != 0 {
-		return false
-	}
-	if req.Host != "" && req.URL.Host != "" && !strings.EqualFold(req.Host, req.URL.Host) {
-		return false
-	}
-	for k := range req.Header {
-		switch strings.ToLower(k) {
-		case "user-agent", "accept", "accept-language", "accept-encoding",
-			"connection", "upgrade-insecure-requests":
-			continue
-		case "host":
-			// net/http never writes Header["Host"] on the wire; it sends
-			// req.Host or req.URL.Host. A Host header that repeats the target
-			// host therefore changes nothing. A different one is a deliberate
-			// Host override and must not share a cached response.
-			if !sameHost(req.Header.Get("Host"), req.URL) {
-				return false
-			}
-		case "cache-control", "pragma":
-			// Explicit cache directives: never serve from the scan cache.
-			return false
-		default:
-			return false
-		}
-	}
-	return true
-}
-
-// sameHost reports whether host names the same authority as u, ignoring the
-// default port for the URL scheme.
-func sameHost(host string, u *url.URL) bool {
-	if host == "" || u == nil {
-		return false
-	}
-	return strings.EqualFold(canonicalHost(host, u.Scheme), canonicalHost(u.Host, u.Scheme))
-}
-
-func canonicalHost(host, scheme string) string {
-	host = strings.TrimSpace(host)
-	defaultPort := ""
-	switch strings.ToLower(scheme) {
-	case "http":
-		defaultPort = ":80"
-	case "https":
-		defaultPort = ":443"
-	}
-	if defaultPort != "" {
-		host = strings.TrimSuffix(host, defaultPort)
-	}
-	return host
-}
-
-// JarAllows reports whether the cookie jar would attach no cookies to u.
-// A jar is created for every input, so its mere presence must not disable the
-// cache. What matters is whether cookies would change the request or response:
-// if the jar has nothing for this URL, a response captured without cookies is
-// equivalent to a fresh one.
-func JarAllows(jar *cookiejar.Jar, u *url.URL) bool {
-	if jar == nil {
-		return true
-	}
-	if u == nil {
-		return false
-	}
-	return len(jar.Cookies(u)) == 0
-}
-
-// ResponseSetsCookie reports whether resp carries Set-Cookie. Such responses are
-// never stored, so serving from cache can never hide a jar update.
-func ResponseSetsCookie(resp *http.Response) bool {
-	return resp != nil && len(resp.Header.Values("Set-Cookie")) > 0
-}
-
-// Get returns a fresh *http.Response (new Body reader) or nil on miss.
-// req is attached as Response.Request so downstream dump/curl code works.
-// An entry stored by owner is not served back to owner; see Entry.Owner.
-func (c *Cache) Get(key, owner string, req *http.Request) *http.Response {
-	if !c.Enabled() || key == "" {
-		return nil
-	}
-	c.mu.Lock()
-	firstLook := c.markSeen(owner, key)
-	ent, ok := c.entries[key]
-	c.mu.Unlock()
-	if !firstLook || !ok || ent == nil || (owner != "" && ent.Owner == owner) {
-		c.misses.Add(1)
-		return nil
-	}
-	c.hits.Add(1)
-	return ent.toResponse(req)
-}
-
-// GetOrLead is Get that also shares requests in flight. On a miss the caller
-// either leads a new flight or, when an identical request from another owner is
-// already on the wire, waits for it and gets its stored response.
-//
-// The returned done func must be called once the caller has stored its
-// response (or knows it will not). It is never nil and is safe to call more
-// than once. Waiters get their own response and body reader. When the leader
-// stores nothing (an error, an uncacheable response, the size limit), waiters
-// get nil and send their own request, so failures are never shared.
-// A canceled req context stops the wait.
-func (c *Cache) GetOrLead(key, owner string, req *http.Request) (*http.Response, func()) {
-	noop := func() {}
-	if !c.Enabled() || key == "" {
-		return nil, noop
-	}
-	c.mu.Lock()
-	firstLook := c.markSeen(owner, key)
-	ent, ok := c.entries[key]
-	if firstLook && ok && ent != nil && (owner == "" || ent.Owner != owner) {
-		c.mu.Unlock()
-		c.hits.Add(1)
-		return ent.toResponse(req), noop
-	}
-	if !firstLook {
-		// A repeat of the owner's own request must reach the server.
-		c.mu.Unlock()
-		c.misses.Add(1)
-		return nil, noop
-	}
-	if c.flights == nil {
-		c.flights = make(map[string]*flight)
-	}
-	f, inFlight := c.flights[key]
-	if !inFlight {
-		f = &flight{owner: owner, done: make(chan struct{})}
-		c.flights[key] = f
-		c.mu.Unlock()
-		c.misses.Add(1)
-		return nil, c.landFunc(key, f)
-	}
-	c.mu.Unlock()
-	if owner != "" && f.owner == owner {
-		// The owner's own response is never served back to it.
-		c.misses.Add(1)
-		return nil, noop
-	}
-	var ctxDone <-chan struct{}
-	if req != nil {
-		ctxDone = req.Context().Done()
-	}
-	select {
-	case <-f.done:
-	case <-ctxDone:
-		c.misses.Add(1)
-		return nil, noop
-	}
-	if f.ent == nil {
-		c.misses.Add(1)
-		return nil, noop
-	}
-	c.hits.Add(1)
-	return f.ent.toResponse(req), noop
-}
-
-// landFunc returns the func that ends flight f and wakes its waiters.
-func (c *Cache) landFunc(key string, f *flight) func() {
-	var once sync.Once
-	return func() {
-		once.Do(func() {
-			c.mu.Lock()
-			if c.flights[key] == f {
-				delete(c.flights, key)
-			}
-			c.mu.Unlock()
-			close(f.done)
-		})
-	}
-}
-
-// Set stores a snapshot for key. body should already be fully read.
-// Entries that would exceed the scan-wide budget are skipped.
-func (c *Cache) Set(key, owner string, resp *http.Response, body []byte) {
-	if !c.Enabled() || key == "" || resp == nil {
-		return
-	}
-	bodyLen := int64(len(body))
-	size := int64(len(key)) + headerSize(resp.Header) + bodyLen
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	c.markSeen(owner, key)
-
-	if old, ok := c.entries[key]; ok {
-		c.curBytes -= old.size
-		delete(c.entries, key)
-	} else if c.maxEntries > 0 && len(c.entries) >= c.maxEntries {
-		c.skipped.Add(1)
-		return
-	}
-	if c.maxBytes > 0 && c.curBytes+size > c.maxBytes {
-		c.skipped.Add(1)
-		return
-	}
-
-	ent := &Entry{
-		Owner:         owner,
-		StatusCode:    resp.StatusCode,
-		Status:        resp.Status,
-		Proto:         resp.Proto,
-		ProtoMajor:    resp.ProtoMajor,
-		ProtoMinor:    resp.ProtoMinor,
-		Header:        resp.Header.Clone(),
-		Body:          append([]byte(nil), body...),
-		ContentLength: bodyLen,
-		size:          size,
-	}
-	c.entries[key] = ent
-	c.curBytes += size
-	c.stores.Add(1)
-	if f, ok := c.flights[key]; ok && f.owner == owner {
-		// Written under c.mu before the leader closes f.done, so waiters
-		// that read it after <-f.done see it.
-		f.ent = ent
-	}
-}
-
-// SeedHTTP stores a fingerprint/probe response under the request's full key
-// (method + URL + representation headers). readLimit is the most the probe
-// read of the body; see CompleteBody.
-func (c *Cache) SeedHTTP(req *http.Request, resp *http.Response, body []byte, readLimit int64) {
-	if !CacheableRequest(req) || !CompleteBody(body, readLimit) {
-		return
-	}
-	c.Set(KeyFromRequest(req), "", resp, body)
-}
-
-// CompleteBody reports whether body was read to its end under readLimit. A body
-// that fills the limit may be cut short, and a reader with a larger limit
-// (another template's max-size) would then match against the cut copy, so such
-// a body must not be cached. A readLimit of zero or less means no limit.
-func CompleteBody(body []byte, readLimit int64) bool {
-	return readLimit <= 0 || int64(len(body)) < readLimit
-}
-
-// Reset drops every entry and the counters, keeping the cache usable. It lets a
-// long-lived engine reuse one cache instance while scoping its content to one
-// scan.
-func (c *Cache) Reset() {
-	if c == nil {
-		return
-	}
-	c.mu.Lock()
-	c.entries = make(map[string]*Entry)
-	c.curBytes = 0
-	c.seen = make(map[uint64]struct{})
-	c.mu.Unlock()
-	c.hits.Store(0)
-	c.misses.Store(0)
-	c.stores.Store(0)
-	c.skipped.Store(0)
-}
-
-// Stats returns hit/miss/store counters.
-func (c *Cache) Stats() (hits, misses, stores uint64) {
-	if c == nil {
-		return 0, 0, 0
-	}
-	return c.hits.Load(), c.misses.Load(), c.stores.Load()
-}
-
-// Len returns the number of cached entries (tests).
-func (c *Cache) Len() int {
-	if c == nil {
-		return 0
-	}
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-	return len(c.entries)
-}
-
-func (e *Entry) toResponse(req *http.Request) *http.Response {
-	return &http.Response{
-		Status:        e.Status,
-		StatusCode:    e.StatusCode,
-		Proto:         e.Proto,
-		ProtoMajor:    e.ProtoMajor,
-		ProtoMinor:    e.ProtoMinor,
-		Header:        e.Header.Clone(),
-		Body:          io.NopCloser(bytes.NewReader(e.Body)),
-		ContentLength: e.ContentLength,
-		Request:       req,
-	}
-}
-
-// markSeen records the (owner, key) pair and reports whether this is its first
-// sighting. It reports false when the record is full, so nothing is served
-// that could not be remembered. An empty owner is not tracked. Must hold c.mu.
-func (c *Cache) markSeen(owner, key string) bool {
-	if owner == "" {
-		return true
-	}
-	if c.seen == nil {
-		c.seen = make(map[uint64]struct{})
-	}
-	h := xxhash.Sum64String(owner + "\x00" + key)
-	if _, ok := c.seen[h]; ok {
-		return false
-	}
-	if c.maxSeen > 0 && len(c.seen) >= c.maxSeen {
-		return false
-	}
-	c.seen[h] = struct{}{}
-	return true
-}
-
-func headerSize(h http.Header) int64 {
-	var n int64
-	for k, vs := range h {
-		for _, v := range vs {
-			n += int64(len(k) + len(v) + 4) // "k: v\r\n"
-		}
-	}
-	return n
-}
-
-// ReplayableEncoding reports whether a response can be replayed from its
-// decoded body with identical results. The response chain decodes the body by
-// its Content-Encoding header, and a cache hit keeps that header (templates may
-// match on it), so a hit is decoded again. That is harmless only when the
-// second decode fails at once and falls back to the same bytes: no encoding, or
-// gzip whose decoded body does not itself start with the gzip magic bytes.
-// Other encodings (deflate, br, zstd) can fail mid-read or decode twice, so
-// they are not cached.
-func ReplayableEncoding(h http.Header, decodedBody []byte) bool {
-	switch strings.ToLower(strings.TrimSpace(h.Get("Content-Encoding"))) {
-	case "", "identity":
-		return true
-	case "gzip":
-		return len(decodedBody) < 2 || decodedBody[0] != 0x1f || decodedBody[1] != 0x8b
-	default:
-		return false
-	}
-}
-
-// StoredStatus reports whether a response status may be cached. Transient
-// failures (408, 429, 5xx) are not: a later request may find the server
-// recovered, and serving the failure would hide that.
-func StoredStatus(code int) bool {
-	return code != http.StatusRequestTimeout && code != http.StatusTooManyRequests && code < http.StatusInternalServerError
-}

pkg/protocols/http/httprespcache/cache_test.go

@@ -1,363 +0,0 @@
-package httprespcache
-
-import (
-	"io"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"testing"
-)
-
-func TestCacheRoundTrip(t *testing.T) {
-	c := New()
-	req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
-	resp := &http.Response{
-		Status:     "200 OK",
-		StatusCode: 200,
-		Proto:      "HTTP/1.1",
-		ProtoMajor: 1,
-		ProtoMinor: 1,
-		Header:     http.Header{"Server": []string{"nginx"}},
-		Body:       io.NopCloser(http.NoBody),
-	}
-	c.Set(KeyFromRequest(req), "", resp, []byte("ok"))
-
-	got := c.Get(KeyFromRequest(req), "", req)
-	if got == nil {
-		t.Fatal("expected cache hit")
-	}
-	body, _ := io.ReadAll(got.Body)
-	if string(body) != "ok" {
-		t.Fatalf("body=%q", body)
-	}
-	if got.Header.Get("Server") != "nginx" {
-		t.Fatalf("header=%v", got.Header)
-	}
-	hits, misses, stores := c.Stats()
-	if hits != 1 || misses != 0 || stores != 1 {
-		t.Fatalf("stats hits=%d misses=%d stores=%d", hits, misses, stores)
-	}
-	if c.Get(Key(http.MethodGet, "http://example.com/other"), "", req) != nil {
-		t.Fatal("expected miss")
-	}
-}
-
-func TestCacheableRequest(t *testing.T) {
-	get := httptest.NewRequest(http.MethodGet, "http://x/", nil)
-	if !CacheableRequest(get) {
-		t.Fatal("GET should be cacheable")
-	}
-	post := httptest.NewRequest(http.MethodPost, "http://x/", nil)
-	if CacheableRequest(post) {
-		t.Fatal("POST should not be cacheable")
-	}
-	auth := httptest.NewRequest(http.MethodGet, "http://x/", nil)
-	auth.Header.Set("Authorization", "Bearer t")
-	if CacheableRequest(auth) {
-		t.Fatal("Authorization must not be cacheable")
-	}
-	cookie := httptest.NewRequest(http.MethodGet, "http://x/", nil)
-	cookie.Header.Set("Cookie", "a=b")
-	if CacheableRequest(cookie) {
-		t.Fatal("Cookie must not be cacheable")
-	}
-	hostOverride := httptest.NewRequest(http.MethodGet, "http://x/", nil)
-	hostOverride.Host = "other.example"
-	if CacheableRequest(hostOverride) {
-		t.Fatal("Host override must not be cacheable")
-	}
-	cc := httptest.NewRequest(http.MethodGet, "http://x/", nil)
-	cc.Header.Set("Cache-Control", "no-cache")
-	if CacheableRequest(cc) {
-		t.Fatal("Cache-Control must not be cacheable")
-	}
-	pragma := httptest.NewRequest(http.MethodGet, "http://x/", nil)
-	pragma.Header.Set("Pragma", "no-cache")
-	if CacheableRequest(pragma) {
-		t.Fatal("Pragma must not be cacheable")
-	}
-}
-
-func TestKeyFromRequestIncludesRepresentationHeaders(t *testing.T) {
-	a := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
-	a.Header.Set("User-Agent", "ua-a")
-	a.Header.Set("Accept-Language", "en")
-
-	b := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
-	b.Header.Set("User-Agent", "ua-a")
-	b.Header.Set("Accept-Language", "fr")
-
-	if KeyFromRequest(a) == KeyFromRequest(b) {
-		t.Fatal("distinct Accept-Language must produce distinct keys")
-	}
-
-	c := New()
-	c.Set(KeyFromRequest(a), "", &http.Response{StatusCode: 200, Header: http.Header{}}, []byte("en-body"))
-	if c.Get(KeyFromRequest(b), "", b) != nil {
-		t.Fatal("must not reuse response across Accept-Language")
-	}
-	if got := c.Get(KeyFromRequest(a), "", a); got == nil {
-		t.Fatal("expected hit for matching headers")
-	}
-}
-
-func TestCacheDisabled(t *testing.T) {
-	c := New()
-	req, err := http.NewRequest(http.MethodGet, "http://example.com/", nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	c.Set(KeyFromRequest(req), "", &http.Response{StatusCode: 200, Header: http.Header{}}, []byte("x"))
-	c.Disable()
-	if c.Get(KeyFromRequest(req), "", req) != nil {
-		t.Fatal("disabled cache must miss")
-	}
-	c.Set(KeyFromRequest(req), "", &http.Response{StatusCode: 200, Header: http.Header{}}, []byte("y"))
-	if c.Enabled() {
-		t.Fatal("expected disabled")
-	}
-}
-
-func TestCacheEntryBudget(t *testing.T) {
-	c := New()
-	c.maxEntries = 2
-	c.maxBytes = 100
-	mk := func(path, body string) {
-		req := httptest.NewRequest(http.MethodGet, "http://example.com"+path, nil)
-		c.Set(KeyFromRequest(req), "", &http.Response{StatusCode: 200, Header: http.Header{}}, []byte(body))
-	}
-	mk("/a", "aa")
-	mk("/b", "bb")
-	mk("/c", "cc") // over entry budget
-	if c.Len() != 2 {
-		t.Fatalf("len=%d want 2", c.Len())
-	}
-	if c.skipped.Load() == 0 {
-		t.Fatal("expected skipped store")
-	}
-
-	c2 := New()
-	c2.maxEntries = 10
-	// The budget covers key, headers and body: room for "/a" with a 5-byte body.
-	c2.maxBytes = int64(len(KeyFromRequest(httptest.NewRequest(http.MethodGet, "http://example.com/a", nil)))) + 5
-	mk2 := func(path, body string) {
-		req := httptest.NewRequest(http.MethodGet, "http://example.com"+path, nil)
-		c2.Set(KeyFromRequest(req), "", &http.Response{StatusCode: 200, Header: http.Header{}}, []byte(body))
-	}
-	mk2("/a", "12345")
-	mk2("/b", "x") // over byte budget
-	if c2.Len() != 1 {
-		t.Fatalf("byte budget len=%d want 1", c2.Len())
-	}
-}
-
-func TestCacheableRequestEmptyBody(t *testing.T) {
-	// Raw templates hand over a non-nil but empty reader with ContentLength 0.
-	// That request is bodyless on the wire, so it stays cacheable.
-	empty := httptest.NewRequest(http.MethodGet, "http://x/", strings.NewReader(""))
-	empty.ContentLength = 0
-	if empty.Body == nil {
-		t.Fatal("test needs a non-nil body")
-	}
-	if !CacheableRequest(empty) {
-		t.Fatal("GET with an empty body should be cacheable")
-	}
-
-	// A GET that really carries bytes is excluded: the body is part of the
-	// request identity but is not part of the cache key.
-	withBody := httptest.NewRequest(http.MethodGet, "http://x/", strings.NewReader("payload"))
-	withBody.ContentLength = int64(len("payload"))
-	if CacheableRequest(withBody) {
-		t.Fatal("GET with a payload must not be cacheable")
-	}
-
-	// Unknown length is treated as a payload.
-	unknown := httptest.NewRequest(http.MethodGet, "http://x/", strings.NewReader("payload"))
-	unknown.ContentLength = -1
-	if CacheableRequest(unknown) {
-		t.Fatal("GET with an unknown body length must not be cacheable")
-	}
-}
-
-func TestCacheableRequestHostHeader(t *testing.T) {
-	// Raw templates repeat the target host in a Host header. net/http drops it
-	// and sends the URL host, so the request is unchanged and stays cacheable.
-	same := httptest.NewRequest(http.MethodGet, "http://x.example/", nil)
-	same.Header.Set("Host", "x.example")
-	if !CacheableRequest(same) {
-		t.Fatal("redundant Host header should stay cacheable")
-	}
-
-	withDefaultPort := httptest.NewRequest(http.MethodGet, "http://x.example/", nil)
-	withDefaultPort.Header.Set("Host", "x.example:80")
-	if !CacheableRequest(withDefaultPort) {
-		t.Fatal("Host header with the default port should stay cacheable")
-	}
-
-	// A different Host is a deliberate override and must not share a response.
-	other := httptest.NewRequest(http.MethodGet, "http://x.example/", nil)
-	other.Header.Set("Host", "evil.example")
-	if CacheableRequest(other) {
-		t.Fatal("Host override must not be cacheable")
-	}
-
-	otherPort := httptest.NewRequest(http.MethodGet, "http://x.example/", nil)
-	otherPort.Header.Set("Host", "x.example:8080")
-	if CacheableRequest(otherPort) {
-		t.Fatal("Host header with a different port must not be cacheable")
-	}
-}
-
-func TestCacheReset(t *testing.T) {
-	c := New()
-	req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
-	c.Set(KeyFromRequest(req), "", &http.Response{StatusCode: 200, Header: http.Header{}}, []byte("body"))
-	c.Get(KeyFromRequest(req), "", req)
-	c.Reset()
-	if c.Len() != 0 {
-		t.Fatal("Reset must drop entries")
-	}
-	if h, m, s := c.Stats(); h != 0 || m != 0 || s != 0 {
-		t.Fatalf("Reset must clear counters: %d %d %d", h, m, s)
-	}
-	c.Set(KeyFromRequest(req), "", &http.Response{StatusCode: 200, Header: http.Header{}}, []byte("body"))
-	if c.Len() != 1 {
-		t.Fatal("cache must stay usable after Reset")
-	}
-}
-
-// A template that repeats its own request means it, so the repeat must reach
-// the server; other templates still share the response.
-func TestCacheNotServedBackToOwner(t *testing.T) {
-	c := New()
-	req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
-	key := KeyFromRequest(req)
-	c.Set(key, "template-a", &http.Response{StatusCode: 200, Header: http.Header{}}, []byte("1"))
-	if c.Get(key, "template-a", req) != nil {
-		t.Fatal("an entry must not be served back to the template that stored it")
-	}
-	if c.Get(key, "template-b", req) == nil {
-		t.Fatal("another template must reuse the entry")
-	}
-}
-
-func ok200() *http.Response { return &http.Response{StatusCode: 200, Header: http.Header{}} }
-
-// Each template gets at most one cached answer per request: its repeat reaches
-// the server even when the entry was stored by someone else.
-func TestAtMostOneCachedAnswerPerTemplate(t *testing.T) {
-	c := New()
-	req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
-	key := KeyFromRequest(req)
-	c.Set(key, "a", ok200(), []byte("1"))
-	if c.Get(key, "b", req) == nil {
-		t.Fatal("first lookup by another template should hit")
-	}
-	if c.Get(key, "b", req) != nil {
-		t.Fatal("the same template's repeat must reach the server")
-	}
-}
-
-// A template that already sent a request (miss) must not later be answered from
-// an entry another template stored in between.
-func TestRepeatAfterOthersStoreStillMisses(t *testing.T) {
-	c := New()
-	req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
-	key := KeyFromRequest(req)
-	if c.Get(key, "a", req) != nil {
-		t.Fatal("empty cache must miss")
-	}
-	c.Set(key, "b", ok200(), []byte("1"))
-	if c.Get(key, "a", req) != nil {
-		t.Fatal("a template that already sent this request must not be served")
-	}
-}
-
-// When the record of seen pairs is full, nothing is served.
-func TestFullSeenRecordStopsHits(t *testing.T) {
-	c := New()
-	c.maxSeen = 1
-	req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
-	key := KeyFromRequest(req)
-	c.Set(key, "a", ok200(), []byte("1")) // fills the record
-	if c.Get(key, "b", req) != nil {
-		t.Fatal("a full record must stop hits")
-	}
-}
-
-// Headers and key count against the memory budget, not only the body.
-func TestBudgetCountsHeaders(t *testing.T) {
-	c := New()
-	c.maxBytes = 512
-	req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
-	resp := ok200()
-	resp.Header.Set("X-Big", strings.Repeat("h", 1024))
-	c.Set(KeyFromRequest(req), "a", resp, []byte("tiny"))
-	if c.Len() != 0 {
-		t.Fatal("an entry whose headers exceed the budget must not be stored")
-	}
-}
-
-func TestReplayableEncoding(t *testing.T) {
-	h := func(enc string) http.Header { x := http.Header{}; x.Set("Content-Encoding", enc); return x }
-	cases := []struct {
-		name string
-		h    http.Header
-		body []byte
-		want bool
-	}{
-		{"no encoding", http.Header{}, []byte("x"), true},
-		{"identity", h("identity"), []byte("x"), true},
-		{"gzip, plain decoded body", h("gzip"), []byte("<html>"), true},
-		{"gzip, decoded body is gzip again", h("gzip"), []byte{0x1f, 0x8b, 0x08}, false},
-		{"deflate", h("deflate"), []byte("x"), false},
-		{"brotli", h("br"), []byte("x"), false},
-		{"zstd", h("zstd"), []byte("x"), false},
-	}
-	for _, tc := range cases {
-		if got := ReplayableEncoding(tc.h, tc.body); got != tc.want {
-			t.Errorf("%s: got %v want %v", tc.name, got, tc.want)
-		}
-	}
-}
-
-func TestStoredStatus(t *testing.T) {
-	for code, want := range map[int]bool{200: true, 301: true, 404: true, 400: true, 408: false, 429: false, 500: false, 503: false} {
-		if got := StoredStatus(code); got != want {
-			t.Errorf("StoredStatus(%d)=%v want %v", code, got, want)
-		}
-	}
-}
-
-func TestCompleteBody(t *testing.T) {
-	cases := []struct {
-		name  string
-		body  int
-		limit int64
-		want  bool
-	}{
-		{"shorter than limit", 10, 50, true},
-		{"fills limit, may be cut", 50, 50, false},
-		{"no limit", 50, 0, true},
-	}
-	for _, tc := range cases {
-		if got := CompleteBody(make([]byte, tc.body), tc.limit); got != tc.want {
-			t.Errorf("%s: got %v want %v", tc.name, got, tc.want)
-		}
-	}
-}
-
-func TestSeedHTTPSkipsCutBody(t *testing.T) {
-	c := New()
-	req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
-	resp := &http.Response{StatusCode: 200, Header: http.Header{}}
-
-	c.SeedHTTP(req, resp, []byte("12345"), 5)
-	if c.Len() != 0 {
-		t.Fatal("a body that fills the probe limit must not be seeded")
-	}
-	c.SeedHTTP(req, resp, []byte("1234"), 5)
-	if c.Len() != 1 {
-		t.Fatal("a complete body must be seeded")
-	}
-}

pkg/protocols/http/httprespcache/inflight_test.go

@@ -1,188 +0,0 @@
-package httprespcache
-
-import (
-	"context"
-	"io"
-	"net/http"
-	"net/http/httptest"
-	"strconv"
-	"sync"
-	"testing"
-	"time"
-)
-
-func okResponse() *http.Response {
-	return &http.Response{
-		Status:     "200 OK",
-		StatusCode: 200,
-		Proto:      "HTTP/1.1",
-		ProtoMajor: 1,
-		ProtoMinor: 1,
-		Header:     http.Header{"Server": []string{"nginx"}},
-		Body:       io.NopCloser(http.NoBody),
-	}
-}
-
-// startWaiters runs n GetOrLead calls from distinct owners and returns their
-// results once all of them return.
-func startWaiters(c *Cache, key string, n int, req *http.Request) (results chan *http.Response, returned chan struct{}) {
-	results = make(chan *http.Response, n)
-	returned = make(chan struct{}, n)
-	for i := 0; i < n; i++ {
-		go func(i int) {
-			resp, done := c.GetOrLead(key, "waiter-"+strconv.Itoa(i), req)
-			done()
-			returned <- struct{}{}
-			results <- resp
-		}(i)
-	}
-	return results, returned
-}
-
-// Concurrent identical requests send one request: the others wait for the
-// leader and each gets its own response and body reader.
-func TestInFlightShared(t *testing.T) {
-	c := New()
-	req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
-	key := KeyFromRequest(req)
-
-	resp, done := c.GetOrLead(key, "leader", req)
-	if resp != nil {
-		t.Fatal("first request must lead, not hit")
-	}
-	const n = 16
-	results, returned := startWaiters(c, key, n, req)
-
-	select {
-	case <-returned:
-		t.Fatal("a waiter returned before the leader finished")
-	case <-time.After(50 * time.Millisecond):
-	}
-
-	c.Set(key, "leader", okResponse(), []byte("shared body"))
-	done()
-	done() // safe to call twice
-
-	var wg sync.WaitGroup
-	for i := 0; i < n; i++ {
-		got := <-results
-		if got == nil {
-			t.Fatal("waiter got no response")
-		}
-		wg.Add(1)
-		go func(r *http.Response) {
-			defer wg.Done()
-			body, _ := io.ReadAll(r.Body)
-			_ = r.Body.Close()
-			if string(body) != "shared body" {
-				t.Errorf("body=%q", body)
-			}
-			r.Header.Set("Server", "changed")
-		}(got)
-	}
-	wg.Wait()
-	hits, misses, stores := c.Stats()
-	if hits != n || misses != 1 || stores != 1 {
-		t.Fatalf("stats hits=%d misses=%d stores=%d", hits, misses, stores)
-	}
-	again, _ := c.GetOrLead(key, "late", req)
-	if again == nil || again.Header.Get("Server") != "nginx" {
-		t.Fatal("a waiter must not change the stored entry")
-	}
-}
-
-// A leader that stores nothing (error, uncacheable response) shares nothing:
-// waiters get nil and send their own request, and nothing is cached.
-func TestInFlightErrorNotShared(t *testing.T) {
-	c := New()
-	req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
-	key := KeyFromRequest(req)
-
-	_, done := c.GetOrLead(key, "leader", req)
-	results, _ := startWaiters(c, key, 4, req)
-	time.Sleep(20 * time.Millisecond)
-	done()
-	for i := 0; i < 4; i++ {
-		if <-results != nil {
-			t.Fatal("a failed leader must not share a response")
-		}
-	}
-	if c.Len() != 0 {
-		t.Fatal("nothing must be cached")
-	}
-	if resp, done := c.GetOrLead(key, "later", req); resp != nil {
-		t.Fatal("a later caller must send its own request")
-	} else {
-		done()
-	}
-}
-
-// A response over the size limit is not stored, so it is not shared either.
-func TestInFlightRespectsSizeLimit(t *testing.T) {
-	c := New()
-	c.maxBytes = 8
-	req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
-	key := KeyFromRequest(req)
-
-	_, done := c.GetOrLead(key, "leader", req)
-	results, _ := startWaiters(c, key, 2, req)
-	time.Sleep(20 * time.Millisecond)
-	c.Set(key, "leader", okResponse(), []byte("a body larger than the limit"))
-	done()
-	for i := 0; i < 2; i++ {
-		if <-results != nil {
-			t.Fatal("an entry over the limit must not be shared")
-		}
-	}
-}
-
-// The owner rule holds in flight: a template never waits for its own request.
-func TestInFlightNotSharedWithOwner(t *testing.T) {
-	c := New()
-	req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
-	key := KeyFromRequest(req)
-
-	_, done := c.GetOrLead(key, "same", req)
-	defer done()
-	returned := make(chan *http.Response, 1)
-	go func() {
-		resp, d := c.GetOrLead(key, "same", req)
-		d()
-		returned <- resp
-	}()
-	select {
-	case resp := <-returned:
-		if resp != nil {
-			t.Fatal("a template must not get its own response")
-		}
-	case <-time.After(time.Second):
-		t.Fatal("a template must not wait for its own request")
-	}
-}
-
-// A canceled request stops waiting.
-func TestInFlightWaitHonorsContext(t *testing.T) {
-	c := New()
-	req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
-	key := KeyFromRequest(req)
-
-	_, done := c.GetOrLead(key, "leader", req)
-	defer done()
-	ctx, cancel := context.WithCancel(context.Background())
-	returned := make(chan *http.Response, 1)
-	go func() {
-		resp, d := c.GetOrLead(key, "waiter", req.WithContext(ctx))
-		d()
-		returned <- resp
-	}()
-	time.Sleep(20 * time.Millisecond)
-	cancel()
-	select {
-	case resp := <-returned:
-		if resp != nil {
-			t.Fatal("a canceled waiter must get nil")
-		}
-	case <-time.After(time.Second):
-		t.Fatal("a canceled waiter must stop waiting")
-	}
-}

pkg/protocols/http/request.go

@@ -41,7 +41,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httputils"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signer"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signerpool"
@@ -1087,8 +1086,6 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 		// including follow-ups when the initial response came from the cache.
 		executingClient *retryablehttp.Client
 	)
-	// respCacheDone ends this request's response cache flight, if it leads one.
-	respCacheDone := func() {}
 
 	// Dump request for variables checks
 	// For race conditions we can't dump the request body at this point as it's already waiting the open-gate event, already handled with a similar code within the race function
@@ -1244,25 +1241,6 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 				fromCache = false
 			}
 		}
-		// Scan-scoped in-memory cache (tech-filter / response reuse). Prefer this
-		// over a network round-trip for identical safe GET/HEAD requests. An
-		// identical request already on the wire is waited for, not sent again.
-		if resp == nil && request.options.HTTPResponseCache != nil && request.usesResponseCache(generatedRequest) &&
-			generatedRequest.request != nil && generatedRequest.request.Request != nil &&
-			httprespcache.CacheableRequest(generatedRequest.request.Request) &&
-			httprespcache.JarAllows(input.CookieJar, generatedRequest.request.Request.URL) {
-			if key := respCacheKey(input, generatedRequest.request.Request); key != "" {
-				cached, done := request.options.HTTPResponseCache.GetOrLead(key, request.options.TemplateID, generatedRequest.request.Request)
-				// Every return path ends the flight, so waiters never hang.
-				defer done()
-				respCacheDone = done
-				if cached != nil {
-					resp = cached
-					fromCache = true
-					err = nil
-				}
-			}
-		}
 		if resp == nil || (request.Analyzer != nil && cookieFuzzing) {
 			connConfig := request.connConfiguration
 			if input.CookieJar != nil && !request.DisableCookie {
@@ -1512,21 +1490,6 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 				errx = errors.Wrap(err, "could not store in project file")
 			}
 		}
-		if request.options.HTTPResponseCache != nil && !fromCache && request.usesResponseCache(generatedRequest) &&
-			generatedRequest.request != nil && generatedRequest.request.Request != nil &&
-			httprespcache.CacheableRequest(generatedRequest.request.Request) &&
-			httprespcache.JarAllows(input.CookieJar, generatedRequest.request.Request.URL) &&
-			!httprespcache.ResponseSetsCookie(resp) &&
-			httprespcache.StoredStatus(resp.StatusCode) &&
-			httprespcache.CompleteBody(respChain.BodyBytes(), int64(maxBodylimit)) &&
-			httprespcache.ReplayableEncoding(resp.Header, respChain.BodyBytes()) &&
-			!isHTTPToHTTPSMismatch(resp.StatusCode, respChain.BodyString()) {
-			if key := respCacheKey(input, generatedRequest.request.Request); key != "" {
-				request.options.HTTPResponseCache.Set(key, request.options.TemplateID, resp, respChain.BodyBytes())
-			}
-		}
-		// Wake requests waiting on this one now, not after matching.
-		respCacheDone()
 	})
 
 	// evaluate responses continuously until first redirect request in reverse order

pkg/protocols/http/respcache.go

@@ -1,63 +0,0 @@
-package http
-
-import (
-	"net/http"
-	"strings"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache"
-)
-
-// followsRedirects reports whether this request's client follows redirects,
-// resolving the policy the same way httpclientpool does: the template setting,
-// overridden by the global follow flags, cancelled by -disable-redirects.
-//
-// The scan response cache must stay out of such requests. A redirect chain is
-// several responses, and a cache entry is one: serving it would hide the chain
-// from matchers that walk it. The cache key also does not carry the redirect
-// policy, so a followed and an unfollowed request for the same URL would
-// otherwise share one entry and each see the other's response.
-func (request *Request) followsRedirects() bool {
-	if request.options == nil || request.options.Options == nil {
-		return true // unknown policy: stay out of the cache
-	}
-	opts := request.options.Options
-	if opts.DisableRedirects {
-		return false
-	}
-	if opts.ShouldFollowHTTPRedirects() {
-		return true
-	}
-	return request.connConfiguration == nil ||
-		request.connConfiguration.RedirectFlow != httpclientpool.DontFollowRedirect
-}
-
-// usesResponseCache reports whether this request may be served from, or stored
-// in, the scan response cache. A template's own disable-http-cache opts out,
-// race requests must all reach the server, and redirect-following requests
-// carry a chain the cache cannot represent.
-func (request *Request) usesResponseCache(req *generatedRequest) bool {
-	if request.DisableHTTPCache || request.followsRedirects() {
-		return false
-	}
-	return req == nil || req.original == nil || !req.original.Race
-}
-
-// respCacheKey is the response cache key for a request. Inputs that share a
-// URL but dial different addresses (-scan-all-ips, -iv) must not share
-// responses, so the selected IP is part of the key.
-func respCacheKey(input *contextargs.Context, req *http.Request) string {
-	key := httprespcache.KeyFromRequest(req)
-	if key == "" || input == nil || input.MetaInput == nil || input.MetaInput.CustomIP == "" {
-		return key
-	}
-	return key + " |ip:" + input.MetaInput.CustomIP
-}
-
-// isHTTPToHTTPSMismatch reports the 400 a server returns for plain HTTP sent to
-// an HTTPS port. The scan records a correction for that host:port, so this
-// response must not be cached: a later hit would skip the correction.
-func isHTTPToHTTPSMismatch(statusCode int, body string) bool {
-	return statusCode == http.StatusBadRequest && strings.Contains(body, httpToHTTPSMismatchSignature)
-}

pkg/protocols/http/respcache_test.go

@@ -1,208 +0,0 @@
-package http
-
-import (
-	"context"
-	"net/http"
-	"net/http/httptest"
-	"strconv"
-	"strings"
-	"sync"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/require"
-
-	"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
-	"github.com/projectdiscovery/nuclei/v3/pkg/model"
-	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
-	"github.com/projectdiscovery/nuclei/v3/pkg/output"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache"
-	"github.com/projectdiscovery/nuclei/v3/pkg/types"
-)
-
-// The scan response cache must never serve or store a request that follows
-// redirects: a chain is several responses, a cache entry is one, and the key
-// does not carry the redirect policy.
-func TestFollowsRedirects(t *testing.T) {
-	mk := func(flow httpclientpool.RedirectFlow, opts *types.Options) *Request {
-		return &Request{
-			options:           &protocols.ExecutorOptions{Options: opts},
-			connConfiguration: &httpclientpool.Configuration{RedirectFlow: flow},
-		}
-	}
-	cases := []struct {
-		name string
-		req  *Request
-		want bool
-	}{
-		{"template does not follow", mk(httpclientpool.DontFollowRedirect, &types.Options{}), false},
-		{"template follows all", mk(httpclientpool.FollowAllRedirect, &types.Options{}), true},
-		{"template follows same host", mk(httpclientpool.FollowSameHostRedirect, &types.Options{}), true},
-		{"template follows same scheme", mk(httpclientpool.FollowSameSchemeRedirect, &types.Options{}), true},
-		{"global -fr overrides template", mk(httpclientpool.DontFollowRedirect, &types.Options{FollowRedirects: true}), true},
-		{"global -fhr overrides template", mk(httpclientpool.DontFollowRedirect, &types.Options{FollowHostRedirects: true}), true},
-		{"-disable-redirects wins", mk(httpclientpool.FollowAllRedirect, &types.Options{FollowRedirects: true, DisableRedirects: true}), false},
-		{"unknown options stay out of cache", &Request{}, true},
-	}
-	for _, c := range cases {
-		if got := c.req.followsRedirects(); got != c.want {
-			t.Errorf("%s: followsRedirects()=%v want %v", c.name, got, c.want)
-		}
-	}
-}
-
-func TestUsesResponseCache(t *testing.T) {
-	plain := &Request{options: &protocols.ExecutorOptions{Options: &types.Options{}}, connConfiguration: &httpclientpool.Configuration{}}
-	if !plain.usesResponseCache(&generatedRequest{original: &Request{}}) {
-		t.Fatal("an ordinary request may use the cache")
-	}
-	optOut := &Request{DisableHTTPCache: true, options: plain.options, connConfiguration: plain.connConfiguration}
-	if optOut.usesResponseCache(&generatedRequest{original: optOut}) {
-		t.Fatal("disable-http-cache must keep the request out of the cache")
-	}
-	if plain.usesResponseCache(&generatedRequest{original: &Request{Race: true}}) {
-		t.Fatal("race requests must all reach the server")
-	}
-	redirect := &Request{options: plain.options, connConfiguration: &httpclientpool.Configuration{RedirectFlow: httpclientpool.FollowAllRedirect}}
-	if redirect.usesResponseCache(&generatedRequest{original: redirect}) {
-		t.Fatal("redirect-following requests must stay out of the cache")
-	}
-}
-
-func TestRespCacheKeyIncludesCustomIP(t *testing.T) {
-	req, _ := http.NewRequest(http.MethodGet, "http://example.com/", nil)
-	plain := contextargs.NewWithInput(context.Background(), "http://example.com/")
-	a := contextargs.NewWithInput(context.Background(), "http://example.com/")
-	a.MetaInput.CustomIP = "10.0.0.1"
-	b := contextargs.NewWithInput(context.Background(), "http://example.com/")
-	b.MetaInput.CustomIP = "10.0.0.2"
-	if respCacheKey(a, req) == respCacheKey(b, req) {
-		t.Fatal("inputs that dial different IPs must not share a cache key")
-	}
-	if respCacheKey(plain, req) == respCacheKey(a, req) {
-		t.Fatal("an IP-pinned input must not share the plain input's key")
-	}
-}
-
-func TestIsHTTPToHTTPSMismatch(t *testing.T) {
-	if !isHTTPToHTTPSMismatch(400, "<h1>The plain HTTP request was sent to HTTPS port</h1>") {
-		t.Fatal("expected the mismatch to be recognised")
-	}
-	if isHTTPToHTTPSMismatch(400, "bad request") || isHTTPToHTTPSMismatch(200, "The plain HTTP request was sent to HTTPS port") {
-		t.Fatal("only the 400 with the mismatch message counts")
-	}
-}
-
-// A body read up to a template's max-size may be cut short. The cache must not
-// keep it, or a later template with a larger limit matches against a cut body.
-func TestResponseCacheSkipsTruncatedBody(t *testing.T) {
-	var hits atomic.Int32
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		hits.Add(1)
-		_, _ = w.Write([]byte(strings.Repeat("A", 150) + "TAIL"))
-	}))
-	defer ts.Close()
-
-	// Other tests change the shared default options, so work on a copy.
-	opts := testutils.DefaultOptions.Copy()
-	// -response-cache pins one user agent, so identical requests share a key.
-	opts.ResponseCache = true
-	opts.FollowRedirects = false
-	opts.FollowHostRedirects = false
-	opts.ResponseReadSize = 0
-	// A custom header makes a request uncacheable.
-	opts.CustomHeaders = nil
-	testutils.Init(opts)
-	cache := httprespcache.New()
-
-	execute := func(id string, maxSize int) string {
-		request := &Request{ID: id, MaxSize: maxSize, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, Path: []string{"{{BaseURL}}/big"}}
-		executerOptions := testutils.NewMockExecuterOptions(opts, &testutils.TemplateInfo{
-			ID:   id,
-			Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
-		})
-		executerOptions.HTTPResponseCache = cache
-		require.NoError(t, request.Compile(executerOptions))
-
-		var body string
-		err := request.ExecuteWithResults(contextargs.NewWithInput(context.Background(), ts.URL), make(output.InternalEvent), make(output.InternalEvent), func(event *output.InternalWrappedEvent) {
-			if event != nil && event.InternalEvent != nil {
-				body, _ = event.InternalEvent["body"].(string)
-			}
-		})
-		require.NoError(t, err)
-		return body
-	}
-
-	require.Len(t, execute("small", 50), 50)
-	require.Equal(t, 0, cache.Len(), "truncated body must not be cached")
-
-	require.True(t, strings.HasSuffix(execute("full", 0), "TAIL"))
-	require.Equal(t, int32(2), hits.Load())
-	require.Equal(t, 1, cache.Len())
-
-	require.Len(t, execute("small-again", 50), 50, "a cached full body is still cut to the reader's limit")
-	require.Equal(t, int32(2), hits.Load())
-}
-
-// Identical cacheable requests from different templates that run at the same
-// time send one request on the wire. Without the cache each one is sent.
-func TestResponseCacheSharesInFlight(t *testing.T) {
-	var hits atomic.Int32
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		hits.Add(1)
-		time.Sleep(300 * time.Millisecond)
-		_, _ = w.Write([]byte("shared"))
-	}))
-	defer ts.Close()
-
-	opts := testutils.DefaultOptions.Copy()
-	opts.ResponseCache = true
-	opts.FollowRedirects = false
-	opts.FollowHostRedirects = false
-	opts.CustomHeaders = nil
-	testutils.Init(opts)
-
-	run := func(cache *httprespcache.Cache) []string {
-		hits.Store(0)
-		const n = 4
-		bodies := make([]string, n)
-		var wg sync.WaitGroup
-		for i := 0; i < n; i++ {
-			id := "tpl-" + strconv.Itoa(i)
-			request := &Request{ID: id, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, Path: []string{"{{BaseURL}}/slow"}}
-			executerOptions := testutils.NewMockExecuterOptions(opts, &testutils.TemplateInfo{
-				ID:   id,
-				Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
-			})
-			executerOptions.HTTPResponseCache = cache
-			require.NoError(t, request.Compile(executerOptions))
-			wg.Add(1)
-			go func(i int) {
-				defer wg.Done()
-				err := request.ExecuteWithResults(contextargs.NewWithInput(context.Background(), ts.URL), make(output.InternalEvent), make(output.InternalEvent), func(event *output.InternalWrappedEvent) {
-					if event != nil && event.InternalEvent != nil {
-						bodies[i], _ = event.InternalEvent["body"].(string)
-					}
-				})
-				require.NoError(t, err)
-			}(i)
-		}
-		wg.Wait()
-		return bodies
-	}
-
-	for _, body := range run(httprespcache.New()) {
-		require.Equal(t, "shared", body)
-	}
-	require.Equal(t, int32(1), hits.Load(), "concurrent identical requests must share one request")
-
-	for _, body := range run(nil) {
-		require.Equal(t, "shared", body)
-	}
-	require.Equal(t, int32(4), hits.Load(), "without the cache every request is sent")
-}

pkg/protocols/protocols.go

@@ -39,7 +39,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/excludematchers"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/variables"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
 	"github.com/projectdiscovery/nuclei/v3/pkg/scan"
 	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
@@ -119,9 +118,6 @@ type ExecutorOptions struct {
 	InteractshScope *interactsh.RequestScope
 	// HostErrorsCache is an optional cache for handling host errors
 	HostErrorsCache hosterrorscache.CacheInterface
-	// HTTPResponseCache reuses identical safe GET/HEAD responses inside one
-	// scan. Nil disables it.
-	HTTPResponseCache *httprespcache.Cache
 	// HostBackoff paces a host that signals it is being overloaded, so a scan
 	// slows down for it instead of being blocked outright. Nil disables it.
 	HostBackoff *hostbackoff.Governor
@@ -401,7 +397,6 @@ func (e *ExecutorOptions) Copy() *ExecutorOptions {
 		Browser:                      e.Browser,
 		Interactsh:                   e.Interactsh,
 		HostErrorsCache:              e.HostErrorsCache,
-		HTTPResponseCache:            e.HTTPResponseCache,
 		InteractshScope:              e.InteractshScope,
 		HostBackoff:                  e.HostBackoff,
 		StopAtFirstMatch:             e.StopAtFirstMatch,
@@ -710,7 +705,6 @@ func (e *ExecutorOptions) ApplyNewEngineOptions(n *ExecutorOptions) {
 	e.Browser = n.Browser
 	e.Interactsh = n.Interactsh
 	e.InteractshScope = n.InteractshScope
-	e.HTTPResponseCache = n.HTTPResponseCache
 	e.HostErrorsCache = n.HostErrorsCache
 	e.HostBackoff = n.HostBackoff
 	e.InputHelper = n.InputHelper

pkg/types/types.go

@@ -239,26 +239,10 @@ type Options struct {
 	// NOTE(dwisiswant0): this is experimental and might be enabled by default
 	// in the future releases.
 	EnableHTTPCache bool
-	// ResponseCache reuses identical safe GET/HEAD responses inside one scan.
-	// Only requests without auth, cookies, Host override, body, cache directives
-	// or non-allowlisted headers are eligible, so the reuse is lossless. Opt-in:
-	// a template that depends on state or timing could see a different result.
-	ResponseCache bool
 	// LeaveDefaultPorts skips normalization of default ports
 	LeaveDefaultPorts bool
 	// AutomaticScan enables automatic tech based template execution
 	AutomaticScan bool
-	// StrictProbe losslessly skips templates that cannot reach their
-	// target service (web templates on non-HTTP ports, network templates on
-	// closed ports).
-	StrictProbe bool
-	// TechFilter is an opt-in heuristic (CLI: -tf). When false (default), scan
-	// behavior matches pre-tech-filter nuclei: no fingerprinting, no product-tag
-	// skipping, no scan-scoped HTTP response cache. When true, fingerprints HTTP
-	// targets (Wappalyzer) and skips templates whose product/macro tags cannot
-	// match the host. Fail-open on unknown hosts and unbound templates. No-op
-	// when no templates are tech-bound.
-	TechFilter bool
 	// Silent suppresses any extra text and only writes found URLs on screen.
 	Silent bool
 	// Validate validates the templates passed to nuclei.
@@ -712,11 +696,8 @@ func (options *Options) Copy() *Options {
 		PerHostRateLimit:               options.PerHostRateLimit,
 		PerHostRateLimitPoolSize:       options.PerHostRateLimitPoolSize,
 		EnableHTTPCache:                options.EnableHTTPCache,
-		ResponseCache:                  options.ResponseCache,
 		LeaveDefaultPorts:              options.LeaveDefaultPorts,
 		AutomaticScan:                  options.AutomaticScan,
-		StrictProbe:                    options.StrictProbe,
-		TechFilter:                     options.TechFilter,
 		Silent:                         options.Silent,
 		Validate:                       options.Validate,
 		NoStrictSyntax:                 options.NoStrictSyntax,
@@ -943,12 +924,6 @@ func (options *Options) ShouldFollowHTTPRedirects() bool {
 	return options.FollowRedirects || options.FollowHostRedirects
 }
 
-// UsesResponseCache reports whether the scan-scoped response cache may be active:
-// -response-cache, or -tech-filter, which seeds the same cache.
-func (options *Options) UsesResponseCache() bool {
-	return options.ResponseCache || options.TechFilter
-}
-
 // HasClientCertificates determines if any client certificate was specified
 func (options *Options) HasClientCertificates() bool {
 	return options.ClientCertFile != "" || options.ClientCAFile != "" || options.ClientKeyFile != ""