review fixes

Give each scan its own DNS resolvers

DESIGN.md

@@ -36,7 +36,7 @@ It also specifies a `WorkflowLoader` interface that is used during workflow load
 
 ```go
 type WorkflowLoader interface {
-	GetTemplatePathsByTags(tags []string) []string
+	GetTemplatePathsByTags(tags []string, noValidate bool) []string
 	GetTemplatePaths(templatesList []string, noValidate bool) []string
 }
 ```

README.md

@@ -145,12 +145,12 @@ TARGET:
    -eh, -exclude-hosts string[]  hosts to exclude to scan from the input list (ip, cidr, hostname)
    -resume string                resume scan from and save to specified file (clustering will be disabled)
    -sa, -scan-all-ips            scan all the IP's associated with dns record
-   -iv, -ip-version string[]     IP version to scan of hostname (4,6) - (default 4)
+   -iv, -ip-version string[]     IP version to scan of hostname (4,6); default uses dialer fallback, -sa scans all addresses
    -dun, -disable-url-normalization  keep targets exactly as supplied instead of collapsing urls that address the same resource
    -mup, -max-urls-per-pattern int   scan at most this many urls sharing one structural shape, e.g. /user/1 and /user/2 (0 to scan all)
 
 TARGET-FORMAT:
-   -im, -input-mode string        mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list")
+   -im, -input-mode string        mode of input file (list, burp, har, jsonl, yaml, openapi, swagger, http) (default "list")
    -ro, -required-only            use only required fields in input format when generating requests
    -sfv, -skip-format-validation  skip format validation (like missing vars) when parsing input file
    -vtt, -vars-text-templating    enable text templating for vars in input file (only for yaml input mode)
@@ -177,7 +177,7 @@ TEMPLATES:
    -egm, -enable-global-matchers          enable loading global matchers templates
    -file                                  enable loading file templates
    -llm                                   enable llm matchers and extractors (semantic matching)
-   -llm-provider string                   llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
+   -llm-provider string                   llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter, jev) (default "openai")
    -llm-base-url string                   openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
    -llm-model string                      model used for semantic matching
    -llm-timeout int                       time in seconds to wait for a single llm call (default 30)
@@ -221,6 +221,7 @@ OUTPUT:
    -jle, -jsonl-export string    file to export results in JSONL(ine) format
    -pe, -pdf-export string       file to export results in PDF format
    -rd, -redact string[]         redact given list of keys from query parameter, request header and body
+   -nrd, -no-redact              disable the built-in redaction of credential headers (Authorization, Cookie, Set-Cookie, ...) in output; keys given with -redact are still redacted
 
 CONFIGURATIONS:
    -config string                        path to the nuclei configuration file
@@ -323,12 +324,10 @@ OPTIMIZATIONS:
    -stream                          stream mode - start elaborating without sorting the input
    -ss, -scan-strategy value        strategy to use while scanning(auto/host-spray/template-spray) (default auto)
    -irt, -input-read-timeout value  timeout on input read (default 3m0s)
-   -response-cache                  reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
    -nh, -no-httpx                   disable httpx probing for non-url input
-   -stp, -strict-probe              skip templates whose target service is unreachable: HTTP/headless on hosts httpx could not confirm and network templates on closed ports (lossless, no raw-input fallback)
-   -tf, -tech-filter                opt-in: fingerprint HTTP targets and skip product/macro-tagged templates that cannot match (off by default; enables scan-scoped GET cache; fail-open; no-op if no tech-bound templates)
    -preflight-portscan              run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
    -no-stdin                        disable stdin processing
+   -ac, -auto-calibrate             drop HTTP matches on a host's catch-all page (answers missing paths with 2xx/3xx)
 
 HEADLESS:
    -headless                        enable templates that require headless browser support (root user on Linux will disable sandbox)

README_CN.md

@@ -148,7 +148,7 @@ TARGET:
    -iv, -ip-version string[]     IP version to scan of hostname (4,6) - (default 4)
 
 TARGET-FORMAT:
-   -im, -input-mode string        mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list")
+   -im, -input-mode string        mode of input file (list, burp, har, jsonl, yaml, openapi, swagger, http) (default "list")
    -ro, -required-only            use only required fields in input format when generating requests
    -sfv, -skip-format-validation  skip format validation (like missing vars) when parsing input file
    -vtt, -vars-text-templating    enable text templating for vars in input file (only for yaml input mode)
@@ -175,7 +175,7 @@ TEMPLATES:
    -egm, -enable-global-matchers          enable loading global matchers templates
    -file                                  enable loading file templates
    -llm                                   enable llm matchers and extractors (semantic matching)
-   -llm-provider string                   llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
+   -llm-provider string                   llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter, jev) (default "openai")
    -llm-base-url string                   openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
    -llm-model string                      model used for semantic matching
    -llm-timeout int                       time in seconds to wait for a single llm call (default 30)
@@ -219,6 +219,7 @@ OUTPUT:
    -jle, -jsonl-export string    file to export results in JSONL(ine) format
    -pe, -pdf-export string       file to export results in PDF format
    -rd, -redact string[]         redact given list of keys from query parameter, request header and body
+   -nrd, -no-redact              disable the built-in redaction of credential headers (Authorization, Cookie, Set-Cookie, ...) in output; keys given with -redact are still redacted
 
 CONFIGURATIONS:
    -config string                        path to the nuclei configuration file
@@ -318,10 +319,7 @@ OPTIMIZATIONS:
    -stream                          stream mode - start elaborating without sorting the input
    -ss, -scan-strategy value        strategy to use while scanning(auto/host-spray/template-spray) (default auto)
    -irt, -input-read-timeout value  timeout on input read (default 3m0s)
-   -response-cache                  reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
    -nh, -no-httpx                   disable httpx probing for non-url input
-   -stp, -strict-probe              skip templates whose target service is unreachable: HTTP/headless on hosts httpx could not confirm and network templates on closed ports (lossless, no raw-input fallback)
-   -tf, -tech-filter                opt-in: fingerprint HTTP targets and skip product/macro-tagged templates that cannot match (off by default; enables scan-scoped GET cache; fail-open; no-op if no tech-bound templates)
    -preflight-portscan              run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
    -no-stdin                        disable stdin processing
 

README_ES.md

@@ -148,7 +148,7 @@ TARGET:
    -iv, -ip-version string[]     IP version to scan of hostname (4,6) - (default 4)
 
 TARGET-FORMAT:
-   -im, -input-mode string        mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list")
+   -im, -input-mode string        mode of input file (list, burp, har, jsonl, yaml, openapi, swagger, http) (default "list")
    -ro, -required-only            use only required fields in input format when generating requests
    -sfv, -skip-format-validation  skip format validation (like missing vars) when parsing input file
    -vtt, -vars-text-templating    enable text templating for vars in input file (only for yaml input mode)
@@ -175,7 +175,7 @@ TEMPLATES:
    -egm, -enable-global-matchers          enable loading global matchers templates
    -file                                  enable loading file templates
    -llm                                   enable llm matchers and extractors (semantic matching)
-   -llm-provider string                   llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
+   -llm-provider string                   llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter, jev) (default "openai")
    -llm-base-url string                   openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
    -llm-model string                      model used for semantic matching
    -llm-timeout int                       time in seconds to wait for a single llm call (default 30)
@@ -219,6 +219,7 @@ OUTPUT:
    -jle, -jsonl-export string    file to export results in JSONL(ine) format
    -pe, -pdf-export string       file to export results in PDF format
    -rd, -redact string[]         redact given list of keys from query parameter, request header and body
+   -nrd, -no-redact              disable the built-in redaction of credential headers (Authorization, Cookie, Set-Cookie, ...) in output; keys given with -redact are still redacted
 
 CONFIGURATIONS:
    -config string                        path to the nuclei configuration file
@@ -318,10 +319,7 @@ OPTIMIZATIONS:
    -stream                          stream mode - start elaborating without sorting the input
    -ss, -scan-strategy value        strategy to use while scanning(auto/host-spray/template-spray) (default auto)
    -irt, -input-read-timeout value  timeout on input read (default 3m0s)
-   -response-cache                  reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
    -nh, -no-httpx                   disable httpx probing for non-url input
-   -stp, -strict-probe              skip templates whose target service is unreachable: HTTP/headless on hosts httpx could not confirm and network templates on closed ports (lossless, no raw-input fallback)
-   -tf, -tech-filter                opt-in: fingerprint HTTP targets and skip product/macro-tagged templates that cannot match (off by default; enables scan-scoped GET cache; fail-open; no-op if no tech-bound templates)
    -preflight-portscan              run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
    -no-stdin                        disable stdin processing
 

README_FA.md

@@ -165,7 +165,7 @@ TARGET:
    -mup, -max-urls-per-pattern int   scan at most this many urls sharing one structural shape, e.g. /user/1 and /user/2 (0 to scan all)
 
 TARGET-FORMAT:
-   -im, -input-mode string        mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list")
+   -im, -input-mode string        mode of input file (list, burp, har, jsonl, yaml, openapi, swagger, http) (default "list")
    -ro, -required-only            use only required fields in input format when generating requests
    -sfv, -skip-format-validation  skip format validation (like missing vars) when parsing input file
    -vtt, -vars-text-templating    enable text templating for vars in input file (only for yaml input mode)
@@ -192,7 +192,7 @@ TEMPLATES:
    -egm, -enable-global-matchers          enable loading global matchers templates
    -file                                  enable loading file templates
    -llm                                   enable llm matchers and extractors (semantic matching)
-   -llm-provider string                   llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
+   -llm-provider string                   llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter, jev) (default "openai")
    -llm-base-url string                   openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
    -llm-model string                      model used for semantic matching
    -llm-timeout int                       time in seconds to wait for a single llm call (default 30)
@@ -236,6 +236,7 @@ OUTPUT:
    -jle, -jsonl-export string    file to export results in JSONL(ine) format
    -pe, -pdf-export string       file to export results in PDF format
    -rd, -redact string[]         redact given list of keys from query parameter, request header and body
+   -nrd, -no-redact              disable the built-in redaction of credential headers (Authorization, Cookie, Set-Cookie, ...) in output; keys given with -redact are still redacted
 
 CONFIGURATIONS:
    -config string                        path to the nuclei configuration file
@@ -338,10 +339,7 @@ OPTIMIZATIONS:
    -stream                          stream mode - start elaborating without sorting the input
    -ss, -scan-strategy value        strategy to use while scanning(auto/host-spray/template-spray) (default auto)
    -irt, -input-read-timeout value  timeout on input read (default 3m0s)
-   -response-cache                  reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
    -nh, -no-httpx                   disable httpx probing for non-url input
-   -stp, -strict-probe              skip templates whose target service is unreachable: HTTP/headless on hosts httpx could not confirm and network templates on closed ports (lossless, no raw-input fallback)
-   -tf, -tech-filter                opt-in: fingerprint HTTP targets and skip product/macro-tagged templates that cannot match (off by default; enables scan-scoped GET cache; fail-open; no-op if no tech-bound templates)
    -preflight-portscan              run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
    -no-stdin                        disable stdin processing
 

README_ID.md

@@ -148,7 +148,7 @@ TARGET:
    -iv, -ip-version string[]     IP version to scan of hostname (4,6) - (default 4)
 
 TARGET-FORMAT:
-   -im, -input-mode string        mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list")
+   -im, -input-mode string        mode of input file (list, burp, har, jsonl, yaml, openapi, swagger, http) (default "list")
    -ro, -required-only            use only required fields in input format when generating requests
    -sfv, -skip-format-validation  skip format validation (like missing vars) when parsing input file
    -vtt, -vars-text-templating    enable text templating for vars in input file (only for yaml input mode)
@@ -175,7 +175,7 @@ TEMPLATES:
    -egm, -enable-global-matchers          enable loading global matchers templates
    -file                                  enable loading file templates
    -llm                                   enable llm matchers and extractors (semantic matching)
-   -llm-provider string                   llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
+   -llm-provider string                   llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter, jev) (default "openai")
    -llm-base-url string                   openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
    -llm-model string                      model used for semantic matching
    -llm-timeout int                       time in seconds to wait for a single llm call (default 30)
@@ -219,6 +219,7 @@ OUTPUT:
    -jle, -jsonl-export string    file to export results in JSONL(ine) format
    -pe, -pdf-export string       file to export results in PDF format
    -rd, -redact string[]         redact given list of keys from query parameter, request header and body
+   -nrd, -no-redact              disable the built-in redaction of credential headers (Authorization, Cookie, Set-Cookie, ...) in output; keys given with -redact are still redacted
 
 CONFIGURATIONS:
    -config string                        path to the nuclei configuration file
@@ -318,10 +319,7 @@ OPTIMIZATIONS:
    -stream                          stream mode - start elaborating without sorting the input
    -ss, -scan-strategy value        strategy to use while scanning(auto/host-spray/template-spray) (default auto)
    -irt, -input-read-timeout value  timeout on input read (default 3m0s)
-   -response-cache                  reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
    -nh, -no-httpx                   disable httpx probing for non-url input
-   -stp, -strict-probe              skip templates whose target service is unreachable: HTTP/headless on hosts httpx could not confirm and network templates on closed ports (lossless, no raw-input fallback)
-   -tf, -tech-filter                opt-in: fingerprint HTTP targets and skip product/macro-tagged templates that cannot match (off by default; enables scan-scoped GET cache; fail-open; no-op if no tech-bound templates)
    -preflight-portscan              run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
    -no-stdin                        disable stdin processing
 

README_JP.md

@@ -148,7 +148,7 @@ TARGET:
    -iv, -ip-version string[]     IP version to scan of hostname (4,6) - (default 4)
 
 TARGET-FORMAT:
-   -im, -input-mode string        mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list")
+   -im, -input-mode string        mode of input file (list, burp, har, jsonl, yaml, openapi, swagger, http) (default "list")
    -ro, -required-only            use only required fields in input format when generating requests
    -sfv, -skip-format-validation  skip format validation (like missing vars) when parsing input file
    -vtt, -vars-text-templating    enable text templating for vars in input file (only for yaml input mode)
@@ -175,7 +175,7 @@ TEMPLATES:
    -egm, -enable-global-matchers          enable loading global matchers templates
    -file                                  enable loading file templates
    -llm                                   enable llm matchers and extractors (semantic matching)
-   -llm-provider string                   llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
+   -llm-provider string                   llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter, jev) (default "openai")
    -llm-base-url string                   openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
    -llm-model string                      model used for semantic matching
    -llm-timeout int                       time in seconds to wait for a single llm call (default 30)
@@ -219,6 +219,7 @@ OUTPUT:
    -jle, -jsonl-export string    file to export results in JSONL(ine) format
    -pe, -pdf-export string       file to export results in PDF format
    -rd, -redact string[]         redact given list of keys from query parameter, request header and body
+   -nrd, -no-redact              disable the built-in redaction of credential headers (Authorization, Cookie, Set-Cookie, ...) in output; keys given with -redact are still redacted
 
 CONFIGURATIONS:
    -config string                        path to the nuclei configuration file
@@ -318,10 +319,7 @@ OPTIMIZATIONS:
    -stream                          stream mode - start elaborating without sorting the input
    -ss, -scan-strategy value        strategy to use while scanning(auto/host-spray/template-spray) (default auto)
    -irt, -input-read-timeout value  timeout on input read (default 3m0s)
-   -response-cache                  reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
    -nh, -no-httpx                   disable httpx probing for non-url input
-   -stp, -strict-probe              skip templates whose target service is unreachable: HTTP/headless on hosts httpx could not confirm and network templates on closed ports (lossless, no raw-input fallback)
-   -tf, -tech-filter                opt-in: fingerprint HTTP targets and skip product/macro-tagged templates that cannot match (off by default; enables scan-scoped GET cache; fail-open; no-op if no tech-bound templates)
    -preflight-portscan              run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
    -no-stdin                        disable stdin processing
 

README_KR.md

@@ -148,7 +148,7 @@ TARGET:
    -iv, -ip-version string[]     IP version to scan of hostname (4,6) - (default 4)
 
 TARGET-FORMAT:
-   -im, -input-mode string        mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list")
+   -im, -input-mode string        mode of input file (list, burp, har, jsonl, yaml, openapi, swagger, http) (default "list")
    -ro, -required-only            use only required fields in input format when generating requests
    -sfv, -skip-format-validation  skip format validation (like missing vars) when parsing input file
    -vtt, -vars-text-templating    enable text templating for vars in input file (only for yaml input mode)
@@ -175,7 +175,7 @@ TEMPLATES:
    -egm, -enable-global-matchers          enable loading global matchers templates
    -file                                  enable loading file templates
    -llm                                   enable llm matchers and extractors (semantic matching)
-   -llm-provider string                   llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
+   -llm-provider string                   llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter, jev) (default "openai")
    -llm-base-url string                   openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
    -llm-model string                      model used for semantic matching
    -llm-timeout int                       time in seconds to wait for a single llm call (default 30)
@@ -219,6 +219,7 @@ OUTPUT:
    -jle, -jsonl-export string    file to export results in JSONL(ine) format
    -pe, -pdf-export string       file to export results in PDF format
    -rd, -redact string[]         redact given list of keys from query parameter, request header and body
+   -nrd, -no-redact              disable the built-in redaction of credential headers (Authorization, Cookie, Set-Cookie, ...) in output; keys given with -redact are still redacted
 
 CONFIGURATIONS:
    -config string                        path to the nuclei configuration file
@@ -318,10 +319,7 @@ OPTIMIZATIONS:
    -stream                          stream mode - start elaborating without sorting the input
    -ss, -scan-strategy value        strategy to use while scanning(auto/host-spray/template-spray) (default auto)
    -irt, -input-read-timeout value  timeout on input read (default 3m0s)
-   -response-cache                  reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
    -nh, -no-httpx                   disable httpx probing for non-url input
-   -stp, -strict-probe              skip templates whose target service is unreachable: HTTP/headless on hosts httpx could not confirm and network templates on closed ports (lossless, no raw-input fallback)
-   -tf, -tech-filter                opt-in: fingerprint HTTP targets and skip product/macro-tagged templates that cannot match (off by default; enables scan-scoped GET cache; fail-open; no-op if no tech-bound templates)
    -preflight-portscan              run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
    -no-stdin                        disable stdin processing
 

README_PT-BR.md

@@ -148,7 +148,7 @@ TARGET:
    -iv, -ip-version string[]     IP version to scan of hostname (4,6) - (default 4)
 
 TARGET-FORMAT:
-   -im, -input-mode string        mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list")
+   -im, -input-mode string        mode of input file (list, burp, har, jsonl, yaml, openapi, swagger, http) (default "list")
    -ro, -required-only            use only required fields in input format when generating requests
    -sfv, -skip-format-validation  skip format validation (like missing vars) when parsing input file
    -vtt, -vars-text-templating    enable text templating for vars in input file (only for yaml input mode)
@@ -175,7 +175,7 @@ TEMPLATES:
    -egm, -enable-global-matchers          enable loading global matchers templates
    -file                                  enable loading file templates
    -llm                                   enable llm matchers and extractors (semantic matching)
-   -llm-provider string                   llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
+   -llm-provider string                   llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter, jev) (default "openai")
    -llm-base-url string                   openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
    -llm-model string                      model used for semantic matching
    -llm-timeout int                       time in seconds to wait for a single llm call (default 30)
@@ -219,6 +219,7 @@ OUTPUT:
    -jle, -jsonl-export string    file to export results in JSONL(ine) format
    -pe, -pdf-export string       file to export results in PDF format
    -rd, -redact string[]         redact given list of keys from query parameter, request header and body
+   -nrd, -no-redact              disable the built-in redaction of credential headers (Authorization, Cookie, Set-Cookie, ...) in output; keys given with -redact are still redacted
 
 CONFIGURATIONS:
    -config string                        path to the nuclei configuration file
@@ -318,10 +319,7 @@ OPTIMIZATIONS:
    -stream                          stream mode - start elaborating without sorting the input
    -ss, -scan-strategy value        strategy to use while scanning(auto/host-spray/template-spray) (default auto)
    -irt, -input-read-timeout value  timeout on input read (default 3m0s)
-   -response-cache                  reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
    -nh, -no-httpx                   disable httpx probing for non-url input
-   -stp, -strict-probe              skip templates whose target service is unreachable: HTTP/headless on hosts httpx could not confirm and network templates on closed ports (lossless, no raw-input fallback)
-   -tf, -tech-filter                opt-in: fingerprint HTTP targets and skip product/macro-tagged templates that cannot match (off by default; enables scan-scoped GET cache; fail-open; no-op if no tech-bound templates)
    -preflight-portscan              run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
    -no-stdin                        disable stdin processing
 

README_TR.md

@@ -148,7 +148,7 @@ TARGET:
    -iv, -ip-version string[]     IP version to scan of hostname (4,6) - (default 4)
 
 TARGET-FORMAT:
-   -im, -input-mode string        mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list")
+   -im, -input-mode string        mode of input file (list, burp, har, jsonl, yaml, openapi, swagger, http) (default "list")
    -ro, -required-only            use only required fields in input format when generating requests
    -sfv, -skip-format-validation  skip format validation (like missing vars) when parsing input file
    -vtt, -vars-text-templating    enable text templating for vars in input file (only for yaml input mode)
@@ -175,7 +175,7 @@ TEMPLATES:
    -egm, -enable-global-matchers          enable loading global matchers templates
    -file                                  enable loading file templates
    -llm                                   enable llm matchers and extractors (semantic matching)
-   -llm-provider string                   llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter) (default "openai")
+   -llm-provider string                   llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter, jev) (default "openai")
    -llm-base-url string                   openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)
    -llm-model string                      model used for semantic matching
    -llm-timeout int                       time in seconds to wait for a single llm call (default 30)
@@ -219,6 +219,7 @@ OUTPUT:
    -jle, -jsonl-export string    file to export results in JSONL(ine) format
    -pe, -pdf-export string       file to export results in PDF format
    -rd, -redact string[]         redact given list of keys from query parameter, request header and body
+   -nrd, -no-redact              disable the built-in redaction of credential headers (Authorization, Cookie, Set-Cookie, ...) in output; keys given with -redact are still redacted
 
 CONFIGURATIONS:
    -config string                        path to the nuclei configuration file
@@ -318,10 +319,7 @@ OPTIMIZATIONS:
    -stream                          stream mode - start elaborating without sorting the input
    -ss, -scan-strategy value        strategy to use while scanning(auto/host-spray/template-spray) (default auto)
    -irt, -input-read-timeout value  timeout on input read (default 3m0s)
-   -response-cache                  reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
    -nh, -no-httpx                   disable httpx probing for non-url input
-   -stp, -strict-probe              skip templates whose target service is unreachable: HTTP/headless on hosts httpx could not confirm and network templates on closed ports (lossless, no raw-input fallback)
-   -tf, -tech-filter                opt-in: fingerprint HTTP targets and skip product/macro-tagged templates that cannot match (off by default; enables scan-scoped GET cache; fail-open; no-op if no tech-bound templates)
    -preflight-portscan              run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
    -no-stdin                        disable stdin processing
 

SECURITY_CONTEXT.md

@@ -1,255 +1,120 @@
-<!-- Generated by securitycontext.dev. Source HEAD: 3e1e0e81f091c7817bbf7af64bb8eec6e8c2bac1. -->
+<!-- Generated by securitycontext.dev. Source HEAD: df57ae6d2955864fd92a1bd2565fe9e4e9a7592b. -->
 # Security Context: projectdiscovery/nuclei @ dev
 
 > Reference context for editing projectdiscovery/nuclei. Before changing any component below, **preserve the listed guards and do not reintroduce these fixed issues**. Each fingerprint is a regression check, and each shared attack surface is a guard that must keep holding on every path.
 
-_Generated 2026-09-01 16:22 UTC by secc_
-
-_Analyzed commit: 3e1e0e81f091c7817bbf7af64bb8eec6e8c2bac1_
-
-**Remediation coverage: 56.7%**
-_17 of 30 historical fixes have no remaining unguarded variant; 368 open remediation leads to verify. Measures fix consistency, not a vulnerability verdict._
-
-- Commits analyzed: **4814**
-- Commits flagged for review: **235**
-- Confirmed historical security fixes: **32**
-- Known published CVEs: **0**
-
-## Summary
-
-Nuclei's historical security posture is dominated by sandbox escapes and code execution via unsigned or unverified templates, with recurring SSRF, path traversal, and auth bypass issues across JS libraries, loaders, and workflow paths. The most critical theme is inconsistent enforcement of signature verification and local-file-access (-lfa) guards across direct, workflow, DAST, and JS-sandbox entry points, enabling variant attacks. A second theme is network policy (host allowlist) gaps in LDAP dialing and proxy handling. The security context also includes dependency upgrades (Alpine, go-git) and output file permission hardening, but the highest-value focus is on template trust boundaries and sandbox enforcement parity.
+_Analyzed [df57ae6d29][c-df57ae6d29] on `dev` · 5174 commits, 548 security-relevant examined · generated 2026-11-02_
 
 ## Recurring Security Patterns
 
-**15.6% of classified fixes repeat a previously fixed pattern** (5 recurring, 27 first-seen). A recurring pattern means this repository fixed a similar issue before — not that the code is currently vulnerable.
-
-- **Code Execution:** [1c440e755a](https://github.com/projectdiscovery/nuclei/commit/1c440e755a97471c56fb0276ee8a8c4132974645) → [9de96e4dda](https://github.com/projectdiscovery/nuclei/commit/9de96e4dda5a03da963b9ae6582f03ea55791a76) — matched on `class:code execution`, `component:pkg/catalog/loader/loader.go`
-- **Code Execution via Unsigned Template:** [fa56800fcc](https://github.com/projectdiscovery/nuclei/commit/fa56800fcca3c8c8fe8e8dec28584210ca1527de) → [fac0b7196d](https://github.com/projectdiscovery/nuclei/commit/fac0b7196d2fbca56c8af8ff8e1899c111041a37) — matched on `class:code execution via unsigned template`, `CWE-347`, `sink:loadtemplates`
-- **SSRF:** [a167e6c57b](https://github.com/projectdiscovery/nuclei/commit/a167e6c57b80f5dc7be231231b62d7c594ee2770) → [9def9af7e3](https://github.com/projectdiscovery/nuclei/commit/9def9af7e3f7ebcf5cd561c37b623b9999ee7572) — matched on `class:ssrf`, `CWE-918`, `component:pkg/js/libs/ldap/ldap.go`
-- **Dependency Vulnerability:** [d93b4a01df](https://github.com/projectdiscovery/nuclei/commit/d93b4a01df9c218a36dd2fccfb1c2c1f96b181fe) → [fcd5c6b111](https://github.com/projectdiscovery/nuclei/commit/fcd5c6b111ddcb746525f88b7d0bf4b4abbc2529) — matched on `class:dependency vulnerability`, `CWE-1104`, `component:dockerfile`
-- **Sandbox Escape:** [96049a32fa](https://github.com/projectdiscovery/nuclei/commit/96049a32fa70aef0483d4180d386a57b3c2d973a) → [cf2a98b03d](https://github.com/projectdiscovery/nuclei/commit/cf2a98b03d76327c20fb0183b7f1c849eb692c91) — matched on `class:sandbox escape`, `CWE-284`, `component:v2/internal/runner/processor.go`, `sink:stdlib allmodulenames`, `sink:stdlib getmodulemap`
+**7.1% of classified fixes repeat a previously fixed pattern** (1 recurring, 13 first-seen). A recurring pattern means this repository fixed a similar issue before — not that the code is currently vulnerable.
 
 ## Top Risks
 
-### 1. Sandbox Escape: `pkg/js/compiler/pool.go, v2/pkg/protocols/javascript/js.go, v2/pkg/protocols/headless/engine/engine.go, v2/internal/runner/processor.go` (high)
+Components with the strongest concentration of past fixes for one class.
 
-Multiple sandbox escapes across JS require(), eval/ReadFile, headless browser flags, and tengo imports, indicating weak default sandbox boundaries that require consistent -lfa and safe-module enforcement.
-
-Evidence: 6f2ade6a9b427c284c15a43445f9c7f055e60e5d, c35162c8ef061a96dd74b54c8f022c1a41cf5833, df55f7a2eb591a7949b5c66a9c54db54aa28f450, cf2a98b03d76327c20fb0183b7f1c849eb692c91, 96049a32fa70aef0483d4180d386a57b3c2d973a
-
-### 2. Code Execution via Unsigned Template: `pkg/catalog/loader/loader.go, pkg/templates/workflows.go, pkg/templates/templates.go` (critical)
-
-Unsigned templates can execute code through direct loading, workflows, and DAST branches; signature verification was bypassed via cache poisoning and missing guards on specific paths.
-
-Evidence: 9de96e4dda5a03da963b9ae6582f03ea55791a76, 1c440e755a97471c56fb0276ee8a8c4132974645, fac0b7196d2fbca56c8af8ff8e1899c111041a37, e86f38299765b82ad724fdb701557e0eaff3884d, fa56800fcca3c8c8fe8e8dec28584210ca1527de
-
-### 3. Auth Bypass: `pkg/templates/compile.go, pkg/templates/signer/tmpl_signer.go, pkg/catalog/loader/loader.go, pkg/protocols/http/request.go` (high)
-
-Signature verification can be bypassed via cached metadata, improper signature extraction, or missing checks on specific load paths, and cookie jar bypass via disabled cookie handling.
-
-Evidence: 4534e9cb3034ebe5cc7b0cecabe399b6fb9ce0e4, 0da993afe6d41b4b1b814e8fad23a2acba13c60a, 9bd4db3f74aedd067af568df4f9198a8a090f68b, a976ca8d3f6a18c9ca746c8ce73f28fdcd518642
-
-### 4. SSRF: `pkg/js/libs/ldap/ldap.go, v2/internal/runner/proxy.go` (high)
-
-SSRF via LDAP dialing to internal hosts and via proxy configuration; host allowlist checks were missing or applied to the wrong target (domain vs controller).
-
-Evidence: 9def9af7e3f7ebcf5cd561c37b623b9999ee7572, a167e6c57b80f5dc7be231231b62d7c594ee2770, 0e46d3e0410f89e4af41099f5cd38fa7f54ae54f
-
-### 5. Path Traversal: `pkg/catalog/config/nucleiconfig.go, v2/internal/runner/update.go, v2/pkg/protocols/common/protocolstate/headless.go` (high)
-
-Path traversal via lexical prefix checks, zip-slip in template extraction, and file:// scheme in headless mode; canonicalization and containment checks are required.
-
-Evidence: 20cbf0bd73893c1270582c9778b6b26a5cb11206, f9c214a66f37a00fc29fe64232b690e6285ec59c, d3928e080d819f25306bfbaecc399f03516db5aa
-
-### 6. Code Execution: `pkg/catalog/loader/loader.go, pkg/templates/workflows.go` (high)
-
-Code execution via cache poisoning and unverified code templates in workflows; requires -code flag and verified templates.
-
-Evidence: 9de96e4dda5a03da963b9ae6582f03ea55791a76, e86f38299765b82ad724fdb701557e0eaff3884d
-
-## Shared Attack Surfaces
-
-Capabilities guarded by more than one past fix on different code paths. A guard enforced on one entry point is often missing on a sibling; verify each alternate path before assuming it's covered.
-
-### 1. Template signature verification and unsigned-template opt-in enforcement across all load paths
-
-- Guard that should hold everywhere: signature verification (template.Verified, Verify), DisableUnsignedTemplates flag, -code flag for code templates, cache invalidation on verification status/fingerprint/digest
-- Entry points: `pkg/catalog/loader/loader.go (LoadTemplatesWithTags, DAST branch)`, `pkg/templates/workflows.go (loadTemplates, workflowExecutor)`, `pkg/templates/templates.go (loadTemplates)`, `pkg/templates/compile.go (template.Verified)`, `pkg/templates/signer/tmpl_signer.go (Verify)`
-- Check: Inspect the DAST and workflow sub-loaders: they must enforce the same DisableUnsignedTemplates and -code checks as the direct loader, and must not rely on cached verification results without re-checking signatures.
-- Evidence: 9de96e4dda5a03da963b9ae6582f03ea55791a76, 1c440e755a97471c56fb0276ee8a8c4132974645, fac0b7196d2fbca56c8af8ff8e1899c111041a37, e86f38299765b82ad724fdb701557e0eaff3884d, fa56800fcca3c8c8fe8e8dec28584210ca1527de, 4534e9cb3034ebe5cc7b0cecabe399b6fb9ce0e4, 0da993afe6d41b4b1b814e8fad23a2acba13c60a, 9bd4db3f74aedd067af568df4f9198a8a090f68b
-
-### 2. Local file access (read/write) outside the sandbox, gated by -lfa flag
-
-- Guard that should hold everywhere: -lfa flag enforcement, path canonicalization and containment (sandbox directory), file:// scheme blocking, allowAllFiles stripping, require() registry rebuild
-- Entry points: `pkg/js/libs/krbforge/krbforge.go (CreateTicket)`, `pkg/js/libs/mysql/mysql_private.go (sql.Open)`, `pkg/js/compiler/pool.go (require)`, `v2/pkg/protocols/javascript/js.go (ReadFile, eval)`, `v2/pkg/protocols/common/protocolstate/headless.go (file:)`, `v2/pkg/protocols/common/generators/load.go (loadPayloadsFromFile)`
-- Check: Verify that every JS library and protocol helper (krbforge, mysql, require, headless file://, payload loaders) enforces the -lfa flag and path containment; a sibling path that skips the -lfa check is a sandbox escape.
-- Evidence: 3a8b65cb45f121fcdf418f60e5177ed9f250c122, 9611926a9a1ca99e8ed554d1b26fbeff60e06368, 6f2ade6a9b427c284c15a43445f9c7f055e60e5d, c35162c8ef061a96dd74b54c8f022c1a41cf5833, d3928e080d819f25306bfbaecc399f03516db5aa, e7fb40a41308f6c41267e858d1781eb67f7e0572
-
-### 3. Network policy (host allowlist) enforcement before outbound connections
-
-- Guard that should hold everywhere: IsHostAllowed check on the actual target host (controller, not domain), proxy validation and connectivity check, ProxyInternal gate for HTTP_PROXY env
-- Entry points: `pkg/js/libs/ldap/ldap.go (Dial, newLdapSession)`, `v2/internal/runner/proxy.go (loadProxyServers, os.Setenv HTTP_PROXY)`
-- Check: Inspect other dialing paths (e.g., HTTP, headless) to ensure they apply the same host allowlist check on the resolved IP/host, not just a user-supplied domain, and that proxy settings cannot redirect traffic to internal hosts.
-- Evidence: 9def9af7e3f7ebcf5cd561c37b623b9999ee7572, a167e6c57b80f5dc7be231231b62d7c594ee2770, 0e46d3e0410f89e4af41099f5cd38fa7f54ae54f, 3f5c055aca1499642caf4f40e8cf55ad32bd4da9
-
-### 4. Path traversal in file extraction and template path trust boundaries
-
-- Guard that should hold everywhere: canonical path containment check (filepath.Rel, prefix), zip-slip validation against template directory, sandbox path prefix validation
-- Entry points: `pkg/catalog/config/nucleiconfig.go (IsCustomTemplate)`, `v2/internal/runner/update.go (os.OpenFile)`, `v2/pkg/protocols/common/generators/load.go (loadPayloadsFromFile)`
-- Check: Audit all file-write and file-read paths that accept template-controlled paths (e.g., template extraction, payload loading) to ensure they use canonical containment, not lexical prefix checks, and apply the same sandbox directory restriction.
-- Evidence: 20cbf0bd73893c1270582c9778b6b26a5cb11206, f9c214a66f37a00fc29fe64232b690e6285ec59c, e7fb40a41308f6c41267e858d1781eb67f7e0572
-
-## Historical Vulnerability Classes
-
-| Class | Count |
-|---|---|
-| Sandbox Escape | 7 |
-| Auth Bypass | 4 |
-| Code Execution | 3 |
-| Dependency Vulnerability | 3 |
-| Path Traversal | 3 |
-| SSRF | 3 |
-| Code Execution via Unsigned Template | 2 |
-| Arbitrary File Read | 1 |
-| Cryptographic Weakness | 1 |
-| Deserialization of Untrusted Data | 1 |
-| Information Disclosure | 1 |
-| Insecure File Permissions | 1 |
-| Race Condition | 1 |
-| Security Misconfiguration | 1 |
+| # | Class | Component | Severity | Fixes | Evidence |
+|---|---|---|---|---|---|
+| 1 | SSRF | `pkg/protocols/common/protocolstate/headless.go` | high | 2 | [d43c1f547f][c-d43c1f547f], [d9dabbb4a1][c-d9dabbb4a1] |
+| 2 | Arbitrary File Access | `v2/pkg/protocols/common/generators/load.go` | high | 0 (1 component gone, not counted) | [e5154d362a][c-e5154d362a] |
+| 3 | Arbitrary File Access | `pkg/js/libs/mysql/mysql.go` | high | 1 | [634cecbb28][c-634cecbb28] |
+| 4 | Code Execution | `pkg/fuzz/execute.go` | high | 1 | [ccbfb12bd0][c-ccbfb12bd0] |
+| 5 | Code Execution | `pkg/protocols/common/expressions/expressions.go` | high | 1 | [d221732016][c-d221732016] |
+| 6 | Information Disclosure | `internal/httpapi/apiendpoint.go` | high | 1 | [a20aa26024][c-a20aa26024] |
 
 ## Fixed-Issue Fingerprints
 
-These are real security fixes mined from git history. Use them as regression checks.
+Real security fixes mined from git history; use them as regression checks. `verified` fixes are linked to a published advisory, `inferred` fixes were confirmed from the diff evidence alone. Each is identified by its short commit SHA, which the variant-leads report uses to reference it back to this history.
 
-- **Auth Bypass** (high) [CWE-287]: Honors disable-cookie by preventing the shared cookie jar from being applied when cookie handling is disabled, closing a path where cookies could be reused across requests.
-  - sink: `SetCookieJar(input.CookieJar)`
-  - fix: Adds !request.DisableCookie check before applying shared cookie jar
-  - commit: https://github.com/projectdiscovery/nuclei/commit/a976ca8d3f6a18c9ca746c8ce73f28fdcd518642 (2026-08-25)
-- **Code Execution** (high) [CWE-347]: This commit fixes a cache poisoning vulnerability where unsigned templates could bypass signature verification and execute malicious code during compilation or request execution.
-  - sink: `template compilation and execution of JavaScript`
-  - fix: cache invalidation based on verification status, verifier fingerprint, and content digest
-  - commit: https://github.com/projectdiscovery/nuclei/commit/9de96e4dda5a03da963b9ae6582f03ea55791a76 (2026-08-04)
-- **Code Execution via Unsigned Template** (high) [CWE-347]: Enforces signature verification on JavaScript templates, preventing execution of unsigned templates that could lead to arbitrary code execution.
-  - sink: `loading unsigned javascript templates into execution paths`
-  - fix: Require signature verification for javascript templates in direct loading and workflows when signing is enforced
-  - commit: https://github.com/projectdiscovery/nuclei/commit/fac0b7196d2fbca56c8af8ff8e1899c111041a37 (2026-07-03)
-- **Sandbox Escape** (high) [CWE-22]: Enforces nuclei's local file access policy on krbforge output paths, preventing arbitrary file writes outside the sandbox.
-  - sink: `gpkrb.CreateTicket(cfg) with cfg.OutputFile`
-  - fix: Path normalization into sandbox and denial of outside paths unless -lfa
-  - commit: https://github.com/projectdiscovery/nuclei/commit/3a8b65cb45f121fcdf418f60e5177ed9f250c122 (2026-06-24)
-- **SSRF** (high) [CWE-918]: Adds network policy host allowlist checks before LDAP dialing, preventing SSRF to restricted internal hosts.
-  - sink: `dialers.Fastdialer.Dial`
-  - fix: host allowlist check via IsHostAllowed before dialing
-  - commit: https://github.com/projectdiscovery/nuclei/commit/9def9af7e3f7ebcf5cd561c37b623b9999ee7572 (2026-06-24)
-- **Arbitrary File Read** (high) [CWE-22]: Gates MySQL's allowAllFiles option behind the -lfa flag, preventing arbitrary local file reads via LOAD DATA LOCAL INFILE.
-  - sink: `sql.Open("mysql", dsn) with AllowAllFiles=true`
-  - fix: DSN sandbox that strips allowAllFiles unless -lfa is set
-  - commit: https://github.com/projectdiscovery/nuclei/commit/9611926a9a1ca99e8ed554d1b26fbeff60e06368 (2026-06-19)
-- **Code Execution** (critical) [CWE-94]: Moves the unsigned code-template guard ahead of the DAST branch, preventing unsigned code templates from being loaded and executed under DAST, which previously bypassed the signature check.
-  - sink: `Template loading and execution`
-  - fix: Hoisting unsigned code-template guard before DAST branch to ensure it applies on all load paths
-  - commit: https://github.com/projectdiscovery/nuclei/commit/1c440e755a97471c56fb0276ee8a8c4132974645 (2026-06-19)
-- **Path Traversal** (high) [CWE-22]: This commit replaces lexical prefix checks with canonical path containment checks across multiple filesystem trust boundaries, preventing path traversal where sibling paths were incorrectly treated as trusted.
-  - sink: `IsCustomTemplate`
-  - fix: path canonicalization and containment check
-  - commit: https://github.com/projectdiscovery/nuclei/commit/20cbf0bd73893c1270582c9778b6b26a5cb11206 (2026-06-04)
-- **Information Disclosure** (medium) [CWE-200]: This commit fixes a data leakage issue in workflows where sibling state could leak across concurrent executions, and race requests ignored dynamic values, by syncing context and cloning inputs.
-  - sink: `syncWorkflowInputToTemplateCtx`
-  - fix: context synchronization and input cloning
-  - commit: https://github.com/projectdiscovery/nuclei/commit/805bcfe76f3629dbe71021795d4ce865406c1555 (2026-05-08)
-- **Sandbox Escape** (high) [CWE-284]: Fixes sandbox escape where require() bypassed local file access restrictions and reused privileged module cache across executions.
-  - sink: `require() using default host filesystem loader`
-  - fix: Rebuild require registry per execution and enforce allow-local-file-access sandbox rules
-  - commit: https://github.com/projectdiscovery/nuclei/commit/6f2ade6a9b427c284c15a43445f9c7f055e60e5d (2026-04-10)
-- **Cryptographic Weakness** (medium) [CWE-338]: Replaces predictable math/rand PRNG with cryptographically secure crypto/rand in JS helper functions.
-  - sink: `rand.Intn(255) / rand.Int63()`
-  - fix: Replace predictable math/rand with crypto/rand
-  - commit: https://github.com/projectdiscovery/nuclei/commit/44ef9ce34ffcfdfca3a96626abc73d99dd0e83ab (2026-03-15)
-- **Auth Bypass** (high) [CWE-347]: Caching signature verification results without cryptographic re-check allows a metadata index compromise to bypass template signature verification.
-  - sink: `template.Verified, template.TemplateVerifier`
-  - fix: cached verification data reuse
-  - commit: https://github.com/projectdiscovery/nuclei/commit/4534e9cb3034ebe5cc7b0cecabe399b6fb9ce0e4 (2026-01-21)
-- **Race Condition** (medium) [CWE-362]: Fixes a race condition in workflow execution caused by shared context callbacks, which could lead to data corruption or crashes; the primary impact is reliability, not a direct security breach.
-  - sink: `shared context callbacks in workflow execution`
-  - fix: fix race condition by making workflow execution sequential
-  - commit: https://github.com/projectdiscovery/nuclei/commit/24d1f58682027e23cb21a299a170abd2bdf20dc2 (2025-12-05)
-- **Auth Bypass** (high) [CWE-347]: Fixes template signature verification by centralizing signature extraction and requiring the signature to be at the end of the template, preventing bypass.
-  - sink: `Verify`
-  - fix: signature extraction and verification
-  - commit: https://github.com/projectdiscovery/nuclei/commit/0da993afe6d41b4b1b814e8fad23a2acba13c60a (2024-08-19)
-- **Dependency Vulnerability** (medium) [CWE-1104]: Upgrades Alpine base image from 3.18.5 to 3.18.6 to fix known security vulnerabilities in bundled packages.
-  - sink: `FROM alpine:3.18.5`
-  - fix: Upgrade base image to patched version
-  - commit: https://github.com/projectdiscovery/nuclei/commit/fcd5c6b111ddcb746525f88b7d0bf4b4abbc2529 (2024-03-15)
-- **Dependency Vulnerability** (medium) [CWE-1104]: Upgrades Alpine base image from 3.18.2 to 3.18.5 to fix known security vulnerabilities in bundled packages.
-  - sink: `FROM alpine:3.18.2`
-  - fix: Upgrade base image to patched version
-  - commit: https://github.com/projectdiscovery/nuclei/commit/d93b4a01df9c218a36dd2fccfb1c2c1f96b181fe (2024-03-15)
-- **Dependency Vulnerability** (high) [CWE-1035]: Updates go-git to fix known vulnerabilities (CVE-2023-49569, CVE-2023-49568), which could be exploited when processing untrusted git data.
-  - sink: `go-git library usage`
-  - fix: dependency upgrade to patched version
-  - commit: https://github.com/projectdiscovery/nuclei/commit/1700cdf4e23128a358066364062246b58c1c013f (2024-03-14)
-- **Code Execution via Unsigned Template** (high) [CWE-347]: Introduces a flag to disable unsigned templates, preventing execution of unverified templates that could lead to arbitrary code execution.
-  - sink: `loading unsigned templates when -dut flag is set`
-  - fix: Adds -disable-unsigned-templates flag to skip unsigned templates and report them separately
-  - commit: https://github.com/projectdiscovery/nuclei/commit/fa56800fcca3c8c8fe8e8dec28584210ca1527de (2024-03-14)
-- **Auth Bypass** (medium) [CWE-347]: This commit introduces a flag to skip unsigned templates, closing a path where an attacker could execute untrusted templates by bypassing signature verification.
-  - sink: `parsed.Verified`
-  - fix: skip unverified templates when DisableUnsignedTemplates is set
-  - commit: https://github.com/projectdiscovery/nuclei/commit/9bd4db3f74aedd067af568df4f9198a8a090f68b (2024-03-10)
-- **Code Execution** (high) [CWE-94]: Prevents execution of unverified code templates in workflows, mitigating code execution risk.
-  - sink: `template.RequestsCode`
-  - fix: require -code flag and verified templates for code templates in workflows
-  - commit: https://github.com/projectdiscovery/nuclei/commit/e86f38299765b82ad724fdb701557e0eaff3884d (2024-03-01)
-- **SSRF** (high) [CWE-918]: Fixes SSRF by validating the actual LDAP controller host against the network policy instead of the domain name.
-  - sink: `newLdapSession`
-  - fix: host allowlist check on controller instead of domain
-  - commit: https://github.com/projectdiscovery/nuclei/commit/a167e6c57b80f5dc7be231231b62d7c594ee2770 (2024-01-20)
-- **Sandbox Escape** (high) [CWE-94]: Disables eval and tightens file read validation in the JavaScript sandbox, preventing template-driven code execution and file access outside the sandbox.
-  - sink: `eval function and file read in sandbox mode`
-  - fix: Disable eval function and rewrite file validation in sandbox mode
-  - commit: https://github.com/projectdiscovery/nuclei/commit/c35162c8ef061a96dd74b54c8f022c1a41cf5833 (2023-10-13)
-- **Path Traversal** (high) [CWE-22]: Adds network policy and blocks file:// protocol in headless mode unless -lfa is set, preventing local file disclosure.
-  - sink: `file:// protocol access in headless browser`
-  - fix: Network policy validation and file:// scheme blocking when local file access is disabled
-  - commit: https://github.com/projectdiscovery/nuclei/commit/d3928e080d819f25306bfbaecc399f03516db5aa (2023-08-25)
-- **Deserialization of Untrusted Data** (medium) [CWE-502]: Fixes JSON deserialization by adding explicit UnmarshalJSON methods and correcting tags, preventing type confusion and potential logic bypasses.
-  - sink: `UnmarshalJSON on user-controlled fields like Type and Name`
-  - fix: Adds proper UnmarshalJSON implementations and corrects json tags to prevent type confusion
-  - commit: https://github.com/projectdiscovery/nuclei/commit/85090b75317977bb9301444c9dfd8e4c2dba667a (2023-02-07)
-- **Sandbox Escape** (medium) [CWE-22]: The commit adds sandbox mode that restricts payload file paths to the template directory, preventing arbitrary file reads when sandbox is enabled.
-  - sink: `loadPayloadsFromFile`
-  - fix: path prefix validation against template directory and template path directory
-  - commit: https://github.com/projectdiscovery/nuclei/commit/e7fb40a41308f6c41267e858d1781eb67f7e0572 (2022-11-16)
-- **Insecure File Permissions** (low) [CWE-732]: Fixes overly permissive file permissions on debug output files by changing from os.ModePerm (0777) to 0644.
-  - sink: `os.OpenFile`
-  - fix: restrict file mode to 0644
-  - commit: https://github.com/projectdiscovery/nuclei/commit/af4854f90de728bb08689d5cae3dec4acfdb6b9a (2022-06-20)
-- **Security Misconfiguration** (medium) [CWE-16]: Prevents setting HTTP_PROXY environment variable unless explicitly enabled, avoiding potential proxy hijacking.
-  - sink: `os.Setenv(types.HTTP_PROXY_ENV, proxyURL.String())`
-  - fix: Added check for options.ProxyInternal before setting HTTP_PROXY environment variable
-  - commit: https://github.com/projectdiscovery/nuclei/commit/3f5c055aca1499642caf4f40e8cf55ad32bd4da9 (2022-03-09)
-- **Sandbox Escape** (high) [CWE-250]: The commit restricts the use of the --no-sandbox flag to Linux root users, preventing sandbox escape vulnerabilities in other environments.
-  - sink: `NoSandbox(true)`
-  - fix: Conditionally disable sandbox only on Linux as root
-  - commit: https://github.com/projectdiscovery/nuclei/commit/df55f7a2eb591a7949b5c66a9c54db54aa28f450 (2021-12-05)
-- **Path Traversal** (high) [CWE-22]: This commit fixes a zip-slip path traversal vulnerability by validating the extracted file path against the configured template directory.
-  - sink: `os.OpenFile`
-  - fix: path canonicalization and prefix check
-  - commit: https://github.com/projectdiscovery/nuclei/commit/f9c214a66f37a00fc29fe64232b690e6285ec59c (2021-11-25)
-- **SSRF** (high) [CWE-918]: Adds validation and connectivity checks for SOCKS5 proxies before launching a scan, preventing SSRF through proxy configuration.
-  - sink: `Proxy connection validation and loading`
-  - fix: Proxy server validation and connectivity check before scan launch
-  - commit: https://github.com/projectdiscovery/nuclei/commit/0e46d3e0410f89e4af41099f5cd38fa7f54ae54f (2021-11-10)
-- **Sandbox Escape** (high) [CWE-284]: This commit adds a sandbox flag that, when enabled, restricts the tengo scripting engine to safe modules, preventing sandbox escape and arbitrary command execution.
-  - sink: `stdlib.GetModuleMap(stdlib.AllModuleNames()...)`
-  - fix: Added sandbox flag to restrict imports to safe modules when sandbox is enabled
-  - commit: https://github.com/projectdiscovery/nuclei/commit/cf2a98b03d76327c20fb0183b7f1c849eb692c91 (2020-12-13)
-- **Sandbox Escape** (high) [CWE-284]: This commit restricts the tengo scripting engine to a safe module list, preventing sandbox escape and arbitrary command execution via workflow templates.
-  - sink: `stdlib.GetModuleMap(stdlib.AllModuleNames()...)`
-  - fix: Restricted tengo script imports to a safe module list
-  - commit: https://github.com/projectdiscovery/nuclei/commit/96049a32fa70aef0483d4180d386a57b3c2d973a (2020-12-13)
+**By component** (directories with the most fixes):
+
+- `pkg/protocols/common/protocolstate` — 2 fixes: SSRF (2)
+
+- [1c0f77d28e][c-1c0f77d28e] **fix(js/net): enforce network policy on proxied target in JS net dial** — SSRF, high [CWE-918] · inferred · 2026-10-18
+  `pkg/js/libs/net/net.go` · sink `dialHTTPProxy` · fix: input validation
+- [d43c1f547f][c-d43c1f547f] **fix(headless): canonicalize IPv4-mapped IPv6 literals and re-validate dial targets** — SSRF, high [CWE-918] · inferred · 2026-10-13
+  `pkg/protocols/common/protocolstate/headless.go` · sink `net.DialTimeout` · fix: canonicalization before validation
+- [d9dabbb4a1][c-d9dabbb4a1] **fix(headless): enforce network policy on ws and wss schemes (#7867)** — SSRF, high [CWE-918] · inferred · 2026-10-12
+  `pkg/protocols/common/protocolstate/headless.go` · sink `isValidHost` · fix: canonicalization before validation
+- [a20aa26024][c-a20aa26024] **Fix pprof/auth exposure on http-api-endpoint and harden file permissions** — Information Disclosure, high [CWE-200] · inferred · 2026-08-03
+  `internal/httpapi/apiendpoint.go` · sink `http.ListenAndServe` · fix: authentication verification
+- [fac0b7196d][c-fac0b7196d] **feat(templates): require signatures for javascript templates (#7514)** — Signature Verification Bypass, high [CWE-347] · verified (CVE-2026-76819, GHSA-vxg7-f2jj-jmqm) · 2026-07-03
+  `pkg/catalog/loader/loader.go` · sink `t.HasJavascriptRequest` · fix: cryptographic verification
+- [ccbfb12bd0][c-ccbfb12bd0] **refactor(protocols): template rendering boundary (#7499)** — Code Execution, high [CWE-94] · verified (CVE-2026-76805, GHSA-jpvm-9frm-hjcq) · 2026-06-30
+  `pkg/fuzz/execute.go` · sink `expressions.Evaluate` · fix: sandbox isolation
+- [3a8b65cb45][c-3a8b65cb45] **fix(js): reject unsandboxed ccache writes in krbforge (#7482)** — Sandbox Escape, high [CWE-693] · inferred · 2026-06-24
+  `pkg/js/libs/krbforge/krbforge.go` · sink `OutputFile` · fix: sandbox isolation
+- [68daef44a7][c-68daef44a7] **fix(js): escape dbname in lib/pq URL (#7479)** — SQL Injection, high [CWE-89] · inferred · 2026-06-23
+  `pkg/js/libs/postgres/postgres.go` · sink `fmt.Sprintf` (library call — the operation around it is the sink) · fix: context-aware output escaping
+- [634cecbb28][c-634cecbb28] **gate mysql allowAllFiles behind -lfa** — Arbitrary File Access, high [CWE-22] · verified (CVE-2026-76803, GHSA-xhmx-w2j4-rw3q) · 2026-06-19
+  `pkg/js/libs/mysql/mysql.go` · sink `sql.Open` · fix: sandbox isolation
+- [20cbf0bd73][c-20cbf0bd73] **refactor: use path-aware filesystem containment checks (#7420)** — Path Traversal, high [CWE-22] · inferred · 2026-06-04
+  `pkg/installer/template.go` · sink `filepath.Join` · fix: canonicalization before validation
+- [6f2ade6a9b][c-6f2ade6a9b] **fix(js): respect \`allow-local-file-access\` in \`require\` (#7332)** — Sandbox Escape, high [CWE-693] · inferred · 2026-04-10
+  `pkg/js/compiler/pool.go` · sink `require.DefaultSourceLoader` · fix: sandbox isolation
+- [d221732016][c-d221732016] **fix(expressions): only eval template-authored expressions (#7221)** — Code Execution, high [CWE-94] · verified (CVE-2026-76805, GHSA-jpvm-9frm-hjcq) · 2026-03-16
+  `pkg/protocols/common/expressions/expressions.go` · sink `FindExpressions` · fix: allowlist enforcement
+- [0da993afe6][c-0da993afe6] **Merge commit from fork** — Signature Verification Bypass, medium [CWE-347] · verified (CVE-2024-43405, GHSA-7h5p-mmpp-hgmm) · 2024-08-19
+  `pkg/templates/signer/tmpl_signer.go` · sink `os.ReadFile` · fix: cryptographic verification
+- [e5154d362a][c-e5154d362a] **fixing payload load (#3927)** — Arbitrary File Access, high [CWE-22] · verified (CVE-2023-37896, GHSA-2xx4-jj5v-6mff) · 2023-07-14
+  `v2/pkg/protocols/common/generators/load.go` · sink `loadPayloads` · fix: sandbox isolation
+
+## Historical Vulnerability Classes
+
+| Class | Fixes | Verified | Most recent fix |
+|---|---|---|---|
+| SSRF | 3 | 0 | [2026-10-18][c-1c0f77d28e] 1c0f77d28e |
+| Arbitrary File Access | 1 (1 component gone, not counted) | 2 | [2026-06-19][c-634cecbb28] 634cecbb28 |
+| Code Execution | 2 | 2 | [2026-06-30][c-ccbfb12bd0] ccbfb12bd0 |
+| Sandbox Escape | 2 | 0 | [2026-06-24][c-3a8b65cb45] 3a8b65cb45 |
+| Signature Verification Bypass | 2 | 2 | [2026-07-03][c-fac0b7196d] fac0b7196d |
+| Information Disclosure | 1 | 0 | [2026-08-03][c-a20aa26024] a20aa26024 |
+| Path Traversal | 1 | 0 | [2026-06-04][c-20cbf0bd73] 20cbf0bd73 |
+| SQL Injection | 1 | 0 | [2026-06-23][c-68daef44a7] 68daef44a7 |
+
+## Known Advisories
+
+### Linked to a fix commit in this history (5)
+
+- **CVE-2026-76805** / GHSA-jpvm-9frm-hjcq (medium 5.3) CWE-200 — fixed in [ccbfb12bd0][c-ccbfb12bd0], [d221732016][c-d221732016]
+  - Environment Variable Disclosure via Response-Derived Data in DAST/Fuzz Mode. A vulnerability in Nuclei's DAST/fuzz expression evaluation path allows a malicious target server to trigger disclosure of ...
+- **CVE-2026-76803** / GHSA-xhmx-w2j4-rw3q (medium 5.3) CWE-284 — fixed in [634cecbb28][c-634cecbb28]
+  - Local File Read via MySQL Client Sandbox Bypass. A vulnerability in Nuclei's JavaScript MySQL client library allows arbitrary local file reads that bypass the `-allow-local-file-access` (`-lfa`) sandb...
+- **CVE-2026-76819** / GHSA-vxg7-f2jj-jmqm (high 8.6) CWE-94,CWE-787 — fixed in [fac0b7196d][c-fac0b7196d]
+  - Arbitrary Code Execution via Goja JavaScript Engine Vulnerability. A vulnerability in the Goja JavaScript engine used by Nuclei's `javascript:` protocol allows arbitrary native code execution on the s...
+- **CVE-2024-43405** / GHSA-7h5p-mmpp-hgmm (high 7.4) CWE-78 — fixed in [0da993afe6][c-0da993afe6]
+  - Nuclei Template Signature Verification Bypass. ## Summary A vulnerability has been identified in Nuclei's template signature verification system that could allow an attacker to bypass the signature ch...
+- **CVE-2023-37896** / GHSA-2xx4-jj5v-6mff (medium 5.9) CWE-22 — fixed in [e5154d362a][c-e5154d362a]
+  - Nuclei < 2.9.9 - Sandbox restriction bypass. ## Overview Affected: < 2.9.9 (patched 2.9.9).
+
+### Without a linked fix commit (6)
+
+Published elsewhere, fixed before the history window, or not yet linked; full descriptions are in the JSON context.
+
+| Advisory | Severity | Published | CWE | Summary |
+|---|---|---|---|---|
+| CVE-2026-76804 / GHSA-qgw5-7j4f-fg97 | medium 5.5 | 2026-07-16 | CWE-284 | Local File Read via Workflow File-Protocol Gate Bypass. A vulnerability in Nuclei's workfl... |
+| CVE-2026-76802 / GHSA-jpf4-98qj-qr67 | medium 4.7 | 2026-07-16 | CWE-78 | Arbitrary Command Execution via DAST Code Signature Bypass. A vulnerability in Nuclei's DA... |
+| CVE-2026-41645 / GHSA-jm34-66cf-qpvr | medium 5.3 | 2026-04-18 | CWE-94 | Environment variable disclosure via Response-Derived DSL Expressions. A vulnerability in N... |
+| CVE-2026-41646 / GHSA-29rg-wmcw-hpf4 | medium 5.5 | 2026-04-18 | CWE-284 | Local File Read via require() Module Loader Bypass. A vulnerability in Nuclei's JavaScript... |
+| CVE-2024-40641 / GHSA-c3q9-c27p-cw9h | high 7.4 | 2024-07-17 | CWE-78 | Unsigned code template execution through workflows in projectdiscovery/nuclei. ### Summary... |
+| CVE-2024-27920 / GHSA-w5wx-6g2r-r78q | high 7.4 | 2024-03-15 | CWE-78 | Unsigned code template execution through workflows in projectdiscovery/nuclei. ### Overvie... |
+
+[c-df57ae6d29]: https://github.com/projectdiscovery/nuclei/commit/df57ae6d2955864fd92a1bd2565fe9e4e9a7592b
+[c-d43c1f547f]: https://github.com/projectdiscovery/nuclei/commit/d43c1f547f98ac56654f4335ad20af0cca4c229c
+[c-d9dabbb4a1]: https://github.com/projectdiscovery/nuclei/commit/d9dabbb4a19e897a6bc12334a030a43035b27b73
+[c-e5154d362a]: https://github.com/projectdiscovery/nuclei/commit/e5154d362af900e83b30fb79028bfab1a7f0c648
+[c-634cecbb28]: https://github.com/projectdiscovery/nuclei/commit/634cecbb289db494247c04a9c4ec6a113aef3cc0
+[c-ccbfb12bd0]: https://github.com/projectdiscovery/nuclei/commit/ccbfb12bd01447ba25f6e755dfb2bee677736da6
+[c-d221732016]: https://github.com/projectdiscovery/nuclei/commit/d2217320162d5782ca7cb95bef9dda17063818f3
+[c-a20aa26024]: https://github.com/projectdiscovery/nuclei/commit/a20aa260245e5db889c5a421d4a65aeb2a64bf46
+[c-1c0f77d28e]: https://github.com/projectdiscovery/nuclei/commit/1c0f77d28ed639a34c2036fa23c7503ffac0bf27
+[c-fac0b7196d]: https://github.com/projectdiscovery/nuclei/commit/fac0b7196d2fbca56c8af8ff8e1899c111041a37
+[c-3a8b65cb45]: https://github.com/projectdiscovery/nuclei/commit/3a8b65cb45f121fcdf418f60e5177ed9f250c122
+[c-68daef44a7]: https://github.com/projectdiscovery/nuclei/commit/68daef44a7eafd41c39690924620e418762909ef
+[c-20cbf0bd73]: https://github.com/projectdiscovery/nuclei/commit/20cbf0bd73893c1270582c9778b6b26a5cb11206
+[c-6f2ade6a9b]: https://github.com/projectdiscovery/nuclei/commit/6f2ade6a9b427c284c15a43445f9c7f055e60e5d
+[c-0da993afe6]: https://github.com/projectdiscovery/nuclei/commit/0da993afe6d41b4b1b814e8fad23a2acba13c60a
 
 ---
-_Fixed-issue fingerprints and CVE matches are derived from real data; LLM-written prose is advisory and should be verified against the cited commits._
+_Fixed-issue fingerprints and advisory matches are derived from git history and published advisories; the risk tables and pattern descriptions above are derived from them and should be verified against the cited commits._

SYNTAX-REFERENCE.md

@@ -530,6 +530,16 @@ Description of the template.
 
 You can go in-depth here on what the template actually does.
 
+Description, impact and remediation may hold {{placeholders}} filled for
+each finding: named extractors, payloads, variables and DSL helpers
+(network helpers excluded), and the result fields host, port, ip, url,
+scheme, path, matched-at, matcher-name and extractor-name, which win over
+values of the same name. Use join(", ", name) for an extractor with
+several values, and brackets for a hyphenated name inside a helper:
+to_upper([matcher-name]). Unknown placeholders stay as written, and so
+does a field whose expression fails. Global matcher templates match on
+other templates' responses, so only the result fields fill their text.
+
 
 
 Examples:
@@ -543,6 +553,10 @@ description: Bower is a package manager which stores package information in the
 description: Subversion ALM for the enterprise before 8.8.2 allows reflected XSS at multiple locations
 ```
 
+```yaml
+description: Redis {{version}} on {{host}}:{{port}} accepts the default password
+```
+
 
 </div>
 
@@ -559,6 +573,8 @@ Impact of the template.
 
 You can go in-depth here on impact of the template.
 
+Supports the same placeholders as description.
+
 
 
 Examples:
@@ -665,6 +681,8 @@ Remediation steps for the template.
 
 You can go in-depth here on how to mitigate the problem found by this template.
 
+Supports the same placeholders as description.
+
 
 
 Examples:
@@ -3757,6 +3775,8 @@ Part Definitions:
 - <code>req</code> - Headless request made from the client
 - <code>duration</code> - Latest measured operation duration in seconds
 - <code>resp,body,data</code> - Headless response received from client (default)
+- <code>header,all_headers</code> - Headless response headers
+- <code>status_code</code> - Status code received from the server
 
 <hr />
 

_typos.toml

@@ -20,6 +20,8 @@ extend-exclude = [
     "vendor/",
     # Go checksum file
     "go.sum",
+    # Generated history. Commit SHAs are not words.
+    "SECURITY_CONTEXT.md",
 ]
 
 [default.extend-identifiers]

cmd/nuclei/issue-tracker-config.yaml

@@ -57,7 +57,8 @@
 #    severity: low
 #  # duplicate-issue-check (optional) flag to enable duplicate tracking issue check
 #  duplicate-issue-check: false
-#  # duplicate-issue-page-size (optional) controls how many issues to fetch per page when searching for duplicates
+#  # duplicate-issue-page-size (optional) sets the number of issues to request per page when searching for duplicates
+#  # (GitLab returns at most 100 issues per page)
 #  duplicate-issue-page-size: 100
 #  # duplicate-issue-max-pages (optional) limits how many pages to fetch when searching for duplicates (0 = no limit)
 #  duplicate-issue-max-pages: 0

cmd/nuclei/main.go

@@ -271,10 +271,10 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.StringSliceVarP(&options.Targets, "target", "u", nil, "target URLs/hosts to scan", goflags.CommaSeparatedStringSliceOptions),
 		flagSet.StringVarP(&options.TargetsFilePath, "list", "l", "", "path to file containing a list of target URLs/hosts to scan (one per line)"),
 		flagSet.StringVarP(&options.InlineTargetsList, "targets-inline", "", "", "inline multiline target list (for use in template profiles)"),
-		flagSet.StringSliceVarP(&options.ExcludeTargets, "exclude-hosts", "eh", nil, "hosts to exclude to scan from the input list (ip, cidr, hostname)", goflags.FileCommaSeparatedStringSliceOptions),
+		flagSet.StringSliceVarP(&options.ExcludeTargets, "exclude-hosts", "eh", nil, "hosts to exclude to scan from the input list (ip, cidr, hostname)", goflags.FileNormalizedOriginalStringSliceOptions),
 		flagSet.StringVar(&options.Resume, "resume", "", "resume scan from and save to specified file (clustering will be disabled)"),
 		flagSet.BoolVarP(&options.ScanAllIPs, "scan-all-ips", "sa", false, "scan all the IP's associated with dns record"),
-		flagSet.StringSliceVarP(&options.IPVersion, "ip-version", "iv", nil, "IP version to scan of hostname (4,6) - (default 4)", goflags.CommaSeparatedStringSliceOptions),
+		flagSet.StringSliceVarP(&options.IPVersion, "ip-version", "iv", nil, "IP version to scan of hostname (4,6); default uses dialer fallback, -sa scans all addresses", goflags.CommaSeparatedStringSliceOptions),
 		flagSet.BoolVarP(&options.DisableURLNormalization, "disable-url-normalization", "dun", false, "keep targets exactly as supplied instead of collapsing urls that address the same resource"),
 		flagSet.IntVarP(&options.MaxURLsPerPattern, "max-urls-per-pattern", "mup", 0, "scan at most this many urls sharing one structural shape, e.g. /user/1 and /user/2 (0 to scan all)"),
 	)
@@ -309,7 +309,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.BoolVarP(&options.EnableGlobalMatchersTemplates, "enable-global-matchers", "egm", false, "enable loading global matchers templates"),
 		flagSet.BoolVar(&options.EnableFileTemplates, "file", false, "enable loading file templates"),
 		flagSet.BoolVar(&options.EnableLLM, "llm", false, "enable llm matchers and extractors (semantic matching)"),
-		flagSet.StringVar(&options.LLMProvider, "llm-provider", "openai", "llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter)"),
+		flagSet.StringVar(&options.LLMProvider, "llm-provider", "openai", "llm provider for semantic matching (openai, ollama, llamacpp, vllm, lmstudio, groq, openrouter, together, orcarouter, jev)"),
 		flagSet.StringVar(&options.LLMBaseURL, "llm-base-url", "", "openai-compatible endpoint for semantic matching (overrides -llm-provider, e.g. a local model)"),
 		flagSet.StringVar(&options.LLMModel, "llm-model", "", "model used for semantic matching"),
 		flagSet.IntVar(&options.LLMTimeout, "llm-timeout", 30, "time in seconds to wait for a single llm call"),
@@ -355,6 +355,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.StringVarP(&options.JSONLExport, "jsonl-export", "jle", "", "file to export results in JSONL(ine) format"),
 		flagSet.StringVarP(&options.PDFExport, "pdf-export", "pe", "", "file to export results in PDF format"),
 		flagSet.StringSliceVarP(&options.Redact, "redact", "rd", nil, "redact given list of keys from query parameter, request header and body", goflags.CommaSeparatedStringSliceOptions),
+		flagSet.BoolVarP(&options.NoRedact, "no-redact", "nrd", false, "disable the built-in redaction of credential headers (Authorization, Cookie, Set-Cookie, ...) in output; keys given with -redact are still redacted"),
 	)
 
 	flagSet.CreateGroup("configs", "Configurations",
@@ -466,13 +467,11 @@ on extensive configurability, massive extensibility and ease of use.`)
 			scanstrategy.TemplateSpray.String(): goflags.EnumVariable(2),
 		}),
 		flagSet.DurationVarP(&options.InputReadTimeout, "input-read-timeout", "irt", time.Duration(3*time.Minute), "timeout on input read"),
-		flagSet.BoolVar(&options.ResponseCache, "response-cache", false, "reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)"),
 		flagSet.BoolVarP(&options.DisableHTTPProbe, "no-httpx", "nh", false, "disable httpx probing for non-url input"),
-		flagSet.BoolVarP(&options.StrictProbe, "strict-probe", "stp", false, "skip templates whose target service is unreachable: HTTP/headless on hosts httpx could not confirm and network templates on closed ports (lossless, no raw-input fallback)"),
-		flagSet.BoolVarP(&options.TechFilter, "tech-filter", "tf", false, "opt-in: fingerprint HTTP targets and skip product/macro-tagged templates that cannot match (off by default; enables scan-scoped GET cache; fail-open; no-op if no tech-bound templates)"),
 		flagSet.BoolVar(&options.PreflightPortScan, "preflight-portscan", false, "run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)"),
 		flagSet.BoolVar(&options.DisableStdin, "no-stdin", false, "disable stdin processing"),
 		flagSet.BoolVar(&options.EnableHTTPCache, "http-cache", false, "enable HTTP cache (RFC 9111) for HTTP requests"),
+		flagSet.BoolVarP(&options.AutoCalibrate, "auto-calibrate", "ac", false, "drop HTTP matches on a host's catch-all page (answers missing paths with 2xx/3xx)"),
 	)
 
 	flagSet.CreateGroup("headless", "Headless",

go.mod

@@ -18,10 +18,11 @@ require (
 	github.com/alecthomas/chroma v0.10.0
 	github.com/alexsnet/go-vnc v0.1.0
 	github.com/alitto/pond v1.9.2
+	github.com/andybalholm/brotli v1.2.0
 	github.com/andygrunwald/go-jira v1.16.1
-	github.com/antchfx/htmlquery v1.3.5
+	github.com/antchfx/htmlquery v1.3.7
 	github.com/antchfx/xmlquery v1.4.4
-	github.com/antchfx/xpath v1.3.6
+	github.com/antchfx/xpath v1.3.9
 	github.com/aws/aws-sdk-go-v2 v1.41.5
 	github.com/aws/aws-sdk-go-v2/config v1.31.16
 	github.com/aws/aws-sdk-go-v2/credentials v1.18.20
@@ -33,10 +34,11 @@ require (
 	github.com/clbanning/mxj/v2 v2.7.0
 	github.com/ditashi/jsbeautifier-go v0.0.0-20141206144643-2520a8026a9c
 	github.com/docker/go-units v0.5.0
+	github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707
 	github.com/elazarl/goproxy v1.7.2
 	github.com/fatih/structs v1.1.0
 	github.com/fullstorydev/grpcurl v1.9.3
-	github.com/getkin/kin-openapi v0.144.0
+	github.com/getkin/kin-openapi v0.149.0
 	github.com/getsops/sops/v3 v3.9.2
 	github.com/go-echarts/go-echarts/v2 v2.6.0
 	github.com/go-git/go-git/v5 v5.19.2
@@ -61,6 +63,7 @@ require (
 	github.com/jhump/protoreflect v1.17.0
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/kitabisa/go-ci v1.0.3
+	github.com/klauspost/compress v1.18.7
 	github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa
 	github.com/lib/pq v1.12.3
 	github.com/logrusorgru/aurora/v4 v4.0.0
@@ -75,7 +78,7 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/praetorian-inc/fingerprintx v1.1.15
 	github.com/projectdiscovery/clistats v0.1.7
-	github.com/projectdiscovery/dsl v0.8.24
+	github.com/projectdiscovery/dsl v0.9.0
 	github.com/projectdiscovery/fastdialer v0.6.0
 	github.com/projectdiscovery/fasttemplate v0.0.2
 	github.com/projectdiscovery/gcache v0.0.0-20241015120333-12546c6e3f4c
@@ -85,7 +88,7 @@ require (
 	github.com/projectdiscovery/goja_nodejs v0.0.0-20260618132410-8519f75f703d
 	github.com/projectdiscovery/gologger v1.1.73
 	github.com/projectdiscovery/gostruct v0.0.2
-	github.com/projectdiscovery/govaluate v0.0.0-20260615100919-5ee2581bbf7e
+	github.com/projectdiscovery/govaluate v0.0.0-20261102102118-9294133b15a2
 	github.com/projectdiscovery/gozero v0.1.1-0.20260530071156-fa1dad563d76
 	github.com/projectdiscovery/hmap v0.0.102
 	github.com/projectdiscovery/httpx v1.12.0
@@ -102,8 +105,8 @@ require (
 	github.com/projectdiscovery/tlsx v1.4.3-0.20261020052910-acb80c2c7c75
 	github.com/projectdiscovery/uncover v1.2.1
 	github.com/projectdiscovery/useragent v0.0.109
-	github.com/projectdiscovery/utils v0.11.12-0.20261025172519-64b3c221cfe5
-	github.com/projectdiscovery/wappalyzergo v0.3.5
+	github.com/projectdiscovery/utils v0.11.13-0.20261103094816-5c0b20e35350
+	github.com/projectdiscovery/wappalyzergo v0.3.6
 	github.com/projectdiscovery/yamldoc-go v1.0.7
 	github.com/redis/go-redis/v9 v9.19.0
 	github.com/remeh/sizedwaitgroup v1.0.0
@@ -113,6 +116,7 @@ require (
 	github.com/segmentio/ksuid v1.0.4
 	github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466
 	github.com/sijms/go-ora/v2 v2.9.0
+	github.com/spaolacci/murmur3 v1.1.0
 	github.com/spf13/cast v1.10.0
 	github.com/stretchr/testify v1.12.1
 	github.com/syndtr/goleveldb v1.0.0
@@ -175,7 +179,6 @@ require (
 	github.com/alecthomas/kingpin/v2 v2.4.0 // indirect
 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
-	github.com/andybalholm/brotli v1.2.0 // indirect
 	github.com/andybalholm/cascadia v1.3.4 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect
@@ -239,7 +242,6 @@ require (
 	github.com/docker/cli v29.2.0+incompatible // indirect
 	github.com/docker/go-connections v0.7.0 // indirect
 	github.com/dop251/goja v0.0.0-20260311135729-065cd970411c // indirect
-	github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/ebitengine/purego v0.10.0 // indirect
 	github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 // indirect
@@ -323,7 +325,6 @@ require (
 	github.com/kaiakz/ubuffer v0.0.0-20200803053910-dd1083087166 // indirect
 	github.com/kataras/jwt v0.1.10 // indirect
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
-	github.com/klauspost/compress v1.18.7 // indirect
 	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
 	github.com/klauspost/pgzip v1.2.6 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
@@ -385,14 +386,13 @@ require (
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
 	github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
-	github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
+	github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 // indirect
 	github.com/sergi/go-diff v1.4.0 // indirect
 	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
 	github.com/shirou/gopsutil/v4 v4.26.3 // indirect
 	github.com/sirupsen/logrus v1.10.2 // indirect
 	github.com/skeema/knownhosts v1.3.1 // indirect
 	github.com/sorairolake/lzip-go v0.3.8 // indirect
-	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/spf13/afero v1.15.0 // indirect
 	github.com/spf13/pflag v1.0.10 // indirect
 	github.com/spiffe/go-spiffe/v2 v2.7.0 // indirect

go.sum

@@ -180,14 +180,13 @@ github.com/andygrunwald/go-jira v1.16.1 h1:WoQEar5XoDRAibOgKzTFELlPNlKAtnfWr296R
 github.com/andygrunwald/go-jira v1.16.1/go.mod h1:UQH4IBVxIYWbgagc0LF/k9FRs9xjIiQ8hIcC6HfLwFU=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
-github.com/antchfx/htmlquery v1.3.5 h1:aYthDDClnG2a2xePf6tys/UyyM/kRcsFRm+ifhFKoU0=
-github.com/antchfx/htmlquery v1.3.5/go.mod h1:5oyIPIa3ovYGtLqMPNjBF2Uf25NPCKsMjCnQ8lvjaoA=
+github.com/antchfx/htmlquery v1.3.7 h1:KCUklE1EO/xm6iS3CWx6iatFlcnENBk4I4UYa2pBmLU=
+github.com/antchfx/htmlquery v1.3.7/go.mod h1:/uBTMnFEl6LaBlIgw3KZ+c061tAQdlfbCXObZoxwQf4=
 github.com/antchfx/xmlquery v1.4.4 h1:mxMEkdYP3pjKSftxss4nUHfjBhnMk4imGoR96FRY2dg=
 github.com/antchfx/xmlquery v1.4.4/go.mod h1:AEPEEPYE9GnA2mj5Ur2L5Q5/2PycJ0N9Fusrx9b12fc=
 github.com/antchfx/xpath v1.3.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
-github.com/antchfx/xpath v1.3.5/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
-github.com/antchfx/xpath v1.3.6 h1:s0y+ElRRtTQdfHP609qFu0+c6bglDv20pqOViQjjdPI=
-github.com/antchfx/xpath v1.3.6/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
+github.com/antchfx/xpath v1.3.9 h1:muXJSod6jRzhsiUjvduNSxsRSptFZ2GOJ1oMD1QkPSA=
+github.com/antchfx/xpath v1.3.9/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
@@ -427,8 +426,8 @@ github.com/gaissmai/bart v0.30.0 h1:K1oTSQ6Nf76bhrgxeTqzZa4hE0M9x4xDfL6n6UG67Sc=
 github.com/gaissmai/bart v0.30.0/go.mod h1:GREWQfTLRWz/c5FTOsIw+KkscuFkIV5t8Rp7Nd1Td5c=
 github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64=
 github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
-github.com/getkin/kin-openapi v0.144.0 h1:hIRcTH+KjLfkLpYU6bSSfdFpi0fZi1fp+hSPi4aQu9Y=
-github.com/getkin/kin-openapi v0.144.0/go.mod h1:3BH9M9XDe/y9M5DSvEocVYAYq1w0qrhJHjC/vZi0AaY=
+github.com/getkin/kin-openapi v0.149.0 h1:ZbhmVJ4yq5RZDUsyP8lcBcGMsjsaTqXEFt6isdtMDfA=
+github.com/getkin/kin-openapi v0.149.0/go.mod h1:1+BHDzstro+P5CKtPy1X4PfofnFgmRe6uvMy9+r9fKY=
 github.com/getsops/gopgagent v0.0.0-20240527072608-0c14999532fe h1:QKe/kmAYbndxwu91TcjHERsnMh5SgOB1x/qicvOdUJ8=
 github.com/getsops/gopgagent v0.0.0-20240527072608-0c14999532fe/go.mod h1:awFzISqLJoZLm+i9QQ4SgMNHDqljH6jWV0B36V5MrUM=
 github.com/getsops/sops/v3 v3.9.2 h1:Cuuahc/UGu7W3+NXd+mrl+2oXGxxj/jIPsw6Ah2CbxA=
@@ -935,8 +934,8 @@ github.com/projectdiscovery/cdncheck v1.3.2 h1:I453ZWNBeG0WWIy3wi9412l4PTzorLgeb
 github.com/projectdiscovery/cdncheck v1.3.2/go.mod h1:5IkUMMlcGaeg1DWW5oJVd3r9X3EpJ88j5HxBeF4Me9M=
 github.com/projectdiscovery/clistats v0.1.7 h1:NXf4lUzx1QcbJd6QycNwwYHTv2FsW9MchQ2LsgSL6C0=
 github.com/projectdiscovery/clistats v0.1.7/go.mod h1:RxRGxhMzkT344VTTvZWIrojurUXdQQRN1yfwgg/88I8=
-github.com/projectdiscovery/dsl v0.8.24 h1:rN2w6LSIjA16qKGRjyj70oxdwXGaspe6sAjEkP7F2fk=
-github.com/projectdiscovery/dsl v0.8.24/go.mod h1:V+EludvS5PS19UNMlM3CuBwZ+sFnMaQhCFGxysDUanQ=
+github.com/projectdiscovery/dsl v0.9.0 h1:i1KyCbKJyKfHkgdbfh0JkWdd14APUjpEYf+GaYANlCg=
+github.com/projectdiscovery/dsl v0.9.0/go.mod h1:ntNFkkiYCTEosGFTbApA61RMrvJTCjnakPxnWj56MyY=
 github.com/projectdiscovery/fastdialer v0.6.0 h1:4ZeFQ6LyRH4D8e3qO/yzqIP4Cf9Fr0DduaVpJD6ENnE=
 github.com/projectdiscovery/fastdialer v0.6.0/go.mod h1:FpHwa9S5PUgfPbh+irOm+cdU6AO631XidxkhPwPv7yY=
 github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
@@ -957,8 +956,8 @@ github.com/projectdiscovery/gologger v1.1.73 h1:WX2nzq29cL7HdvOHqIttacoid/5lMwap
 github.com/projectdiscovery/gologger v1.1.73/go.mod h1:mJwODZcFDg70ihINpOvZevmBtgvpP8H9/l8Y+OPhZPY=
 github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
 github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
-github.com/projectdiscovery/govaluate v0.0.0-20260615100919-5ee2581bbf7e h1:vxzgQlz2Cy/YvizYDQx9OhucBcmBotfDhbQ4yCY2vfA=
-github.com/projectdiscovery/govaluate v0.0.0-20260615100919-5ee2581bbf7e/go.mod h1:xH7bPwHxUlz1yx9UlVeTF+UVCUaKhTnZgaxHb5z362E=
+github.com/projectdiscovery/govaluate v0.0.0-20261102102118-9294133b15a2 h1:aSLbDvrJQPlp4HvojU+bTU5XV8xdAVCjoc5gX1Wfbw4=
+github.com/projectdiscovery/govaluate v0.0.0-20261102102118-9294133b15a2/go.mod h1:Z3VjpUJV1pnXDwvAddA8aZ4e839SoeftB7CSdZntAVI=
 github.com/projectdiscovery/gozero v0.1.1-0.20260530071156-fa1dad563d76 h1:AN70bbi6BBs7KpIM9w0LxygUN7uzT/oH+owDIQ+Fz/k=
 github.com/projectdiscovery/gozero v0.1.1-0.20260530071156-fa1dad563d76/go.mod h1:cWHYnRXoYWHtTpOYyAp5laGYX8GH8ITUhgQaP8G/8FA=
 github.com/projectdiscovery/hmap v0.0.102 h1:ybPuf7UQvuOLiHLOUbVMvd+PSlItQU5iQpP1Erl8kaU=
@@ -999,10 +998,10 @@ github.com/projectdiscovery/uncover v1.2.1 h1:8U46T/96CLT7BPoXBgkTvWqB06lOyeTSLv
 github.com/projectdiscovery/uncover v1.2.1/go.mod h1:0p8onrWxfpXQEYs90ZDzTSpu1107gWmodX1NWqu/+z4=
 github.com/projectdiscovery/useragent v0.0.109 h1:b86BPdZvmgbJousa04jAugDRY4Y+rAvEamE0TFCPmeE=
 github.com/projectdiscovery/useragent v0.0.109/go.mod h1:sRyBqDAcD31RKYrO8fyXGowLd36fNwY1g0MNW9mp7zk=
-github.com/projectdiscovery/utils v0.11.12-0.20261025172519-64b3c221cfe5 h1:AOspf7ScbwZr8A2YnjCFd4ZJAMSFqzfEaIJIbJPuu10=
-github.com/projectdiscovery/utils v0.11.12-0.20261025172519-64b3c221cfe5/go.mod h1:TJgsyVjB02CavwZVFzUzPg6YOSJXQxLhL808JCiQKlU=
-github.com/projectdiscovery/wappalyzergo v0.3.5 h1:UZradYF1GIcruSIV5wwo/Coph9/VnM17oajLD9eXR2U=
-github.com/projectdiscovery/wappalyzergo v0.3.5/go.mod h1:4E6+fh/ckaY3wo5tlM7lNYSkhuTYRHzu2YmGIRraXP0=
+github.com/projectdiscovery/utils v0.11.13-0.20261103094816-5c0b20e35350 h1:uXqpYa6XKV9dCvoti9PbSgIWIsBKPu5YKMywB9G4q0Q=
+github.com/projectdiscovery/utils v0.11.13-0.20261103094816-5c0b20e35350/go.mod h1:TJgsyVjB02CavwZVFzUzPg6YOSJXQxLhL808JCiQKlU=
+github.com/projectdiscovery/wappalyzergo v0.3.6 h1:1FXCMeCa1xVu2zaW+eeBcbVxtlMStqGOlKcMLyh2pzI=
+github.com/projectdiscovery/wappalyzergo v0.3.6/go.mod h1:4E6+fh/ckaY3wo5tlM7lNYSkhuTYRHzu2YmGIRraXP0=
 github.com/projectdiscovery/yamldoc-go v1.0.7 h1:Odmjd1o4EkMff2F6QHi1dIYPIQPZUFy24YqmQCprmTk=
 github.com/projectdiscovery/yamldoc-go v1.0.7/go.mod h1:nqD1iPMcofpRA1DbXavPByJhWURrshT+l8WN9XjyQag=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
@@ -1055,8 +1054,8 @@ github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7
 github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
 github.com/sandrolain/httpcache v1.4.0 h1:Jf4Vx62X2ybvNPSpPvI1kT3xvMdDG1AsApQjOQKO9E0=
 github.com/sandrolain/httpcache v1.4.0/go.mod h1:kHBuXveitSn39SNPBhdf/ybG272X706HJ2RJqOQ+Em0=
-github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
-github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
+github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28=
+github.com/santhosh-tekuri/jsonschema/v6 v6.0.3/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
 github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c=
 github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE=
 github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=

internal/runner/host_group.go

@@ -1,445 +0,0 @@
-package runner
-
-import (
-	"fmt"
-	"sort"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/projectdiscovery/httpx/common/httpx"
-	"github.com/projectdiscovery/nuclei/v3/pkg/input"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
-	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
-	"github.com/projectdiscovery/nuclei/v3/pkg/utils"
-	syncutil "github.com/projectdiscovery/utils/sync"
-)
-
-const hostGroupProbeWorkers = 100
-
-// hostGroup is a set of targets that share the same open-port / HTTP signature.
-// Used for savings estimates / logging; execution itself is a single spray pass
-// with a per-(template,target) filter derived from the same probe data.
-type hostGroup struct {
-	key          string
-	hosts        []*contextargs.MetaInput
-	openPorts    map[string]struct{}
-	httpOK       bool
-	explicitPort string // operator host:port; network dials this, not template defaults
-}
-
-type hostProbeResult struct {
-	input        *contextargs.MetaInput
-	openPorts    []string
-	httpOK       bool
-	explicitPort string
-}
-
-// hostReachabilityIndex maps each input to its probed reachability so the
-// engine can skip impossible template×target pairs in a single Execute pass.
-type hostReachabilityIndex struct {
-	byInput map[string]hostReachability
-}
-
-type hostReachability struct {
-	openPorts map[string]struct{}
-	httpOK    bool
-	// explicitPort is the operator-specified port from host:port input (if any).
-	// Network execution keeps this port (UseNetworkPort does not replace bare
-	// host:port), so reachability must not require the template's default port.
-	explicitPort string
-}
-
-// Allow reports whether template may produce a finding on input (lossless).
-func (idx *hostReachabilityIndex) Allow(t *templates.Template, mi *contextargs.MetaInput) bool {
-	if idx == nil || t == nil || mi == nil {
-		return true
-	}
-	h, ok := idx.byInput[mi.Input]
-	if !ok {
-		return true
-	}
-	return templateAllowedOnHost(t, h.httpOK, h.openPorts, h.explicitPort)
-}
-
-func templateAllowedOnHost(t *templates.Template, httpOK bool, openPorts map[string]struct{}, explicitPort string) bool {
-	if t.SelfContained || len(t.Workflows) > 0 || isUniversalTemplate(t) {
-		return true
-	}
-	if isWebTemplate(t) {
-		return httpOK
-	}
-	ports, ok := tcpNetworkOnlyPorts(t)
-	if !ok {
-		return true
-	}
-	// host:port inputs: dial the operator-chosen port at execution time, not the
-	// template default (see contextargs.UseNetworkPort). If that port is
-	// reachable, network templates may run against it.
-	if explicitPort != "" {
-		if _, open := openPorts[explicitPort]; open {
-			return true
-		}
-		return false
-	}
-	for _, p := range ports {
-		if _, open := openPorts[p]; open {
-			return true
-		}
-	}
-	return false
-}
-
-// estimateTemplateRequests mirrors core.getRequestCount for scheduling math.
-func estimateTemplateRequests(tpls []*templates.Template) int {
-	count := 0
-	for _, t := range tpls {
-		if len(t.Workflows) > 0 {
-			continue
-		}
-		if t.TotalRequests > 0 {
-			count += t.TotalRequests
-			continue
-		}
-		count++
-	}
-	return count
-}
-
-func hostGroupKey(openPorts []string, httpOK bool) string {
-	ports := append([]string(nil), openPorts...)
-	sort.Strings(ports)
-	http := "nohttp"
-	if httpOK {
-		http = "http"
-	}
-	if len(ports) == 0 {
-		return http + "|-"
-	}
-	return http + "|" + strings.Join(ports, ",")
-}
-
-func isWebTemplate(t *templates.Template) bool {
-	return len(t.RequestsHTTP)+len(t.RequestsWithHTTP)+len(t.RequestsHeadless)+len(t.RequestsWebsocket) > 0 &&
-		len(t.RequestsNetwork)+len(t.RequestsWithTCP)+len(t.RequestsDNS)+len(t.RequestsFile)+
-			len(t.RequestsSSL)+len(t.RequestsCode)+len(t.RequestsJavascript)+len(t.Workflows) == 0
-}
-
-func isUniversalTemplate(t *templates.Template) bool {
-	if t.SelfContained || len(t.Workflows) > 0 {
-		return true
-	}
-	if _, ok := tcpNetworkOnlyPorts(t); ok {
-		return false
-	}
-	if isWebTemplate(t) {
-		return false
-	}
-	return true
-}
-
-// templatesForHostGroup returns templates that can produce findings on this group.
-func templatesForHostGroup(all []*templates.Template, g hostGroup) []*templates.Template {
-	out := make([]*templates.Template, 0, len(all))
-	for _, t := range all {
-		if t.SelfContained || len(t.Workflows) > 0 || isUniversalTemplate(t) {
-			continue
-		}
-		if templateAllowedOnHost(t, g.httpOK, g.openPorts, g.explicitPort) {
-			out = append(out, t)
-		}
-	}
-	return out
-}
-
-func partitionTemplates(all []*templates.Template) (web, network, universal, selfContained []*templates.Template) {
-	for _, t := range all {
-		switch {
-		case t.SelfContained:
-			selfContained = append(selfContained, t)
-		case len(t.Workflows) > 0:
-			universal = append(universal, t)
-		case isWebTemplate(t):
-			web = append(web, t)
-		case isUniversalTemplate(t):
-			universal = append(universal, t)
-		default:
-			network = append(network, t)
-		}
-	}
-	return
-}
-
-// buildHostReachability probes each target once, then returns an index for
-// single-pass filtering plus host groups for logging/estimates.
-//
-// HTTP reachability prefers the already-populated InputHelper.InputsHTTP map
-// (from initializeTemplatesHTTPInput) so we do not re-run httpx.
-// When probePorts is false, only explicit input ports are recorded (HTTP-only mode).
-func (r *Runner) buildHostReachability(tpls []*templates.Template, httpHelper *input.Helper, probePorts bool) (*hostReachabilityIndex, []hostGroup, error) {
-	if r.inputProvider == nil {
-		return nil, nil, fmt.Errorf("no input provider")
-	}
-
-	var portsToProbe []string
-	if probePorts {
-		// Only concrete network-template ports. Do not pull 80/443 from HTTP
-		// templates via portsPopularityFromTemplates — httpOK already comes from
-		// scheme / InputsHTTP, and spraying web ports across every host inflates
-		// probe cost and flakes under parallel dial load.
-		seen := map[string]struct{}{}
-		for _, t := range tpls {
-			ps, ok := tcpNetworkOnlyPorts(t)
-			if !ok {
-				continue
-			}
-			for _, p := range ps {
-				if _, dup := seen[p]; dup {
-					continue
-				}
-				seen[p] = struct{}{}
-				portsToProbe = append(portsToProbe, p)
-			}
-		}
-		sort.Strings(portsToProbe)
-	}
-
-	var targets []*contextargs.MetaInput
-	r.inputProvider.Iterate(func(mi *contextargs.MetaInput) bool {
-		targets = append(targets, mi.Clone())
-		return true
-	})
-	if len(targets) == 0 {
-		return &hostReachabilityIndex{byInput: map[string]hostReachability{}}, nil, nil
-	}
-
-	// Only build an httpx client if we still need live HTTP probes.
-	needLiveHTTPProbe := false
-	for _, mi := range targets {
-		if strings.HasPrefix(mi.Input, "http://") || strings.HasPrefix(mi.Input, "https://") {
-			continue
-		}
-		if httpHelper != nil && httpHelper.InputsHTTPProbed {
-			continue // already probed (hit or miss) during initializeTemplatesHTTPInput
-		}
-		needLiveHTTPProbe = true
-		break
-	}
-
-	var httpClient *httpx.HTTPX
-	if needLiveHTTPProbe {
-		dialers := protocolstate.GetDialersWithId(r.options.ExecutionId)
-		if dialers != nil {
-			httpxOptions := httpx.DefaultOptions
-			if r.options.AliveHttpProxy != "" {
-				httpxOptions.Proxy = r.options.AliveHttpProxy
-			} else if r.options.AliveSocksProxy != "" {
-				httpxOptions.Proxy = r.options.AliveSocksProxy
-			}
-			httpxOptions.RetryMax = r.options.Retries
-			if r.options.Timeout > 0 {
-				httpxOptions.Timeout = time.Duration(r.options.Timeout) * time.Second
-			}
-			httpxOptions.NetworkPolicy = dialers.NetworkPolicy
-			if c, err := httpx.New(&httpxOptions); err == nil {
-				httpClient = c
-			}
-		}
-	}
-
-	workers := hostGroupProbeWorkers
-	// Fan out one task per host:port so closed-port timeouts do not serialize per host.
-	type probeJob struct {
-		hostIdx int
-		host    string
-		port    string
-	}
-	var jobs []probeJob
-	hostMeta := make([]struct {
-		mi           *contextargs.MetaInput
-		host         string
-		explicitPort string
-	}, len(targets))
-
-	for i, mi := range targets {
-		host, explicitPort := hostAndExplicitPort(mi.Target())
-		if host == "" {
-			host, explicitPort = hostAndExplicitPort(mi.Input)
-		}
-		hostMeta[i].mi = mi
-		hostMeta[i].host = host
-		hostMeta[i].explicitPort = explicitPort
-		if !probePorts {
-			continue
-		}
-		// Always probe the operator-specified port (may differ from template
-		// defaults). Do not spray unrelated template ports onto host:port inputs.
-		if explicitPort != "" && isNumericPort(explicitPort) {
-			jobs = append(jobs, probeJob{hostIdx: i, host: host, port: explicitPort})
-			continue
-		}
-		for _, p := range portsToProbe {
-			jobs = append(jobs, probeJob{hostIdx: i, host: host, port: p})
-		}
-	}
-
-	if len(jobs) < workers {
-		workers = len(jobs)
-	}
-	if workers < 1 {
-		workers = 1
-	}
-
-	openByHost := make([]map[string]struct{}, len(targets))
-	for i := range openByHost {
-		openByHost[i] = map[string]struct{}{}
-		// Only assume explicit ports are open when we are not probing them.
-		if !probePorts {
-			if ep := hostMeta[i].explicitPort; ep != "" && isNumericPort(ep) {
-				openByHost[i][ep] = struct{}{}
-			}
-		}
-	}
-
-	if len(jobs) > 0 {
-		swg, err := syncutil.New(syncutil.WithSize(workers))
-		if err != nil {
-			return nil, nil, err
-		}
-		var mu sync.Mutex
-		for _, job := range jobs {
-			swg.Add()
-			go func(job probeJob) {
-				defer swg.Done()
-				switch r.probe(job.host, job.port) {
-				case portOpen, portUnknown:
-					mu.Lock()
-					openByHost[job.hostIdx][job.port] = struct{}{}
-					mu.Unlock()
-				}
-			}(job)
-		}
-		swg.Wait()
-	}
-
-	results := make([]hostProbeResult, len(targets))
-	for i, meta := range hostMeta {
-		open := openByHost[i]
-		ports := make([]string, 0, len(open))
-		for p := range open {
-			ports = append(ports, p)
-		}
-		sort.Strings(ports)
-		results[i] = hostProbeResult{
-			input:        meta.mi,
-			openPorts:    ports,
-			httpOK:       resolveHTTPReachability(meta.mi, open, httpHelper, httpClient),
-			explicitPort: meta.explicitPort,
-		}
-	}
-
-	idx := &hostReachabilityIndex{byInput: make(map[string]hostReachability, len(results))}
-	grouped := map[string]*hostGroup{}
-	order := make([]string, 0)
-	for _, res := range results {
-		if res.input == nil {
-			continue
-		}
-		portsSet := make(map[string]struct{}, len(res.openPorts))
-		for _, p := range res.openPorts {
-			portsSet[p] = struct{}{}
-		}
-		idx.byInput[res.input.Input] = hostReachability{
-			openPorts:    portsSet,
-			httpOK:       res.httpOK,
-			explicitPort: res.explicitPort,
-		}
-
-		key := hostGroupKey(res.openPorts, res.httpOK)
-		if res.explicitPort != "" {
-			// Keep explicit host:port inputs in their own estimate bucket so
-			// network-template fan-out is not attributed to bare hosts.
-			key = key + "|ep:" + res.explicitPort
-		}
-		g, ok := grouped[key]
-		if !ok {
-			g = &hostGroup{
-				key:          key,
-				openPorts:    portsSet,
-				httpOK:       res.httpOK,
-				explicitPort: res.explicitPort,
-			}
-			grouped[key] = g
-			order = append(order, key)
-		}
-		g.hosts = append(g.hosts, res.input)
-	}
-
-	out := make([]hostGroup, 0, len(order))
-	for _, key := range order {
-		out = append(out, *grouped[key])
-	}
-	return idx, out, nil
-}
-
-func resolveHTTPReachability(mi *contextargs.MetaInput, open map[string]struct{}, helper *input.Helper, client *httpx.HTTPX) bool {
-	if mi == nil {
-		return false
-	}
-	if strings.HasPrefix(mi.Input, "http://") || strings.HasPrefix(mi.Input, "https://") {
-		return true
-	}
-	// Reuse httpx results from initializeTemplatesHTTPInput when available.
-	if helper != nil && helper.InputsHTTP != nil {
-		if probed, ok := helper.InputsHTTP.Get(mi.Input); ok && len(probed) > 0 {
-			return true
-		}
-		// Empty/skipped probe map (e.g. MultiFormat) must not force non-HTTP.
-		if !helper.InputsHTTPProbed {
-			// fall through to live probe / open ports
-		} else {
-			// Map was built for all inputs: absence means not HTTP.
-			return false
-		}
-	}
-	if client != nil {
-		return utils.ProbeURL(mi.Input, client) != ""
-	}
-	if _, ok := open["80"]; ok {
-		return true
-	}
-	if _, ok := open["443"]; ok {
-		return true
-	}
-	return false
-}
-
-// estimateGroupedExecutions returns baseline and filtered template×host counts.
-func estimateGroupedExecutions(all []*templates.Template, groups []hostGroup, hostCount int) (baseline, filtered int) {
-	baseline = estimateTemplateRequests(all) * hostCount
-	_, _, universal, selfContained := partitionTemplates(all)
-	once := append([]*templates.Template{}, selfContained...)
-	once = append(once, universal...)
-	filtered = estimateTemplateRequests(once) * hostCount
-	for _, g := range groups {
-		filtered += estimateTemplateRequests(templatesForHostGroup(all, g)) * len(g.hosts)
-	}
-	return baseline, filtered
-}
-
-// countReachabilityStats returns concrete network template count and distinct numeric ports to probe.
-func countReachabilityStats(tpls []*templates.Template) (concreteNetwork int, portsToProbe int) {
-	ports := map[string]struct{}{}
-	for _, t := range tpls {
-		ps, ok := tcpNetworkOnlyPorts(t)
-		if !ok {
-			continue
-		}
-		concreteNetwork++
-		for _, p := range ps {
-			ports[p] = struct{}{}
-		}
-	}
-	return concreteNetwork, len(ports)
-}

internal/runner/host_group_test.go

@@ -1,217 +0,0 @@
-package runner
-
-import (
-	"testing"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/input"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/network"
-	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
-	"github.com/stretchr/testify/require"
-)
-
-func TestHostGroupKey(t *testing.T) {
-	// Lexicographic sort: "443" < "80".
-	require.Equal(t, "http|443,80", hostGroupKey([]string{"443", "80"}, true))
-	require.Equal(t, "nohttp|22", hostGroupKey([]string{"22"}, false))
-	require.Equal(t, "nohttp|-", hostGroupKey(nil, false))
-}
-
-func TestTemplatesForHostGroup(t *testing.T) {
-	web := &templates.Template{
-		ID:           "web",
-		RequestsHTTP: []*http.Request{{}},
-	}
-	ftp := &templates.Template{
-		ID: "ftp",
-		RequestsNetwork: []*network.Request{{
-			Port: "21",
-		}},
-	}
-	ssh := &templates.Template{
-		ID: "ssh",
-		RequestsNetwork: []*network.Request{{
-			Port: "22",
-		}},
-	}
-	all := []*templates.Template{web, ftp, ssh}
-
-	webGroup := hostGroup{
-		key:       "http|80,443",
-		httpOK:    true,
-		openPorts: map[string]struct{}{"80": {}, "443": {}},
-	}
-	got := templatesForHostGroup(all, webGroup)
-	require.Len(t, got, 1)
-	require.Equal(t, "web", got[0].ID)
-
-	ftpGroup := hostGroup{
-		key:       "nohttp|21",
-		httpOK:    false,
-		openPorts: map[string]struct{}{"21": {}},
-	}
-	got = templatesForHostGroup(all, ftpGroup)
-	require.Len(t, got, 1)
-	require.Equal(t, "ftp", got[0].ID)
-
-	sshGroup := hostGroup{
-		key:       "nohttp|22",
-		httpOK:    false,
-		openPorts: map[string]struct{}{"22": {}},
-	}
-	got = templatesForHostGroup(all, sshGroup)
-	require.Len(t, got, 1)
-	require.Equal(t, "ssh", got[0].ID)
-}
-
-func TestPartitionTemplates(t *testing.T) {
-	web := &templates.Template{ID: "web", RequestsHTTP: []*http.Request{{}}}
-	ftp := &templates.Template{ID: "ftp", RequestsNetwork: []*network.Request{{Port: "21"}}}
-	self := &templates.Template{ID: "self", SelfContained: true}
-	dynNet := &templates.Template{
-		ID:              "dyn",
-		RequestsNetwork: []*network.Request{{Port: ""}},
-	}
-
-	w, n, u, s := partitionTemplates([]*templates.Template{web, ftp, self, dynNet})
-	require.Len(t, w, 1)
-	require.Equal(t, "web", w[0].ID)
-	require.Len(t, n, 1)
-	require.Equal(t, "ftp", n[0].ID)
-	require.Len(t, u, 1)
-	require.Equal(t, "dyn", u[0].ID)
-	require.Len(t, s, 1)
-	require.Equal(t, "self", s[0].ID)
-}
-
-func TestEstimateGroupedVsBaseline(t *testing.T) {
-	web := &templates.Template{ID: "web", RequestsHTTP: []*http.Request{{}}, TotalRequests: 1}
-	ftp := &templates.Template{ID: "ftp", RequestsNetwork: []*network.Request{{Port: "21"}}, TotalRequests: 1}
-	ssh := &templates.Template{ID: "ssh", RequestsNetwork: []*network.Request{{Port: "22"}}, TotalRequests: 1}
-	all := []*templates.Template{web, ftp, ssh}
-
-	groups := []hostGroup{
-		{key: "http|80", httpOK: true, openPorts: map[string]struct{}{"80": {}}, hosts: metaInputs("h1", "h2")},
-		{key: "nohttp|21", httpOK: false, openPorts: map[string]struct{}{"21": {}}, hosts: metaInputs("h3")},
-		{key: "nohttp|22", httpOK: false, openPorts: map[string]struct{}{"22": {}}, hosts: metaInputs("h4")},
-	}
-
-	baseline, filtered := estimateGroupedExecutions(all, groups, 4)
-	require.Equal(t, 12, baseline)
-	// web*2 + ftp*1 + ssh*1 = 4
-	require.Equal(t, 4, filtered)
-	require.Equal(t, 8, baseline-filtered)
-}
-
-func TestHostReachabilityAllow(t *testing.T) {
-	web := &templates.Template{ID: "web", RequestsHTTP: []*http.Request{{}}}
-	ftp := &templates.Template{ID: "ftp", RequestsNetwork: []*network.Request{{Port: "21"}}}
-	redis := &templates.Template{ID: "redis", RequestsNetwork: []*network.Request{{Port: "6379"}}}
-	idx := &hostReachabilityIndex{byInput: map[string]hostReachability{
-		"h1":              {httpOK: true, openPorts: map[string]struct{}{"80": {}}},
-		"h2":              {httpOK: false, openPorts: map[string]struct{}{"21": {}}},
-		"127.0.0.1:19637": {httpOK: false, openPorts: map[string]struct{}{"19637": {}}, explicitPort: "19637"},
-	}}
-	require.True(t, idx.Allow(web, &contextargs.MetaInput{Input: "h1"}))
-	require.False(t, idx.Allow(web, &contextargs.MetaInput{Input: "h2"}))
-	require.False(t, idx.Allow(ftp, &contextargs.MetaInput{Input: "h1"}))
-	require.True(t, idx.Allow(ftp, &contextargs.MetaInput{Input: "h2"}))
-	require.True(t, idx.Allow(web, &contextargs.MetaInput{Input: "unknown"})) // lossless
-	// Explicit host:port must allow network templates even when template default
-	// port (6379) differs from the operator-specified port (19637).
-	require.True(t, idx.Allow(redis, &contextargs.MetaInput{Input: "127.0.0.1:19637"}))
-	require.False(t, idx.Allow(web, &contextargs.MetaInput{Input: "127.0.0.1:19637"}))
-}
-
-func TestTemplateAllowedOnHostUniversal(t *testing.T) {
-	dnsish := &templates.Template{ID: "dns", RequestsDNS: nil} // no concrete network ports
-	// force universal via empty network dynamic port
-	dyn := &templates.Template{ID: "dyn", RequestsNetwork: []*network.Request{{Port: ""}}}
-	require.True(t, templateAllowedOnHost(dyn, false, map[string]struct{}{}, ""))
-	_ = dnsish
-	self := &templates.Template{ID: "self", SelfContained: true}
-	require.True(t, templateAllowedOnHost(self, false, nil, ""))
-}
-
-func TestTemplateAllowedOnHostExplicitPort(t *testing.T) {
-	redis := &templates.Template{ID: "redis", RequestsNetwork: []*network.Request{{Port: "6379"}}}
-	web := &templates.Template{ID: "web", RequestsHTTP: []*http.Request{{}}}
-	open := map[string]struct{}{"19637": {}}
-	require.True(t, templateAllowedOnHost(redis, false, open, "19637"))
-	require.False(t, templateAllowedOnHost(redis, false, open, "")) // bare host: need 6379
-	require.False(t, templateAllowedOnHost(web, false, open, "19637"))
-	// Closed explicit port: openPorts empty => network templates denied.
-	require.False(t, templateAllowedOnHost(redis, false, map[string]struct{}{}, "19637"))
-}
-
-func TestTemplatesForHostGroupExplicitPort(t *testing.T) {
-	redis := &templates.Template{ID: "redis", RequestsNetwork: []*network.Request{{Port: "6379"}}}
-	ftp := &templates.Template{ID: "ftp", RequestsNetwork: []*network.Request{{Port: "21"}}}
-	web := &templates.Template{ID: "web", RequestsHTTP: []*http.Request{{}}}
-	g := hostGroup{
-		httpOK:       false,
-		openPorts:    map[string]struct{}{"19637": {}},
-		explicitPort: "19637",
-	}
-	got := templatesForHostGroup([]*templates.Template{redis, ftp, web}, g)
-	require.Len(t, got, 2)
-	ids := map[string]bool{got[0].ID: true, got[1].ID: true}
-	require.True(t, ids["redis"] && ids["ftp"])
-}
-
-func TestCountReachabilityStats(t *testing.T) {
-	web := &templates.Template{ID: "web", RequestsHTTP: []*http.Request{{}}}
-	redis := &templates.Template{ID: "redis", RequestsNetwork: []*network.Request{{Port: "6379"}}}
-	pg := &templates.Template{ID: "pg", RequestsNetwork: []*network.Request{{Port: "5432,5432"}}}
-	dyn := &templates.Template{ID: "dyn", RequestsNetwork: []*network.Request{{Port: ""}}}
-	concrete, ports := countReachabilityStats([]*templates.Template{web, redis, pg, dyn})
-	require.Equal(t, 2, concrete)
-	require.Equal(t, 2, ports) // 6379 + 5432
-}
-
-func TestResolveHTTPReachability(t *testing.T) {
-	require.True(t, resolveHTTPReachability(&contextargs.MetaInput{Input: "http://web"}, nil, nil, nil))
-	require.True(t, resolveHTTPReachability(&contextargs.MetaInput{Input: "https://web"}, nil, nil, nil))
-	require.False(t, resolveHTTPReachability(&contextargs.MetaInput{Input: "redis1"}, map[string]struct{}{}, nil, nil))
-	require.True(t, resolveHTTPReachability(&contextargs.MetaInput{Input: "host"}, map[string]struct{}{"80": {}}, nil, nil))
-	require.True(t, resolveHTTPReachability(&contextargs.MetaInput{Input: "host"}, map[string]struct{}{"443": {}}, nil, nil))
-}
-
-func TestResolveHTTPReachabilitySkippedProbeFailOpen(t *testing.T) {
-	// MultiFormat leaves an empty InputsHTTP map without probing — must not
-	// force non-HTTP when open ports still suggest HTTP.
-	helper := &input.Helper{InputsHTTPProbed: false}
-	require.True(t, resolveHTTPReachability(&contextargs.MetaInput{Input: "host"}, map[string]struct{}{"80": {}}, helper, nil))
-}
-
-func TestTemplatesForHostGroupSkipsUniversal(t *testing.T) {
-	web := &templates.Template{ID: "web", RequestsHTTP: []*http.Request{{}}}
-	univ := &templates.Template{ID: "univ", RequestsNetwork: []*network.Request{{Port: ""}}}
-	g := hostGroup{httpOK: true, openPorts: map[string]struct{}{"80": {}}}
-	got := templatesForHostGroup([]*templates.Template{web, univ}, g)
-	require.Len(t, got, 1)
-	require.Equal(t, "web", got[0].ID)
-}
-
-func TestEstimateGroupedIncludesUniversalsOnce(t *testing.T) {
-	web := &templates.Template{ID: "web", RequestsHTTP: []*http.Request{{}}, TotalRequests: 1}
-	univ := &templates.Template{ID: "univ", RequestsNetwork: []*network.Request{{Port: ""}}, TotalRequests: 1}
-	groups := []hostGroup{
-		{httpOK: true, openPorts: map[string]struct{}{"80": {}}, hosts: metaInputs("h1")},
-	}
-	baseline, filtered := estimateGroupedExecutions([]*templates.Template{web, univ}, groups, 1)
-	require.Equal(t, 2, baseline)
-	// universal counted once for all hosts + web for http group
-	require.Equal(t, 2, filtered)
-}
-
-func metaInputs(hosts ...string) []*contextargs.MetaInput {
-	out := make([]*contextargs.MetaInput, 0, len(hosts))
-	for _, h := range hosts {
-		mi := contextargs.NewMetaInput()
-		mi.Input = h
-		out = append(out, mi)
-	}
-	return out
-}

internal/runner/inputs.go

@@ -39,6 +39,9 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 	}
 	httpxOptions.RetryMax = r.options.Retries
 	httpxOptions.Timeout = time.Duration(r.options.Timeout) * time.Second
+	if len(r.options.InternalResolversList) > 0 {
+		httpxOptions.Resolvers = r.options.InternalResolversList
+	}
 
 	dialers := protocolstate.GetDialersWithId(r.options.ExecutionId)
 	if dialers == nil {

internal/runner/llm.go

@@ -2,8 +2,12 @@ package runner
 
 import (
 	"context"
+	"errors"
+	"sync"
 	"time"
 
+	"github.com/projectdiscovery/gologger"
+
 	llmclient "github.com/projectdiscovery/nuclei/v3/pkg/operators/common/llm"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	utilsllm "github.com/projectdiscovery/utils/llm"
@@ -28,6 +32,32 @@ func (a *llmMatcherClient) Complete(ctx context.Context, prompt string, asJSON b
 	})
 }
 
+// llmJudgeClient is an llmMatcherClient whose provider classifies natively, so
+// llm matchers ask it through Judge instead of Complete.
+type llmJudgeClient struct {
+	*llmMatcherClient
+	warnComplete sync.Once
+}
+
+func (a *llmJudgeClient) Judge(ctx context.Context, question string, labels, inputs []string) (llmclient.Verdict, error) {
+	verdict, err := a.client.Judge(ctx, utilsllm.JudgeRequest{Question: question, Labels: labels, Inputs: inputs})
+	if err != nil {
+		return llmclient.Verdict{}, err
+	}
+	return llmclient.Verdict{Label: verdict.Label, Confidence: verdict.Confidence, Model: verdict.Model}, nil
+}
+
+// Complete is only reached by llm extractors, which need free text a
+// classifier cannot produce. Returning here spends no call budget and
+// makes no provider request. Extractors turn the error into an empty
+// result, and the warning says why.
+func (a *llmJudgeClient) Complete(context.Context, string, bool) (string, error) {
+	a.warnComplete.Do(func() {
+		gologger.Warning().Msgf("llm provider %q cannot run llm extractors, they return nothing", utilsllm.ProviderJev)
+	})
+	return "", errors.New("llm provider cannot run extractors")
+}
+
 // configureLLM builds the scan's llm client from options. It returns nil unless
 // -llm is set, so nothing reaches a model until the user opts in. The API key is
 // read from the environment by the provider layer, never from options.
@@ -52,5 +82,9 @@ func configureLLM(options *types.Options) (llmclient.Client, error) {
 		return nil, err
 	}
 
-	return &llmMatcherClient{client: client, model: options.LLMModel}, nil
+	matcherClient := &llmMatcherClient{client: client, model: options.LLMModel}
+	if client.CanJudge() {
+		return &llmJudgeClient{llmMatcherClient: matcherClient}, nil
+	}
+	return matcherClient, nil
 }

internal/runner/llm_test.go

@@ -0,0 +1,65 @@
+package runner
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+
+	llmclient "github.com/projectdiscovery/nuclei/v3/pkg/operators/common/llm"
+	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConfigureLLMExposesJudgeOnlyForClassifiers(t *testing.T) {
+	t.Setenv("LLM_API_KEY", "key")
+
+	client, err := configureLLM(&types.Options{EnableLLM: true, LLMProvider: "jev", LLMModel: "jev-latest"})
+	require.NoError(t, err)
+	require.Implements(t, (*llmclient.Judge)(nil), client)
+
+	client, err = configureLLM(&types.Options{EnableLLM: true, LLMProvider: "ollama", LLMModel: "llama3.2"})
+	require.NoError(t, err)
+	_, isJudge := client.(llmclient.Judge)
+	require.False(t, isJudge, "chat providers keep the Complete path")
+}
+
+func TestJevExtractorDoesNotSpendCallBudget(t *testing.T) {
+	t.Setenv("LLM_API_KEY", "key")
+	// ProxyFromEnvironment caches the first request in the process. Match the
+	// proxy the interactsh client test expects, and keep the local server direct.
+	for _, key := range []string{"HTTP_PROXY", "http_proxy", "ALL_PROXY", "all_proxy", "https_proxy", "no_proxy"} {
+		t.Setenv(key, "")
+	}
+	t.Setenv("HTTPS_PROXY", "http://127.0.0.1:9")
+	t.Setenv("NO_PROXY", "127.0.0.1,localhost")
+
+	var calls atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		calls.Add(1)
+		_, _ = w.Write([]byte(`{"model":"jev-1.14.0","answers":{"q":{"type":"noul","noul":0.9}}}`))
+	}))
+	t.Cleanup(server.Close)
+
+	client, err := configureLLM(&types.Options{
+		EnableLLM:   true,
+		LLMProvider: "jev",
+		LLMBaseURL:  server.URL + "/v1",
+		LLMModel:    "jev-latest",
+		LLMMaxCalls: 1,
+	})
+	require.NoError(t, err)
+
+	_, err = client.Complete(context.Background(), "extract the title", true)
+	require.ErrorContains(t, err, "cannot run extractors")
+	_, err = client.Complete(context.Background(), "extract another", true)
+	require.Error(t, err)
+	require.Zero(t, calls.Load())
+
+	judge := client.(llmclient.Judge)
+	verdict, err := judge.Judge(context.Background(), "login page?", nil, []string{"<form>"})
+	require.NoError(t, err)
+	require.Equal(t, "yes", verdict.Label)
+	require.EqualValues(t, 1, calls.Load())
+}

internal/runner/options.go

@@ -228,25 +228,14 @@ func ValidateOptions(options *types.Options) error {
 		}
 	}
 
-	// verify that a valid ip version type was selected (4, 6)
-	if len(options.IPVersion) == 0 {
-		// add ipv4 as default
-		options.IPVersion = append(options.IPVersion, "4")
-	}
-	var useIPV4, useIPV6 bool
+	// An empty -iv leaves one unpinned target so the dialer can fall back
+	// between address families. -sa resolves and scans every selected address.
+	// Explicit -iv 4, -iv 6 and -iv 4,6 retain their usual selection semantics.
 	for _, ipv := range options.IPVersion {
-		switch ipv {
-		case "4":
-			useIPV4 = true
-		case "6":
-			useIPV6 = true
-		default:
+		if ipv != "4" && ipv != "6" {
 			return fmt.Errorf("unsupported ip version: %s", ipv)
 		}
 	}
-	if !useIPV4 && !useIPV6 {
-		return errors.New("ipv4 and/or ipv6 must be selected")
-	}
 	return nil
 }
 

internal/runner/options_test.go

@@ -90,3 +90,15 @@ func TestReadsTargetsFromStdin(t *testing.T) {
 		})
 	}
 }
+
+// The default must remain empty so the provider can distinguish automatic
+// IPv6 preference from explicit user-supplied -iv selections.
+func TestValidateOptionsIPVersionPreference(t *testing.T) {
+	automatic := &types.Options{}
+	require.NoError(t, ValidateOptions(automatic))
+	require.Empty(t, automatic.IPVersion)
+	for _, versions := range []goflags.StringSlice{{"4"}, {"6"}, {"4", "6"}} {
+		require.NoError(t, ValidateOptions(&types.Options{IPVersion: versions}))
+	}
+	require.ErrorContains(t, ValidateOptions(&types.Options{IPVersion: goflags.StringSlice{"7"}}), "unsupported ip version")
+}

internal/runner/plan_scope.go

@@ -1,104 +0,0 @@
-package runner
-
-import (
-	"sync"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
-	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
-)
-
-// planScope narrows the templates run on each target by the scan plan filters
-// (strict-probe reachability, tech filter), on top of the per-target profiles
-// in base. It is a protocols.TargetScope, so the engine skips, counts and
-// narrows clusters for these pairs the same way it does for profiles.
-type planScope struct {
-	base  protocols.TargetScope
-	allow func(*templates.Template, *contextargs.MetaInput) bool
-	// byPath maps a template path to its template. A path shared by several
-	// templates, or empty, is left out, so such templates are never filtered.
-	byPath map[string]*templates.Template
-
-	mu         sync.RWMutex
-	selections map[planKey]*planSelection
-}
-
-// planKey identifies one selection: the target and the profile selection it
-// narrows. Profile selections are comparable by contract.
-type planKey struct {
-	input string
-	ip    string
-	base  protocols.TemplateSelection
-}
-
-// planSelection is the selection of one target.
-type planSelection struct {
-	base  protocols.TemplateSelection
-	scope *planScope
-	input *contextargs.MetaInput
-}
-
-var _ protocols.TargetScope = (*planScope)(nil)
-
-func newPlanScope(base protocols.TargetScope, allow func(*templates.Template, *contextargs.MetaInput) bool, tpls []*templates.Template) *planScope {
-	byPath := make(map[string]*templates.Template, len(tpls))
-	shared := make(map[string]struct{})
-	for _, t := range tpls {
-		if t == nil || t.Path == "" {
-			continue
-		}
-		if _, ok := byPath[t.Path]; ok {
-			shared[t.Path] = struct{}{}
-			continue
-		}
-		byPath[t.Path] = t
-	}
-	for path := range shared {
-		delete(byPath, path)
-	}
-	return &planScope{
-		base:       base,
-		allow:      allow,
-		byPath:     byPath,
-		selections: make(map[planKey]*planSelection),
-	}
-}
-
-// For returns the selection for input. It is the same value for the same
-// input, so the engine can group targets by selection.
-func (s *planScope) For(input *contextargs.MetaInput) protocols.TemplateSelection {
-	var base protocols.TemplateSelection
-	if s.base != nil {
-		base = s.base.For(input)
-	}
-	if input == nil {
-		return base
-	}
-	key := planKey{input: input.Input, ip: input.CustomIP, base: base}
-	s.mu.RLock()
-	sel, ok := s.selections[key]
-	s.mu.RUnlock()
-	if ok {
-		return sel
-	}
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	if sel, ok := s.selections[key]; ok {
-		return sel
-	}
-	sel = &planSelection{base: base, scope: s, input: input.Clone()}
-	s.selections[key] = sel
-	return sel
-}
-
-// Allows reports whether the profiles and the plan filters select the template.
-func (p *planSelection) Allows(templatePath string) bool {
-	if p.base != nil && !p.base.Allows(templatePath) {
-		return false
-	}
-	t, ok := p.scope.byPath[templatePath]
-	if !ok {
-		return true
-	}
-	return p.scope.allow(t, p.input)
-}

internal/runner/plan_scope_test.go

@@ -1,52 +0,0 @@
-package runner
-
-import (
-	"testing"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
-	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
-	"github.com/stretchr/testify/require"
-)
-
-// profileScope selects by path, like per-target profiles do.
-type profileScope struct{ sel *profileSelection }
-
-func (p profileScope) For(*contextargs.MetaInput) protocols.TemplateSelection { return p.sel }
-
-type profileSelection struct{ allowed map[string]bool }
-
-func (p *profileSelection) Allows(path string) bool { return p.allowed[path] }
-
-func TestPlanScope(t *testing.T) {
-	web := &templates.Template{ID: "web", Path: "/t/web.yaml"}
-	net := &templates.Template{ID: "net", Path: "/t/net.yaml"}
-	dupA := &templates.Template{ID: "dup-a", Path: "/t/dup.yaml"}
-	dupB := &templates.Template{ID: "dup-b", Path: "/t/dup.yaml"}
-	// the filter drops the network template on host-a only
-	allow := func(tpl *templates.Template, mi *contextargs.MetaInput) bool {
-		return mi.Input != "host-a" || tpl.ID == "web"
-	}
-	input := func(raw string) *contextargs.MetaInput {
-		mi := contextargs.NewMetaInput()
-		mi.Input = raw
-		return mi
-	}
-	a, b := input("host-a"), input("host-b")
-
-	scope := newPlanScope(nil, allow, []*templates.Template{web, net, dupA, dupB})
-	selA := scope.For(a)
-	require.Same(t, selA, scope.For(input("host-a")), "one selection per target, so the engine can group targets")
-	require.True(t, web.SelectedBy(selA))
-	require.False(t, net.SelectedBy(selA))
-	require.True(t, dupA.SelectedBy(selA), "a path shared by several templates is never filtered")
-	require.True(t, net.SelectedBy(scope.For(b)))
-	require.True(t, (&templates.Template{Path: "/t/unknown.yaml"}).SelectedBy(selA), "unknown templates are never filtered")
-
-	// The plan narrows the profiles, never widens them.
-	profiles := profileScope{sel: &profileSelection{allowed: map[string]bool{"/t/net.yaml": true}}}
-	scope = newPlanScope(profiles, allow, []*templates.Template{web, net})
-	require.False(t, web.SelectedBy(scope.For(b)), "a template the profile leaves out stays out")
-	require.True(t, net.SelectedBy(scope.For(b)))
-	require.False(t, net.SelectedBy(scope.For(a)))
-}

internal/runner/reachability.go

@@ -1,199 +0,0 @@
-package runner
-
-import (
-	"context"
-	"net"
-	"net/url"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/projectdiscovery/httpx/common/httpx"
-	"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/network"
-	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
-	"github.com/projectdiscovery/nuclei/v3/pkg/utils"
-	sliceutil "github.com/projectdiscovery/utils/slice"
-)
-
-type probeResult int
-
-const (
-	portClosed  probeResult = iota // connection refused / host unreachable
-	portOpen                       // connect succeeded
-	portUnknown                    // timeout / filtered — treat as possibly open
-)
-
-const reachabilityProbeTimeout = 750 * time.Millisecond
-
-// strictProbeEnabled reports whether the lossless reachability prune
-// applies to this run. It is limited to standard target inputs; request-shaped
-// (DAST / multi-format) inputs are never pruned, because probing a request
-// template as a live service is meaningless and could drop coverage.
-func (r *Runner) strictProbeEnabled() bool {
-	return r.options.StrictProbe &&
-		r.inputProvider != nil &&
-		r.inputProvider.InputType() != provider.MultiFormatInputProvider
-}
-
-// noHTTPServiceReachable reports whether NO target actually speaks HTTP(S).
-//
-// When it returns true, web templates (HTTP/headless/websocket) can be excluded
-// losslessly: a port that does not serve HTTP has no HTTP application, so no
-// HTTP template can produce a real finding there. Unlike a bare TCP-open check,
-// this catches the open-but-non-HTTP case (e.g. redis on 6379): the port is
-// open, but it is not an HTTP service, so HTTP templates are pure waste.
-//
-// It reuses nuclei's own httpx probe (the same one used for input routing), so
-// the determination is consistent with the engine and respects the network
-// policy (the probe dials only through the policy-configured httpx client).
-// Conservative: an explicit http/https target, any scheme that responds, or the
-// inability to probe safely all keep web templates in place.
-// Skipped when -no-httpx is set so we do not probe or prune against user intent.
-func (r *Runner) noHTTPServiceReachable() bool {
-	if r.inputProvider == nil || r.options.DisableHTTPProbe {
-		return false
-	}
-	dialers := protocolstate.GetDialersWithId(r.options.ExecutionId)
-	if dialers == nil {
-		return false // cannot probe within policy — keep web templates
-	}
-	httpxOptions := httpx.DefaultOptions
-	if r.options.AliveHttpProxy != "" {
-		httpxOptions.Proxy = r.options.AliveHttpProxy
-	} else if r.options.AliveSocksProxy != "" {
-		httpxOptions.Proxy = r.options.AliveSocksProxy
-	}
-	httpxOptions.RetryMax = r.options.Retries
-	if r.options.Timeout > 0 {
-		httpxOptions.Timeout = time.Duration(r.options.Timeout) * time.Second
-	}
-	httpxOptions.NetworkPolicy = dialers.NetworkPolicy
-	client, err := httpx.New(&httpxOptions)
-	if err != nil {
-		return false // cannot probe — keep web templates
-	}
-
-	anyHTTP := false
-	r.inputProvider.Iterate(func(mi *contextargs.MetaInput) bool {
-		if strings.HasPrefix(mi.Input, "http://") || strings.HasPrefix(mi.Input, "https://") {
-			anyHTTP = true
-			return false
-		}
-		if utils.ProbeURL(mi.Input, client) != "" {
-			anyHTTP = true
-			return false
-		}
-		return true
-	})
-	return !anyHTTP
-}
-
-// tcpNetworkOnlyPorts returns the concrete numeric TCP ports of a single-protocol
-// network template, or ok=false if the template is ineligible for port pruning.
-func tcpNetworkOnlyPorts(t *templates.Template) ([]string, bool) {
-	if len(t.RequestsNetwork)+len(t.RequestsWithTCP) == 0 {
-		return nil, false
-	}
-	// must be network-only (no other protocol request could still be reachable)
-	otherReqs := len(t.RequestsHTTP) + len(t.RequestsWithHTTP) + len(t.RequestsHeadless) +
-		len(t.RequestsDNS) + len(t.RequestsFile) + len(t.RequestsSSL) +
-		len(t.RequestsCode) + len(t.RequestsJavascript) + len(t.Workflows)
-	if otherReqs > 0 {
-		return nil, false
-	}
-	reqs := make([]*network.Request, 0, len(t.RequestsNetwork)+len(t.RequestsWithTCP))
-	reqs = append(reqs, t.RequestsNetwork...)
-	reqs = append(reqs, t.RequestsWithTCP...)
-
-	var ports []string
-	for _, req := range reqs {
-		for _, addr := range req.Address {
-			if strings.Contains(strings.ToLower(addr), "udp://") {
-				return nil, false // UDP: a TCP probe cannot prove unreachability
-			}
-		}
-		pp := splitPorts(req.Port)
-		if len(pp) == 0 {
-			return nil, false // empty/dynamic port (uses input port) — keep
-		}
-		for _, p := range pp {
-			if !isNumericPort(p) {
-				return nil, false // service name or template var — keep
-			}
-			ports = append(ports, p)
-		}
-	}
-	if len(ports) == 0 {
-		return nil, false
-	}
-	return sliceutil.Dedupe(ports), true
-}
-
-func isNumericPort(p string) bool {
-	n, err := strconv.Atoi(p)
-	return err == nil && n > 0 && n < 65536
-}
-
-// hostAndExplicitPort splits a target into host and explicit port (port empty if
-// the target is a bare host).
-func hostAndExplicitPort(input string) (string, string) {
-	if strings.Contains(input, "://") {
-		if u, err := url.Parse(input); err == nil && u.Hostname() != "" {
-			return u.Hostname(), u.Port()
-		}
-	}
-	if host, port, err := net.SplitHostPort(input); err == nil {
-		return host, port
-	}
-	return input, ""
-}
-
-// probe classifies reachability of host:port WITHIN the scan's network policy.
-// Security first: it never dials a target the policy forbids and never bypasses
-// the sandbox — all probing goes through the execution's fastdialer, which
-// enforces the same DenyList (‑lna, exclude-targets, private/metadata ranges,
-// DNS-rebind protection) as the scan itself.
-//
-// It reconciles security with losslessness by mapping every uncertain case to
-// portUnknown ("keep, don't dial"): a policy-denied host, a missing dialer, or a
-// timeout all leave templates in place. Pruning happens only on a definitive,
-// policy-permitted, refused connection.
-func (r *Runner) probe(host, port string) probeResult {
-	if !protocolstate.IsHostAllowed(r.options.ExecutionId, host) {
-		// Policy denies this host: do not dial (security), do not prune (lossless).
-		return portUnknown
-	}
-	dialers := protocolstate.GetDialersWithId(r.options.ExecutionId)
-	if dialers == nil || dialers.Fastdialer == nil {
-		// No policy-enforcing dialer available: never fall back to a raw dial.
-		return portUnknown
-	}
-	return classifyDial(dialers.Fastdialer.Dial, net.JoinHostPort(host, port), reachabilityProbeTimeout)
-}
-
-type dialFunc func(ctx context.Context, network, addr string) (net.Conn, error)
-
-// classifyDial performs one connect via the supplied (policy-enforcing) dialer.
-func classifyDial(dial dialFunc, addr string, timeout time.Duration) probeResult {
-	ctx, cancel := context.WithTimeout(context.Background(), timeout)
-	defer cancel()
-	conn, err := dial(ctx, "tcp", addr)
-	if err == nil {
-		_ = conn.Close()
-		return portOpen
-	}
-	if ne, ok := err.(net.Error); ok && ne.Timeout() {
-		return portUnknown
-	}
-	msg := strings.ToLower(err.Error())
-	// Only definitive negatives prune. DNS blips / temporary dialer errors must
-	// stay unknown so mixed-fleet probing under load remains lossless.
-	if strings.Contains(msg, "refused") || strings.Contains(msg, "unreachable") ||
-		strings.Contains(msg, "no route to host") || strings.Contains(msg, "network is unreachable") {
-		return portClosed
-	}
-	return portUnknown
-}

internal/runner/reachability_test.go

@@ -1,133 +0,0 @@
-package runner
-
-import (
-	"context"
-	"net"
-	"testing"
-	"time"
-
-	httpproto "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http"
-	jsproto "github.com/projectdiscovery/nuclei/v3/pkg/protocols/javascript"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/network"
-	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
-)
-
-func TestHostAndExplicitPort(t *testing.T) {
-	cases := []struct{ in, host, port string }{
-		{"127.0.0.1", "127.0.0.1", ""},
-		{"127.0.0.1:6379", "127.0.0.1", "6379"},
-		{"http://example.com:8080", "example.com", "8080"},
-		{"https://example.com", "example.com", ""},
-	}
-	for _, c := range cases {
-		h, p := hostAndExplicitPort(c.in)
-		if h != c.host || p != c.port {
-			t.Errorf("hostAndExplicitPort(%q) = (%q,%q), want (%q,%q)", c.in, h, p, c.host, c.port)
-		}
-	}
-}
-
-func TestIsNumericPort(t *testing.T) {
-	for _, p := range []string{"80", "6379", "65535", "1"} {
-		if !isNumericPort(p) {
-			t.Errorf("isNumericPort(%q) = false, want true", p)
-		}
-	}
-	for _, p := range []string{"ftp", "{{Port}}", "0", "70000", "", "80a"} {
-		if isNumericPort(p) {
-			t.Errorf("isNumericPort(%q) = true, want false", p)
-		}
-	}
-}
-
-// TestTCPNetworkOnlyPorts locks in the losslessness guards: only single-protocol
-// network templates with concrete numeric TCP ports are eligible for pruning.
-func TestTCPNetworkOnlyPorts(t *testing.T) {
-	netTmpl := func(addr []string, port string) *templates.Template {
-		return &templates.Template{RequestsNetwork: []*network.Request{{Address: addr, Port: port}}}
-	}
-
-	t.Run("concrete tcp port eligible", func(t *testing.T) {
-		ports, ok := tcpNetworkOnlyPorts(netTmpl([]string{"{{Hostname}}"}, "6379,6380"))
-		if !ok || len(ports) != 2 {
-			t.Fatalf("want ok with 2 ports, got ok=%v ports=%v", ok, ports)
-		}
-	})
-	t.Run("tls address still tcp", func(t *testing.T) {
-		if _, ok := tcpNetworkOnlyPorts(netTmpl([]string{"tls://{{Hostname}}"}, "6379")); !ok {
-			t.Fatal("tls:// should remain eligible (still TCP)")
-		}
-	})
-	t.Run("udp address rejected", func(t *testing.T) {
-		if _, ok := tcpNetworkOnlyPorts(netTmpl([]string{"udp://{{Hostname}}"}, "161")); ok {
-			t.Fatal("udp:// must be ineligible — a TCP probe cannot prove UDP unreachability")
-		}
-	})
-	t.Run("service-name port rejected", func(t *testing.T) {
-		if _, ok := tcpNetworkOnlyPorts(netTmpl([]string{"{{Hostname}}"}, "ftp")); ok {
-			t.Fatal("service-name port must be ineligible")
-		}
-	})
-	t.Run("dynamic port rejected", func(t *testing.T) {
-		if _, ok := tcpNetworkOnlyPorts(netTmpl([]string{"{{Hostname}}"}, "{{Port}}")); ok {
-			t.Fatal("template-var port must be ineligible")
-		}
-	})
-	t.Run("empty port rejected", func(t *testing.T) {
-		if _, ok := tcpNetworkOnlyPorts(netTmpl([]string{"{{Hostname}}"}, "")); ok {
-			t.Fatal("empty port (uses input port) must be ineligible")
-		}
-	})
-	t.Run("mixed protocol rejected", func(t *testing.T) {
-		mixed := netTmpl([]string{"{{Hostname}}"}, "6379")
-		mixed.RequestsHTTP = []*httpproto.Request{{}}
-		if _, ok := tcpNetworkOnlyPorts(mixed); ok {
-			t.Fatal("template with an HTTP request must be ineligible")
-		}
-	})
-	t.Run("javascript protocol rejected", func(t *testing.T) {
-		js := &templates.Template{RequestsJavascript: []*jsproto.Request{{}}}
-		if _, ok := tcpNetworkOnlyPorts(js); ok {
-			t.Fatal("javascript template must be ineligible (opaque dial protocol)")
-		}
-	})
-}
-
-func TestClassifyDial(t *testing.T) {
-	ln, err := net.Listen("tcp", "127.0.0.1:0")
-	if err != nil {
-		t.Skip("cannot listen:", err)
-	}
-	defer func() { _ = ln.Close() }()
-	d := &net.Dialer{}
-	if got := classifyDial(d.DialContext, ln.Addr().String(), reachabilityProbeTimeout); got != portOpen {
-		t.Errorf("classifyDial(open) = %v, want portOpen", got)
-	}
-	// an unused low port on loopback should refuse quickly
-	if got := classifyDial(d.DialContext, "127.0.0.1:1", reachabilityProbeTimeout); got != portClosed {
-		t.Errorf("classifyDial(closed) = %v, want portClosed", got)
-	}
-}
-
-func TestClassifyDialDNSFailureIsUnknown(t *testing.T) {
-	// NXDOMAIN / resolution failure must not prune (lossless under load).
-	dial := func(ctx context.Context, network, addr string) (net.Conn, error) {
-		return nil, &net.OpError{Op: "dial", Net: network, Err: &net.DNSError{Err: "no such host", Name: "no-such-host.invalid", IsNotFound: true}}
-	}
-	got := classifyDial(dial, "no-such-host.invalid:6379", reachabilityProbeTimeout)
-	if got != portUnknown {
-		t.Fatalf("classifyDial(dns-fail) = %v, want portUnknown", got)
-	}
-}
-
-func TestClassifyDialTimeoutIsUnknown(t *testing.T) {
-	// A dropped connect runs into the timeout and must not prune.
-	dial := func(ctx context.Context, network, addr string) (net.Conn, error) {
-		<-ctx.Done()
-		return nil, &net.OpError{Op: "dial", Net: network, Err: ctx.Err()}
-	}
-	got := classifyDial(dial, "192.0.2.1:65535", 50*time.Millisecond)
-	if got != portUnknown {
-		t.Fatalf("classifyDial(timeout) = %v, want portUnknown", got)
-	}
-}

internal/runner/runner.go

@@ -45,8 +45,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
 	"github.com/projectdiscovery/nuclei/v3/pkg/core"
-	scanplan "github.com/projectdiscovery/nuclei/v3/pkg/core/plan"
-	"github.com/projectdiscovery/nuclei/v3/pkg/core/techfilter"
 	"github.com/projectdiscovery/nuclei/v3/pkg/external/customtemplates"
 	fuzzStats "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/stats"
 	"github.com/projectdiscovery/nuclei/v3/pkg/input"
@@ -69,19 +67,16 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/excludematchers"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
 	httpProtocol "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/catchall"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
-	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
-	"github.com/projectdiscovery/nuclei/v3/pkg/types/scanstrategy"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils/stats"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml"
 	"github.com/projectdiscovery/retryablehttp-go"
 	ptrutil "github.com/projectdiscovery/utils/ptr"
-	stringsutil "github.com/projectdiscovery/utils/strings"
 )
 
 var (
@@ -695,12 +690,6 @@ func (r *Runner) RunEnumeration() error {
 		DoNotCache:          r.options.DoNotCacheTemplates,
 		Logger:              r.Logger,
 	}
-	// Attach scan-scoped HTTP response cache only for opt-in -tf, and only
-	// before templates compile so request executers share the same pointer.
-	// Default scans leave this nil (no behavioral change).
-	if r.options.UsesResponseCache() {
-		executorOpts.HTTPResponseCache = httprespcache.New()
-	}
 
 	if config.DefaultConfig.IsDebugArgEnabled(config.DebugExportURLPattern) {
 		// Go StdLib style experimental/debug feature switch
@@ -746,6 +735,9 @@ func (r *Runner) RunEnumeration() error {
 	if r.options.HostBackoff {
 		executorOpts.HostBackoff = hostbackoff.New(hostbackoff.Config{Max: r.options.HostBackoffMax})
 	}
+	if r.options.AutoCalibrate {
+		executorOpts.CatchAll = catchall.New(0)
+	}
 
 	executorEngine := core.New(r.options)
 	executorEngine.SetExecuterOptions(executorOpts)
@@ -758,16 +750,6 @@ func (r *Runner) RunEnumeration() error {
 
 	// If using input-file flags, only load http fuzzing based templates.
 	loaderConfig := loader.NewConfig(r.options, r.catalog, executorOpts)
-
-	// Lossless reachability prune: if no web port is reachable on any target,
-	// exclude web-protocol templates at load time — they cannot connect, so
-	// cannot produce a finding. Reuses the standard protocol-type exclusion on
-	// the loader config only, so shared options stay unchanged across runs.
-	if r.strictProbeEnabled() && r.noHTTPServiceReachable() {
-		loaderConfig.ExcludeProtocols = append(slices.Clone(loaderConfig.ExcludeProtocols),
-			templateTypes.HTTPProtocol, templateTypes.HeadlessProtocol, templateTypes.WebsocketProtocol)
-		r.Logger.Info().Msgf("reachability prune: no HTTP service on any target; excluding web-protocol templates (lossless)")
-	}
 	profiles := r.targetProfiles()
 	if profiles != nil {
 		if r.options.AutomaticScan {
@@ -879,15 +861,6 @@ func (r *Runner) RunEnumeration() error {
 			return errors.Wrap(err, "could not probe http input")
 		}
 		executorOpts.InputHelper.InputsHTTP = inputHelpers
-		executorOpts.InputHelper.InputsHTTPProbed = r.inputProvider.InputType() != provider.MultiFormatInputProvider
-		// Under strict-probe, skip web templates per-input for targets that
-		// probed as non-HTTP (lossless), instead of falling back to raw URL.
-		executorOpts.InputHelper.StrictProbe = r.strictProbeEnabled()
-		if r.strictProbeEnabled() {
-			r.Logger.Info().Msgf("Strict probe enabled: hosts without httpx confirmation are skipped for HTTP/headless templates")
-		}
-	} else if r.strictProbeEnabled() && r.options.DisableHTTPProbe {
-		r.Logger.Warning().Msgf("strict-probe has no effect with -no-httpx (httpx probing is disabled)")
 	}
 
 	inputCount := int(r.inputProvider.Count())
@@ -1062,157 +1035,15 @@ func (r *Runner) executeTemplatesInput(store *loader.Store, engine *core.Engine)
 		return nil, errors.New("no templates provided for scan")
 	}
 
+	// pass input provider to engine
+	// TODO: this should be not necessary after r.hmapInputProvider is removed + refactored
 	if r.inputProvider == nil {
 		return nil, errors.New("no input provider found")
 	}
-
-	return r.executeTemplatesWithScanPlan(engine, finalTemplates)
-}
-
-// executeTemplatesWithScanPlan chooses spray strategy and optionally attaches a
-// reachability / tech filter, then runs a single Execute pass. Wall clock must
-// match or beat baseline spray; probing is skipped when its cost would dominate.
-func (r *Runner) executeTemplatesWithScanPlan(engine *core.Engine, finalTemplates []*templates.Template) (*atomic.Bool, error) {
-	defer r.attachScanPlan(engine, finalTemplates)()
 	results := engine.ExecuteScanWithOpts(context.Background(), finalTemplates, r.inputProvider, r.options.DisableClustering)
 	return results, nil
 }
 
-// attachScanPlan applies the scan plan to engine for one scan. It returns a
-// func that undoes the per-scan state once the scan is done.
-func (r *Runner) attachScanPlan(engine *core.Engine, finalTemplates []*templates.Template) func() {
-	var undo []func()
-	cleanup := func() {
-		for i := len(undo) - 1; i >= 0; i-- {
-			undo[i]()
-		}
-	}
-	hostCount := int(r.inputProvider.Count())
-	baselineReqs := estimateTemplateRequests(finalTemplates) * hostCount
-	concreteNet, portsToProbe := countReachabilityStats(finalTemplates)
-	techBound := techfilter.CountTechBound(finalTemplates)
-
-	planIn := scanplan.Input{
-		Hosts:                    hostCount,
-		Templates:                len(finalTemplates),
-		Requests:                 baselineReqs,
-		BulkSize:                 r.options.BulkSize,
-		Stream:                   r.options.Stream,
-		StrictProbe:              r.strictProbeEnabled(),
-		PortsToProbe:             portsToProbe,
-		ConcreteNetworkTemplates: concreteNet,
-		TechFilter:               r.options.TechFilter,
-		TechBoundTemplates:       techBound,
-	}
-	p := scanplan.Decide(planIn)
-
-	// Resolve -ss auto before Execute so the engine placeholder is unused.
-	if stringsutil.EqualFoldAny(r.options.ScanStrategy, scanstrategy.Auto.String(), "") {
-		r.options.ScanStrategy = p.Strategy
-	}
-
-	var httpHelper *input.Helper
-	if opts := engine.ExecuterOptions(); opts != nil {
-		httpHelper = opts.InputHelper
-	}
-
-	var reachFilter func(*templates.Template, *contextargs.MetaInput) bool
-	var techFilterFn func(*templates.Template, *contextargs.MetaInput) bool
-	var techIdx *techReachabilityIndex
-	filteredReqs := baselineReqs
-
-	if p.BuildReachability {
-		idx, groups, err := r.buildHostReachability(finalTemplates, httpHelper, p.ProbePorts)
-		if err != nil {
-			r.Logger.Warning().Msgf("could not build host reachability index, continuing unfiltered: %s", err)
-		} else {
-			baseline, filtered := estimateGroupedExecutions(finalTemplates, groups, hostCount)
-			filteredReqs = filtered
-			saved := baseline - filtered
-			pct := 0.0
-			if baseline > 0 {
-				pct = 100 * float64(saved) / float64(baseline)
-			}
-			for _, g := range groups {
-				r.Logger.Info().Msgf("host-group %s: hosts=%d templates=%d", g.key, len(g.hosts), len(templatesForHostGroup(finalTemplates, g)))
-			}
-			r.Logger.Info().Msgf(
-				"host-groups: groups=%d baseline_exec=%d filtered_exec=%d saved=%d (%.1f%%) mode=single-pass",
-				len(groups), baseline, filtered, saved, pct,
-			)
-			if p.UseReachabilityFilter {
-				reachFilter = idx.Allow
-			}
-		}
-	}
-
-	if p.UseTechFilter {
-		var err error
-		var cache *httprespcache.Cache
-		if opts := engine.ExecuterOptions(); opts != nil {
-			if opts.HTTPResponseCache == nil {
-				opts.HTTPResponseCache = httprespcache.New()
-			}
-			cache = opts.HTTPResponseCache
-		}
-		techIdx, err = r.buildTechReachability(finalTemplates, httpHelper, cache)
-		if err != nil {
-			r.Logger.Warning().Msgf("could not build tech filter index, continuing without tech-filter: %s", err)
-			if cache != nil {
-				cache.Disable()
-			}
-			techIdx = nil
-		} else {
-			base, techFiltered := estimateTechFilteredExecutions(finalTemplates, techIdx, hostCount)
-			// When stacked with reachability, take the more aggressive (lower) estimate
-			// as a lower bound for progress; actual AND filter may skip more.
-			if techFiltered < filteredReqs {
-				filteredReqs = techFiltered
-			}
-			saved := base - techFiltered
-			pct := 0.0
-			if base > 0 {
-				pct = 100 * float64(saved) / float64(base)
-			}
-			r.Logger.Info().Msgf(
-				"tech-filter: baseline_exec=%d filtered_exec=%d saved=%d (%.1f%%) bound_templates=%d",
-				base, techFiltered, saved, pct, techBound,
-			)
-			techFilterFn = techIdx.Allow
-		}
-	} else if opts := engine.ExecuterOptions(); opts != nil && opts.HTTPResponseCache != nil && !r.options.ResponseCache {
-		// -tf set but planner skipped (e.g. no tech-bound templates): do not
-		// alter request paths via an idle response cache. Keep it when the user
-		// asked for -response-cache explicitly.
-		opts.HTTPResponseCache.Disable()
-	}
-
-	p.ApplyFiltered(filteredReqs)
-	if opts := engine.ExecuterOptions(); opts != nil {
-		// The filters narrow the templates per target through the same
-		// TargetScope as per-target profiles, on top of them. The engine then
-		// skips the pairs, counts progress and narrows clusters as it does for
-		// profiles.
-		if filter := composeTemplateFilters(reachFilter, techFilterFn); filter != nil {
-			profiles := opts.TargetScope
-			opts.TargetScope = newPlanScope(profiles, filter, finalTemplates)
-			undo = append(undo, func() { opts.TargetScope = profiles })
-		}
-		if opts.HTTPResponseCache != nil {
-			undo = append(undo, func() {
-				hits, misses, stores := opts.HTTPResponseCache.Stats()
-				r.Logger.Info().Msgf("http-response-cache: hits=%d misses=%d stores=%d", hits, misses, stores)
-				opts.HTTPResponseCache = nil
-			})
-		}
-	}
-
-	r.Logger.Info().Msgf("scan-plan: strategy=%s reach=%v tech=%v expected_req=%d reason=%s",
-		r.options.ScanStrategy, p.UseReachabilityFilter, p.UseTechFilter, p.ExpectedRequests, p.Reason)
-
-	return cleanup
-}
-
 // displayExecutionInfo prints parser stats, version info, and scan counts.
 func (r *Runner) displayExecutionInfo(store *loader.Store) {
 	// Display parser stats for templates loaded into the store.
@@ -1245,6 +1076,7 @@ func (r *Runner) displayExecutionInfo(store *loader.Store) {
 	stats.DisplayAsWarning(httpProtocol.SetThreadToCountZero)
 	stats.ForceDisplayWarning(templates.SkippedUnverifiedTemplateStats)
 	stats.ForceDisplayWarning(templates.SkippedRequestSignatureTemplateStats)
+	stats.ForceDisplayWarning(templates.SkippedWorkflowStepStats)
 
 	cfg := config.DefaultConfig
 

internal/runner/runner_test.go

@@ -6,11 +6,17 @@ import (
 	"net/http/httptest"
 	"os"
 	"path/filepath"
+	"strconv"
+	"sync/atomic"
 	"testing"
 	"time"
 
+	"github.com/miekg/dns"
+	"github.com/projectdiscovery/gologger"
 	"github.com/stretchr/testify/require"
 
+	"github.com/projectdiscovery/nuclei/v3/pkg/input"
+	"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
 	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
@@ -197,3 +203,63 @@ func TestNewInteractshHTTPClient(t *testing.T) {
 	require.NotNil(t, proxyURL)
 	require.Equal(t, "http://127.0.0.1:9", proxyURL.String())
 }
+
+func TestInitializeTemplatesHTTPInputUsesResolvers(t *testing.T) {
+	for _, key := range []string{"HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy"} {
+		t.Setenv(key, "")
+	}
+
+	listener, err := net.ListenPacket("udp", "127.0.0.1:0")
+	require.NoError(t, err)
+	queries := new(atomic.Int32)
+	server := &dns.Server{PacketConn: listener, Handler: dns.HandlerFunc(func(writer dns.ResponseWriter, request *dns.Msg) {
+		queries.Add(1)
+		response := new(dns.Msg).SetReply(request)
+		for _, question := range request.Question {
+			if question.Qtype == dns.TypeA {
+				response.Answer = append(response.Answer, &dns.A{Hdr: dns.RR_Header{Name: question.Name, Rrtype: dns.TypeA, Class: dns.ClassINET}, A: net.ParseIP("127.0.0.1")})
+			}
+		}
+		_ = writer.WriteMsg(response)
+	})}
+	started := make(chan struct{})
+	server.NotifyStartedFunc = func() { close(started) }
+	done := make(chan error, 1)
+	go func() { done <- server.ActivateAndServe() }()
+	<-started
+	t.Cleanup(func() { require.NoError(t, server.Shutdown()); require.NoError(t, <-done) })
+
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	t.Cleanup(srv.Close)
+
+	resolverFile := filepath.Join(t.TempDir(), "resolv.conf")
+	require.NoError(t, os.WriteFile(resolverFile, nil, 0o600))
+	t.Setenv("RESOLVERS_PATH", resolverFile)
+
+	options := &types.Options{
+		ExecutionId:           t.Name(),
+		Timeout:               5,
+		BulkSize:              1,
+		InternalResolversList: []string{listener.LocalAddr().String()},
+	}
+	require.NoError(t, protocolstate.Init(options))
+	t.Cleanup(func() { protocolstate.Close(options.ExecutionId) })
+
+	target := "probe.nuclei.test:" + strconv.Itoa(srv.Listener.Addr().(*net.TCPAddr).Port)
+	r := &Runner{
+		options:       options,
+		inputProvider: provider.NewSimpleInputProviderWithUrls(options.ExecutionId, target),
+		Logger:        &gologger.Logger{},
+	}
+
+	hm, err := r.initializeTemplatesHTTPInput()
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = hm.Close() })
+
+	probed, ok := hm.Get(input.ProbeKey(target, ""))
+	require.True(t, ok, "the probe should resolve the target through the -r resolver")
+	require.Equal(t, "http://"+target, string(probed))
+	require.Positive(t, queries.Load())
+}

internal/runner/sdk_scan_plan.go

@@ -1,40 +0,0 @@
-package runner
-
-import (
-	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v3/pkg/core"
-	"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache"
-	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
-	"github.com/projectdiscovery/nuclei/v3/pkg/types"
-)
-
-// AttachScanPlan applies -strict-probe and -tech-filter to an SDK engine for
-// one scan, the same way the CLI runner does. It returns a func that undoes the
-// per-scan state; call it once the scan is done. When neither option is set it
-// does nothing, so default SDK scans keep their behavior.
-func AttachScanPlan(opts *types.Options, inputProvider provider.InputProvider, logger *gologger.Logger, engine *core.Engine, tpls []*templates.Template) func() {
-	if opts == nil || engine == nil || inputProvider == nil || (!opts.StrictProbe && !opts.TechFilter) {
-		return func() {}
-	}
-	if logger == nil {
-		logger = gologger.DefaultLogger
-	}
-	// The SDK engine is reused across scans, so keep the options and the
-	// response cache it owns as they were before this scan.
-	strategy := opts.ScanStrategy
-	executerOpts := engine.ExecuterOptions()
-	var responseCache *httprespcache.Cache
-	if executerOpts != nil {
-		responseCache = executerOpts.HTTPResponseCache
-	}
-	r := &Runner{options: opts, inputProvider: inputProvider, Logger: logger}
-	cleanup := r.attachScanPlan(engine, tpls)
-	return func() {
-		cleanup()
-		opts.ScanStrategy = strategy
-		if executerOpts != nil {
-			executerOpts.HTTPResponseCache = responseCache
-		}
-	}
-}

internal/runner/sdk_scan_plan_test.go

@@ -1,44 +0,0 @@
-package runner
-
-import (
-	"testing"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/core"
-	"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache"
-	"github.com/projectdiscovery/nuclei/v3/pkg/types"
-	"github.com/stretchr/testify/require"
-)
-
-// Without -strict-probe or -tech-filter the SDK hook changes nothing.
-func TestAttachScanPlanNoop(t *testing.T) {
-	opts := types.DefaultOptions()
-	opts.ScanStrategy = "auto"
-	engine := core.New(opts)
-	engine.SetExecuterOptions(&protocols.ExecutorOptions{Options: opts})
-	ip := provider.NewSimpleInputProviderWithUrls(opts.ExecutionId, "http://127.0.0.1:1")
-
-	AttachScanPlan(opts, ip, nil, engine, nil)()
-	require.Equal(t, "auto", opts.ScanStrategy)
-	require.Nil(t, engine.ExecuterOptions().TargetScope)
-}
-
-// The SDK engine is reused across scans, so the hook must leave its options
-// and its response cache as they were before the scan.
-func TestAttachScanPlanRestoresEngineState(t *testing.T) {
-	opts := types.DefaultOptions()
-	opts.ScanStrategy = "auto"
-	opts.TechFilter = true
-	cache := httprespcache.New()
-	engine := core.New(opts)
-	engine.SetExecuterOptions(&protocols.ExecutorOptions{Options: opts, HTTPResponseCache: cache})
-	ip := provider.NewSimpleInputProviderWithUrls(opts.ExecutionId, "http://127.0.0.1:1")
-
-	cleanup := AttachScanPlan(opts, ip, nil, engine, nil)
-	require.NotEqual(t, "auto", opts.ScanStrategy, "the plan resolves the auto strategy for the scan")
-	cleanup()
-	require.Equal(t, "auto", opts.ScanStrategy)
-	require.Same(t, cache, engine.ExecuterOptions().HTTPResponseCache)
-	require.Nil(t, engine.ExecuterOptions().TargetScope)
-}

internal/runner/tech_filter.go

@@ -1,283 +0,0 @@
-package runner
-
-import (
-	"io"
-	"net/http"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v3/pkg/core/techfilter"
-	"github.com/projectdiscovery/nuclei/v3/pkg/input"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
-	httpProtocol "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache"
-	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
-	"github.com/projectdiscovery/retryablehttp-go"
-	syncutil "github.com/projectdiscovery/utils/sync"
-	unitutils "github.com/projectdiscovery/utils/unit"
-	wappalyzer "github.com/projectdiscovery/wappalyzergo"
-)
-
-const (
-	techFilterWorkers      = 50
-	techFilterMaxBody      = 1 * unitutils.Mega
-	techFilterHTTPTimeout  = 5 * time.Second
-	techFilterMaxRedirects = 5
-)
-
-// techReachabilityIndex filters template×target pairs by product/macro tags.
-type techReachabilityIndex struct {
-	byInput map[string]techfilter.HostProfile // keyed by normalizeInputKey
-
-	// boundTags memoizes template ID -> product tags. It is filled lazily from
-	// the template execution goroutines, so every access needs the mutex.
-	tagsMu    sync.RWMutex
-	boundTags map[string][]string
-}
-
-func normalizeInputKey(raw string) string {
-	return strings.TrimRight(strings.TrimSpace(raw), "/")
-}
-
-// Allow is fail-open: missing profile or unbound template => true.
-func (idx *techReachabilityIndex) Allow(t *templates.Template, mi *contextargs.MetaInput) bool {
-	if idx == nil || t == nil || mi == nil {
-		return true
-	}
-	profile, ok := idx.lookup(mi.Input)
-	if !ok {
-		return true
-	}
-	return techfilter.AllowTags(profile, idx.tagsFor(t))
-}
-
-func (idx *techReachabilityIndex) tagsFor(t *templates.Template) []string {
-	if t == nil {
-		return nil
-	}
-	idx.tagsMu.RLock()
-	tags, ok := idx.boundTags[t.ID]
-	idx.tagsMu.RUnlock()
-	if ok {
-		return tags
-	}
-
-	tags = techfilter.TemplateProductTags(t)
-
-	idx.tagsMu.Lock()
-	defer idx.tagsMu.Unlock()
-	if idx.boundTags == nil {
-		idx.boundTags = make(map[string][]string)
-	}
-	// Another goroutine may have won the race; keep one slice per template ID.
-	if existing, ok := idx.boundTags[t.ID]; ok {
-		return existing
-	}
-	idx.boundTags[t.ID] = tags
-	return tags
-}
-
-func (idx *techReachabilityIndex) lookup(raw string) (techfilter.HostProfile, bool) {
-	if idx == nil || idx.byInput == nil {
-		return techfilter.HostProfile{}, false
-	}
-	if p, ok := idx.byInput[normalizeInputKey(raw)]; ok {
-		return p, true
-	}
-	if p, ok := idx.byInput[raw]; ok {
-		return p, true
-	}
-	return techfilter.HostProfile{}, false
-}
-
-// buildTechReachability fingerprints HTTP(S) targets and builds a tag index.
-// When cache is non-nil, fingerprint GET responses seed it so matching template
-// GETs pay no extra RTT.
-func (r *Runner) buildTechReachability(tpls []*templates.Template, httpHelper *input.Helper, cache *httprespcache.Cache) (*techReachabilityIndex, error) {
-	_ = tpls
-	wapp, err := wappalyzer.New()
-	if err != nil {
-		return nil, err
-	}
-	// Redirects must be followed. A site that answers "/" with a 302 to its real
-	// entry point (WordPress -> /wp-admin/install.php, any app -> /login) returns
-	// an empty body, so the probe would detect nothing and every template bound
-	// to the undetected product would be pruned. Same-host only, so the probe
-	// never leaves the target authority.
-	httpclient, err := httpclientpool.Get(r.options, &httpclientpool.Configuration{
-		DisableCookie:         true,
-		ResponseHeaderTimeout: techFilterHTTPTimeout,
-		RedirectFlow:          httpclientpool.FollowSameHostRedirect,
-		MaxRedirects:          techFilterMaxRedirects,
-	}, "")
-	if err != nil {
-		return nil, err
-	}
-
-	idx := &techReachabilityIndex{
-		byInput:   make(map[string]techfilter.HostProfile),
-		boundTags: make(map[string][]string),
-	}
-	var mu sync.Mutex
-	sg, err := syncutil.New(syncutil.WithSize(techFilterWorkers))
-	if err != nil {
-		return nil, err
-	}
-
-	fingerprinted := 0
-	eligible := 0
-	r.inputProvider.Iterate(func(mi *contextargs.MetaInput) bool {
-		if mi == nil || mi.Input == "" {
-			return true
-		}
-		fpURL := fingerprintURLForInput(mi.Input, httpHelper)
-		if fpURL == "" {
-			return true
-		}
-		eligible++
-		sg.Add()
-		go func(inputKey, url string) {
-			defer sg.Done()
-			profile := fingerprintTarget(wapp, httpclient, url, cache)
-			key := normalizeInputKey(inputKey)
-			mu.Lock()
-			idx.byInput[key] = profile
-			if profile.HasTags() {
-				fingerprinted++
-			}
-			mu.Unlock()
-		}(mi.Input, fpURL)
-		return true
-	})
-	sg.Wait()
-
-	gologger.Info().Msgf("tech-filter: fingerprinted=%d/%d hosts with tags (eligible=%d)", fingerprinted, len(idx.byInput), eligible)
-	return idx, nil
-}
-
-// fingerprintURLForInput returns an absolute http(s) URL to fingerprint, or "".
-// Bare hosts reuse InputsHTTP probed URLs when available.
-func fingerprintURLForInput(raw string, helper *input.Helper) string {
-	if looksLikeHTTPTarget(raw) {
-		return raw
-	}
-	if helper != nil && helper.InputsHTTP != nil {
-		if probed, ok := helper.InputsHTTP.Get(raw); ok && len(probed) > 0 {
-			return string(probed)
-		}
-	}
-	return ""
-}
-
-func looksLikeHTTPTarget(raw string) bool {
-	lower := strings.ToLower(raw)
-	return strings.HasPrefix(lower, "http://") || strings.HasPrefix(lower, "https://")
-}
-
-func fingerprintTarget(wapp *wappalyzer.Wappalyze, client *retryablehttp.Client, rawURL string, cache *httprespcache.Cache) techfilter.HostProfile {
-	// Prefer a trailing-slash form that matches typical {{BaseURL}}/ template URLs.
-	seedURL := rawURL
-	if !strings.HasSuffix(seedURL, "/") {
-		seedURL = seedURL + "/"
-	}
-	req, err := retryablehttp.NewRequest(http.MethodGet, seedURL, nil)
-	if err != nil {
-		return techfilter.HostProfile{}
-	}
-	// Same user agent the templates will send, so the response seeded below is
-	// reachable from the cache instead of being stored under a key nothing hits.
-	req.Header.Set("User-Agent", httpProtocol.StableUserAgent())
-	resp, err := client.Do(req)
-	if err != nil {
-		return techfilter.HostProfile{}
-	}
-	defer func() { _ = resp.Body.Close() }()
-	body, err := io.ReadAll(io.LimitReader(resp.Body, techFilterMaxBody))
-	if err != nil {
-		return techfilter.HostProfile{}
-	}
-	if cache != nil {
-		seedFingerprintResponse(cache, req, resp, body, seedURL)
-	}
-	info := wapp.FingerprintWithInfo(resp.Header, body)
-	products := make(map[string][]string, len(info))
-	for name, app := range info {
-		products[name] = append([]string(nil), app.Categories...)
-	}
-	return techfilter.ProfileFromFingerprint(products)
-}
-
-// seedFingerprintResponse stores the probe response under the URL it actually
-// belongs to, so templates can reuse the RTT.
-//
-// The probe follows redirects, so resp may describe a different URL than the one
-// requested. Seeding it under the requested URL would hand every template that
-// asks for that URL the redirect target's status, headers and body instead of
-// the redirect itself, which silently changes what they match on.
-func seedFingerprintResponse(cache *httprespcache.Cache, req *retryablehttp.Request, resp *http.Response, body []byte, seedURL string) {
-	finalURL := seedURL
-	if resp != nil && resp.Request != nil && resp.Request.URL != nil {
-		finalURL = resp.Request.URL.String()
-	}
-	if finalURL == seedURL {
-		cache.SeedHTTP(req.Request, resp, body, techFilterMaxBody)
-		return
-	}
-	finalReq := req.Request.Clone(req.Context())
-	finalReq.URL = resp.Request.URL
-	finalReq.Host = ""
-	cache.SeedHTTP(finalReq, resp, body, techFilterMaxBody)
-}
-
-// estimateTechFilteredExecutions counts template×host pairs that survive the tech filter.
-func estimateTechFilteredExecutions(tpls []*templates.Template, idx *techReachabilityIndex, hostCount int) (baseline, filtered int) {
-	baseline = estimateTemplateRequests(tpls) * hostCount
-	if idx == nil || len(idx.byInput) == 0 {
-		return baseline, baseline
-	}
-	filtered = 0
-	for _, t := range tpls {
-		reqs := t.TotalRequests
-		if reqs <= 0 {
-			reqs = 1
-		}
-		if len(t.Workflows) > 0 {
-			continue
-		}
-		tags := idx.tagsFor(t)
-		for _, profile := range idx.byInput {
-			if techfilter.AllowTags(profile, tags) {
-				filtered += reqs
-			}
-		}
-		missing := hostCount - len(idx.byInput)
-		if missing > 0 {
-			filtered += reqs * missing
-		}
-	}
-	return baseline, filtered
-}
-
-// composeTemplateFilters ANDs optional filters; nil filters are ignored.
-func composeTemplateFilters(filters ...func(*templates.Template, *contextargs.MetaInput) bool) func(*templates.Template, *contextargs.MetaInput) bool {
-	var active []func(*templates.Template, *contextargs.MetaInput) bool
-	for _, f := range filters {
-		if f != nil {
-			active = append(active, f)
-		}
-	}
-	if len(active) == 0 {
-		return nil
-	}
-	return func(t *templates.Template, mi *contextargs.MetaInput) bool {
-		for _, f := range active {
-			if !f(t, mi) {
-				return false
-			}
-		}
-		return true
-	}
-}

internal/runner/tech_filter_race_test.go

@@ -1,33 +0,0 @@
-package runner
-
-import (
-	"sync"
-	"testing"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/core/techfilter"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
-	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
-)
-
-// tagsFor is filled lazily from the template execution goroutines. Run it under
-// -race to catch the concurrent map access that used to crash whole scans.
-func TestTechReachabilityIndexConcurrentTagsFor(t *testing.T) {
-	idx := &techReachabilityIndex{
-		byInput:   map[string]techfilter.HostProfile{},
-		boundTags: map[string][]string{},
-	}
-
-	var wg sync.WaitGroup
-	for i := 0; i < 64; i++ {
-		wg.Add(1)
-		go func(i int) {
-			defer wg.Done()
-			for j := 0; j < 64; j++ {
-				tpl := &templates.Template{ID: string(rune('a' + (i+j)%26))}
-				_ = idx.tagsFor(tpl)
-				_ = idx.Allow(tpl, &contextargs.MetaInput{Input: "http://x/"})
-			}
-		}(i)
-	}
-	wg.Wait()
-}

internal/runner/tech_filter_seed_test.go

@@ -1,70 +0,0 @@
-package runner
-
-import (
-	"net/http"
-	"net/url"
-	"testing"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache"
-	"github.com/projectdiscovery/retryablehttp-go"
-)
-
-func mustReq(t *testing.T, raw string) *retryablehttp.Request {
-	t.Helper()
-	req, err := retryablehttp.NewRequest(http.MethodGet, raw, nil)
-	if err != nil {
-		t.Fatalf("request: %v", err)
-	}
-	return req
-}
-
-func respFor(raw string, status int) *http.Response {
-	u, _ := url.Parse(raw)
-	return &http.Response{
-		StatusCode: status,
-		Status:     http.StatusText(status),
-		Header:     http.Header{"Server": []string{"probe"}},
-		Request:    &http.Request{Method: http.MethodGet, URL: u},
-	}
-}
-
-// The probe follows redirects. The body it comes back with belongs to the
-// redirect target, so it must never be stored under the requested URL: a
-// template asking for "/" must still see the redirect, not the target page.
-func TestSeedFingerprintResponseAfterRedirect(t *testing.T) {
-	const seedURL = "http://example.test/"
-	const finalURL = "http://example.test/wp-admin/install.php"
-
-	c := httprespcache.New()
-	req := mustReq(t, seedURL)
-	req.Header.Set("User-Agent", "probe-ua")
-	seedFingerprintResponse(c, req, respFor(finalURL, 200), []byte("install page"), seedURL)
-
-	lookup := req.Request.Clone(req.Context())
-	if got := c.Get(httprespcache.KeyFromRequest(lookup), "template", lookup); got != nil {
-		t.Fatal("the redirect target must not be cached under the requested URL")
-	}
-
-	finalLookup := req.Request.Clone(req.Context())
-	finalLookup.URL, _ = url.Parse(finalURL)
-	finalLookup.Host = ""
-	if got := c.Get(httprespcache.KeyFromRequest(finalLookup), "template", finalLookup); got == nil {
-		t.Fatal("the response must be cached under the URL it belongs to")
-	}
-}
-
-// Without a redirect the response does describe the requested URL, so seeding
-// it there is what saves the round trip.
-func TestSeedFingerprintResponseWithoutRedirect(t *testing.T) {
-	const seedURL = "http://example.test/"
-
-	c := httprespcache.New()
-	req := mustReq(t, seedURL)
-	req.Header.Set("User-Agent", "probe-ua")
-	seedFingerprintResponse(c, req, respFor(seedURL, 200), []byte("home"), seedURL)
-
-	lookup := req.Request.Clone(req.Context())
-	if got := c.Get(httprespcache.KeyFromRequest(lookup), "template", lookup); got == nil {
-		t.Fatal("a non-redirected response must be cached under the requested URL")
-	}
-}

internal/runner/tech_filter_test.go

@@ -1,21 +0,0 @@
-package runner
-
-import (
-	"testing"
-
-	"github.com/projectdiscovery/hmap/store/hybrid"
-	"github.com/projectdiscovery/nuclei/v3/pkg/input"
-	"github.com/stretchr/testify/require"
-)
-
-func TestFingerprintURLForInput(t *testing.T) {
-	require.Equal(t, "http://a/", fingerprintURLForInput("http://a/", nil))
-	require.Equal(t, "", fingerprintURLForInput("example.com", nil))
-
-	hm, err := hybrid.New(hybrid.DefaultDiskOptions)
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = hm.Close() })
-	require.NoError(t, hm.Set("example.com", []byte("https://example.com")))
-	helper := &input.Helper{InputsHTTP: hm, InputsHTTPProbed: true}
-	require.Equal(t, "https://example.com", fingerprintURLForInput("example.com", helper))
-}

internal/runner/templates.go

@@ -10,9 +10,11 @@ import (
 	"github.com/logrusorgru/aurora/v4"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
+	"github.com/projectdiscovery/nuclei/v3/pkg/model"
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	"github.com/projectdiscovery/nuclei/v3/pkg/utils"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
 )
 
@@ -38,12 +40,17 @@ func (r *Runner) verboseTemplate(tpl *templates.Template) {
 		tpl.Info.SeverityHolder.Severity))
 }
 
+// listedTemplate names its fields like a result event, so a -jsonl listing
+// joins with scan results on the same keys.
+type listedTemplate struct {
+	Template     string     `json:"template,omitempty"`
+	TemplateID   string     `json:"template-id"`
+	TemplatePath string     `json:"template-path"`
+	Info         model.Info `json:"info"`
+}
+
 func (r *Runner) listAvailableStoreTemplates(store *loader.Store) {
-	r.Logger.Print().Msgf(
-		"\nListing available %v nuclei templates for %v",
-		config.DefaultConfig.TemplateVersion,
-		config.DefaultConfig.TemplatesDirectory,
-	)
+	r.listingHeader("templates")
 	// order templates alphabetically by path
 	templates := store.Templates()
 	sort.Slice(templates, func(i, j int) bool {
@@ -51,6 +58,10 @@ func (r *Runner) listAvailableStoreTemplates(store *loader.Store) {
 	})
 
 	for _, tpl := range templates {
+		if r.options.JSONL && !r.options.TemplateDisplay {
+			r.listTemplateJSON(tpl)
+			continue
+		}
 		if hasExtraFlags(r.options) {
 			if r.options.TemplateDisplay {
 				colorize := !r.options.NoColor
@@ -78,12 +89,36 @@ func (r *Runner) listAvailableStoreTemplates(store *loader.Store) {
 	}
 }
 
-func (r *Runner) listAvailableTags(tagsMap map[string]int) {
+func (r *Runner) listTemplateJSON(tpl *templates.Template) {
+	path, _ := utils.TemplatePathURL(tpl.Path, tpl.ID, "")
+	marshalled, err := json.Marshal(listedTemplate{
+		Template:     path,
+		TemplateID:   tpl.ID,
+		TemplatePath: tpl.Path,
+		Info:         tpl.Info,
+	})
+	if err != nil {
+		r.Logger.Error().Msgf("Could not marshal the template %s: %s", tpl.Path, err)
+		return
+	}
+	r.Logger.Print().Msgf("%s", marshalled)
+}
+
+// listingHeader is skipped for -jsonl, where every line must parse as JSON.
+func (r *Runner) listingHeader(what string) {
+	if r.options.JSONL {
+		return
+	}
 	r.Logger.Print().Msgf(
-		"\nListing available %v nuclei tags for %v",
+		"\nListing available %v nuclei %s for %v",
 		config.DefaultConfig.TemplateVersion,
+		what,
 		config.DefaultConfig.TemplatesDirectory,
 	)
+}
+
+func (r *Runner) listAvailableTags(tagsMap map[string]int) {
+	r.listingHeader("tags")
 
 	type kv struct {
 		Key   string `json:"tag"`

internal/runner/templates_test.go

@@ -0,0 +1,76 @@
+package runner
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/gologger/formatter"
+	"github.com/projectdiscovery/gologger/levels"
+	"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
+	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
+	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk"
+	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
+	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
+	"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
+	"github.com/stretchr/testify/require"
+)
+
+const listedTemplateYAML = `id: listed-template
+info:
+  name: Listed Template
+  author: pdteam
+  severity: info
+  tags: listing
+http:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+`
+
+func TestTemplateListWritesJSONLines(t *testing.T) {
+	root := t.TempDir()
+	cfg := config.DefaultConfig
+	oldRoot := cfg.TemplatesDirectory
+	t.Cleanup(func() { cfg.SetTemplatesDir(oldRoot) })
+	cfg.SetTemplatesDir(root)
+
+	path := filepath.Join(root, "http", "listed-template.yaml")
+	require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755))
+	require.NoError(t, os.WriteFile(path, []byte(listedTemplateYAML), 0o600))
+
+	writer := &ignoreFileLogWriter{}
+	logger := &gologger.Logger{}
+	logger.SetFormatter(formatter.NewCLI(true))
+	logger.SetWriter(writer)
+	logger.SetMaxLevel(levels.LevelInfo)
+
+	options := testutils.DefaultOptions.Copy()
+	options.ExecutionId = t.Name()
+	options.Logger = logger
+	options.Templates = []string{path}
+	options.TemplateList = true
+	options.JSONL = true
+	testutils.Init(options)
+	t.Cleanup(func() { testutils.Cleanup(options) })
+
+	executerOpts := testutils.NewMockExecuterOptions(options, nil)
+	executerOpts.Parser = templates.NewParser()
+	store, err := loader.New(loader.NewConfig(options, disk.NewCatalog(root), executerOpts))
+	require.NoError(t, err)
+	require.NoError(t, store.LoadTemplatesOnlyMetadata())
+
+	require.Len(t, store.Templates(), 1)
+
+	r := &Runner{options: options, Logger: logger}
+	r.listAvailableStoreTemplates(store)
+
+	require.Len(t, writer.records, 1, "JSONL output carries no banner line")
+	var listed map[string]any
+	require.NoError(t, json.Unmarshal([]byte(strings.TrimSpace(writer.records[0].message)), &listed))
+	require.Equal(t, "listed-template", listed["template-id"])
+	require.Equal(t, filepath.Join("http", "listed-template.yaml"), listed["template"])
+	require.Equal(t, "Listed Template", listed["info"].(map[string]any)["name"])
+}

internal/server/nuclei_sdk.go

@@ -33,6 +33,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/excludematchers"
 	browserEngine "github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/catchall"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
@@ -110,6 +111,9 @@ func newNucleiExecutor(opts *NucleiExecutorOptions) (*nucleiExecutor, error) {
 	if opts.Options.HostBackoff {
 		executorOpts.HostBackoff = hostbackoff.New(hostbackoff.Config{Max: opts.Options.HostBackoffMax})
 	}
+	if opts.Options.AutoCalibrate {
+		executorOpts.CatchAll = catchall.New(0)
+	}
 
 	executorEngine := core.New(opts.Options)
 	executorEngine.SetExecuterOptions(executorOpts)

internal/tests/integration/http_test.go

@@ -102,6 +102,7 @@ var httpTestcases = []integrationCase{
 	{Path: "protocols/http/raw-unsafe-path-single-slash.yaml", TestCase: &httpRawUnsafePathSingleSlash{}},
 	{Path: "protocols/http/disable-http-cache.yaml", TestCase: &httpDisableCache{}},
 	{Path: "protocols/http/http-cache.yaml", TestCase: &httpCache{}},
+	{Path: "protocols/http/get.yaml", TestCase: &httpExcludeHostsFileComments{}},
 }
 
 type httpMultiVarSharing struct{}
@@ -1983,3 +1984,59 @@ func (h *httpDisableCache) Execute(filePath string) error {
 
 	return nil
 }
+
+// ensure that comments in an -exclude-hosts file are ignored
+type httpExcludeHostsFileComments struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *httpExcludeHostsFileComments) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		_, _ = fmt.Fprintf(w, "This is test matcher text")
+	})
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	excludeHosts := func(content string) ([]string, error) {
+		file, err := os.CreateTemp("", "exclude-hosts-*.txt")
+		if err != nil {
+			return nil, err
+		}
+		defer func() { _ = os.Remove(file.Name()) }()
+		_, err = file.WriteString(content)
+		_ = file.Close()
+		if err != nil {
+			return nil, err
+		}
+
+		return testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-eh", file.Name())
+	}
+
+	// a comment line is not a host, even when it reads as an invalid regex
+	results, err := excludeHosts("# hosts out of scope (see ticket\n203.0.113.10\n")
+	if err != nil {
+		return fmt.Errorf("comment line: %w", err)
+	}
+	if err := expectResultsCount(results, 1); err != nil {
+		return fmt.Errorf("comment line: %w", err)
+	}
+
+	// a trailing comment does not stop the host from being excluded
+	results, err = excludeHosts("127.0.0.1 # test server\n")
+	if err != nil {
+		return fmt.Errorf("trailing comment: %w", err)
+	}
+	if err := expectResultsCount(results, 0); err != nil {
+		return fmt.Errorf("trailing comment: %w", err)
+	}
+
+	// comma separated hosts are still split when passed on the command line
+	results, err = testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-eh", "203.0.113.10, 127.0.0.1")
+	if err != nil {
+		return fmt.Errorf("comma separated: %w", err)
+	}
+	if err := expectResultsCount(results, 0); err != nil {
+		return fmt.Errorf("comma separated: %w", err)
+	}
+	return nil
+}

lib/config.go

@@ -173,16 +173,6 @@ func WithResponseReadSize(responseReadSize int) NucleiSDKOptions {
 	}
 }
 
-// EnableResponseCache reuses identical safe GET/HEAD responses within one scan.
-// It is off by default: a template that depends on state or timing could see a
-// different result.
-func EnableResponseCache() NucleiSDKOptions {
-	return func(e *NucleiEngine) error {
-		e.opts.ResponseCache = true
-		return nil
-	}
-}
-
 // WithGlobalRateLimit sets global rate (i.e all hosts combined) limit options
 // Deprecated: will be removed in favour of WithGlobalRateLimitCtx in next release
 func WithGlobalRateLimit(maxTokens int, duration time.Duration) NucleiSDKOptions {
@@ -766,31 +756,3 @@ func WithTemporaryDirectory(parentDir string) NucleiSDKOptions {
 		return nil
 	}
 }
-
-// WithPerHostRateLimit applies the rate limit per host instead of globally.
-func WithPerHostRateLimit(enabled bool) NucleiSDKOptions {
-	return func(e *NucleiEngine) error {
-		e.opts.PerHostRateLimit = enabled
-		return nil
-	}
-}
-
-// EnableStrictProbe skips templates whose target service is unreachable: web
-// templates on hosts that do not speak HTTP and network templates on closed
-// ports. It is the SDK form of -strict-probe.
-func EnableStrictProbe() NucleiSDKOptions {
-	return func(e *NucleiEngine) error {
-		e.opts.StrictProbe = true
-		return nil
-	}
-}
-
-// EnableTechFilter fingerprints HTTP targets and skips product-bound templates
-// that cannot match them. It also turns on the response cache, which the
-// fingerprint requests seed. It is the SDK form of -tech-filter.
-func EnableTechFilter() NucleiSDKOptions {
-	return func(e *NucleiEngine) error {
-		e.opts.TechFilter = true
-		return nil
-	}
-}

lib/multi.go

@@ -5,7 +5,6 @@ import (
 	"time"
 
 	"github.com/logrusorgru/aurora/v4"
-	"github.com/projectdiscovery/nuclei/v3/internal/runner"
 	"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
 	"github.com/projectdiscovery/nuclei/v3/pkg/core"
@@ -13,7 +12,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/loader/workflow"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils"
 	"github.com/projectdiscovery/utils/errkit"
@@ -62,11 +60,6 @@ func createEphemeralObjects(ctx context.Context, base *NucleiEngine, opts *types
 		opts.RateLimitDuration = time.Second
 	}
 	u.executerOpts.RateLimiter = utils.GetRateLimiter(ctx, opts.RateLimit, opts.RateLimitDuration)
-	// Each execution gets its own response cache, created before its
-	// templates compile.
-	if opts.UsesResponseCache() && u.executerOpts.HTTPResponseCache == nil {
-		u.executerOpts.HTTPResponseCache = httprespcache.New()
-	}
 	return u, nil
 }
 
@@ -247,7 +240,6 @@ func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOptsCtx(ctx context.Context, t
 	engine := core.New(tmpEngine.opts)
 	engine.SetExecuterOptions(unsafeOpts.executerOpts)
 
-	defer runner.AttachScanPlan(tmpEngine.opts, inputProvider, tmpEngine.opts.Logger, engine, store.Templates())()
 	_ = engine.ExecuteScanWithOpts(ctx, store.Templates(), inputProvider, false)
 
 	engine.WorkPool().Wait()

lib/scan_plan_test.go

@@ -1,120 +0,0 @@
-package nuclei_test
-
-import (
-	"context"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"path/filepath"
-	"sync"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	nuclei "github.com/projectdiscovery/nuclei/v3/lib"
-	"github.com/projectdiscovery/nuclei/v3/pkg/output"
-	"github.com/stretchr/testify/require"
-)
-
-func TestScanPlanOptions(t *testing.T) {
-	ne, err := nuclei.NewNucleiEngineCtx(context.Background(),
-		nuclei.DisableUpdateCheck(),
-		nuclei.EnableStrictProbe(),
-		nuclei.EnableTechFilter(),
-		nuclei.EnableResponseCache(),
-	)
-	require.NoError(t, err)
-	defer ne.Close()
-
-	opts := ne.Options()
-	require.True(t, opts.StrictProbe)
-	require.True(t, opts.TechFilter)
-	require.True(t, opts.ResponseCache)
-}
-
-// sdkCacheTemplates writes two templates that both GET /same. The second also
-// GETs /other, so request clustering cannot merge them into one request.
-func sdkCacheTemplates(t *testing.T) []string {
-	t.Helper()
-	dir := t.TempDir()
-	a := `id: sdk-cache-a
-info:
-  name: SDK Cache A
-  author: test
-  severity: info
-http:
-  - method: GET
-    path:
-      - "{{BaseURL}}/same"
-    matchers:
-      - type: word
-        words:
-          - "cached-body"
-`
-	b := `id: sdk-cache-b
-info:
-  name: SDK Cache B
-  author: test
-  severity: info
-http:
-  - method: GET
-    path:
-      - "{{BaseURL}}/same"
-      - "{{BaseURL}}/other"
-    matchers:
-      - type: word
-        words:
-          - "cached-body"
-`
-	pa, pb := filepath.Join(dir, "a.yaml"), filepath.Join(dir, "b.yaml")
-	require.NoError(t, os.WriteFile(pa, []byte(a), 0o600))
-	require.NoError(t, os.WriteFile(pb, []byte(b), 0o600))
-	return []string{pa, pb}
-}
-
-// With the SDK options for the scan plan set, both templates still run and
-// match, and the response cache sends /same once. Without them it is sent
-// twice.
-func TestSDKScanPlanAndResponseCache(t *testing.T) {
-	var sameHits atomic.Int32
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/same" {
-			sameHits.Add(1)
-			time.Sleep(200 * time.Millisecond)
-		}
-		_, _ = w.Write([]byte("cached-body"))
-	}))
-	defer ts.Close()
-	tpls := sdkCacheTemplates(t)
-
-	run := func(extra ...nuclei.NucleiSDKOptions) map[string]bool {
-		sameHits.Store(0)
-		opts := append([]nuclei.NucleiSDKOptions{
-			nuclei.DisableUpdateCheck(),
-			nuclei.WithTemplatesOrWorkflows(nuclei.TemplateSources{Templates: tpls}),
-		}, extra...)
-		ne, err := nuclei.NewNucleiEngineCtx(context.Background(), opts...)
-		require.NoError(t, err)
-		defer ne.Close()
-		ne.Options().RestrictLocalNetworkAccess = false
-		ne.Options().Retries = 0
-		ne.LoadTargets([]string{ts.URL}, false)
-
-		var mu sync.Mutex
-		matched := map[string]bool{}
-		require.NoError(t, ne.ExecuteCallbackWithCtx(context.Background(), func(event *output.ResultEvent) {
-			mu.Lock()
-			defer mu.Unlock()
-			matched[event.TemplateID] = true
-		}))
-		return matched
-	}
-
-	matched := run(nuclei.EnableResponseCache(), nuclei.EnableStrictProbe(), nuclei.EnableTechFilter())
-	require.Equal(t, map[string]bool{"sdk-cache-a": true, "sdk-cache-b": true}, matched)
-	require.Equal(t, int32(1), sameHits.Load(), "the response cache must send /same once")
-
-	matched = run()
-	require.Equal(t, map[string]bool{"sdk-cache-a": true, "sdk-cache-b": true}, matched)
-	require.Equal(t, int32(2), sameHits.Load(), "without the cache /same is sent by each template")
-}

lib/sdk.go

@@ -9,7 +9,6 @@ import (
 	"sync"
 
 	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/nuclei/v3/internal/runner"
 	"github.com/projectdiscovery/nuclei/v3/pkg/authprovider"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog"
 	metadataindex "github.com/projectdiscovery/nuclei/v3/pkg/catalog/index"
@@ -332,10 +331,6 @@ func (e *NucleiEngine) ExecuteCallbackWithCtx(ctx context.Context, callback ...f
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		// A response cached by an earlier scan of this engine is never served
-		// to this one.
-		e.executerOpts.HTTPResponseCache.Reset()
-		defer runner.AttachScanPlan(e.opts, e.inputProvider, e.opts.Logger, e.engine, templatesAndWorkflows)()
 		_ = e.engine.ExecuteScanWithOpts(ctx, templatesAndWorkflows, e.inputProvider, false)
 	}()
 

lib/sdk_private.go

@@ -33,7 +33,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	nucleiUtils "github.com/projectdiscovery/nuclei/v3/pkg/utils"
@@ -354,17 +353,14 @@ func (e *NucleiEngine) init(ctx context.Context) error {
 		e.executerOpts.Options.Parser = op
 	}
 
-	// Templates compile against e.executerOpts, so the response cache must
-	// exist before they load. Each scan resets it.
-	if e.opts.UsesResponseCache() && e.executerOpts.HTTPResponseCache == nil {
-		e.executerOpts.HTTPResponseCache = httprespcache.New()
-	}
-
 	e.engine = core.New(e.opts)
 	e.engine.SetExecuterOptions(e.executerOpts)
 
 	httpxOptions := httpx.DefaultOptions
 	httpxOptions.Timeout = 5 * time.Second
+	if len(e.opts.InternalResolversList) > 0 {
+		httpxOptions.Resolvers = e.opts.InternalResolversList
+	}
 	if client, err := httpx.New(&httpxOptions); err != nil {
 		return err
 	} else {

pkg/catalog/loader/loader.go

@@ -631,10 +631,10 @@ func (store *Store) noteMissingCapabilities(templatePath string, missingCaps []t
 // erroring out on discovering any faulty templates.
 func (store *Store) ValidateTemplates() error {
 	templatePaths, errs := store.config.Catalog.GetTemplatesPath(store.finalTemplates)
-	store.logErroredTemplates(errs)
+	notFound := store.logErroredTemplates(errs)
 
 	workflowPaths, errs := store.config.Catalog.GetTemplatesPath(store.finalWorkflows)
-	store.logErroredTemplates(errs)
+	notFound += store.logErroredTemplates(errs)
 
 	templatePathsMap := make(map[string]struct{}, len(templatePaths))
 	for _, path := range templatePaths {
@@ -646,7 +646,8 @@ func (store *Store) ValidateTemplates() error {
 		workflowPathsMap[path] = struct{}{}
 	}
 
-	if store.areTemplatesValid(templatePathsMap) && store.areWorkflowsValid(workflowPathsMap) {
+	// a path that can't be resolved is not validated, so it can't count as valid
+	if store.areTemplatesValid(templatePathsMap) && store.areWorkflowsValid(workflowPathsMap) && notFound == 0 {
 		return nil
 	}
 
@@ -1056,10 +1057,15 @@ func workflowContainsProtocol(workflow []*workflows.WorkflowTemplate) bool {
 	return false
 }
 
-func (s *Store) logErroredTemplates(erred map[string]error) {
+// logErroredTemplates logs the templates that could not be found and returns
+// how many of them were not handled by the NotFoundCallback.
+func (s *Store) logErroredTemplates(erred map[string]error) int {
+	notFound := 0
 	for template, err := range erred {
 		if s.NotFoundCallback == nil || !s.NotFoundCallback(template) {
 			s.logger.Error().Msgf("Could not find template '%s': %s", template, err)
+			notFound++
 		}
 	}
+	return notFound
 }

pkg/catalog/loader/loader_test.go

@@ -176,25 +176,8 @@ func TestLoadTemplatesOnlyMetadataLogsCachedTemplateParseErrors(t *testing.T) {
 	require.Contains(t, output.String(), "Could not load template")
 }
 
-func TestValidateTemplatesRejectsRuntimeCompilationError(t *testing.T) {
-	templatePath := filepath.Join(t.TempDir(), "invalid-regex-group.yaml")
-	require.NoError(t, os.WriteFile(templatePath, []byte(`id: invalid-regex-group
-
-info:
-  name: Invalid Regex Group
-  author: pdteam
-  severity: info
-
-http:
-  - method: GET
-    path:
-      - "{{BaseURL}}"
-    extractors:
-      - type: regex
-        group: -1
-        regex:
-          - "(a)(b)"
-`), 0o600))
+func newValidateStore(t *testing.T, executionID, templatePath string) (*Store, *bytes.Buffer) {
+	t.Helper()
 
 	options := testutils.DefaultOptions.Copy()
 	var output bytes.Buffer
@@ -203,7 +186,7 @@ http:
 	logger.SetWriter(&utils.CaptureWriter{Buffer: &output})
 	logger.SetMaxLevel(levels.LevelDebug)
 	options.Logger = logger
-	options.ExecutionId = "loader-invalid-regex-group"
+	options.ExecutionId = executionID
 	options.Templates = []string{templatePath}
 	options.Validate = true
 	options.TemplateLoadingConcurrency = 1
@@ -226,10 +209,50 @@ http:
 
 	store, err := New(NewConfig(options, catalog, executerOpts))
 	require.NoError(t, err)
+	return store, &output
+}
+
+func TestValidateTemplatesRejectsRuntimeCompilationError(t *testing.T) {
+	templatePath := filepath.Join(t.TempDir(), "invalid-regex-group.yaml")
+	require.NoError(t, os.WriteFile(templatePath, []byte(`id: invalid-regex-group
+
+info:
+  name: Invalid Regex Group
+  author: pdteam
+  severity: info
+
+http:
+  - method: GET
+    path:
+      - "{{BaseURL}}"
+    extractors:
+      - type: regex
+        group: -1
+        regex:
+          - "(a)(b)"
+`), 0o600))
+
+	store, output := newValidateStore(t, "loader-invalid-regex-group", templatePath)
 	require.Error(t, store.ValidateTemplates())
 	require.Contains(t, output.String(), "regex extractor group must be >= 0")
 }
 
+func TestValidateTemplatesRejectsMissingTemplatePath(t *testing.T) {
+	templatePath := filepath.Join(t.TempDir(), "no-such-templates")
+
+	store, output := newValidateStore(t, "loader-missing-template-path", templatePath)
+	require.Error(t, store.ValidateTemplates())
+	require.Contains(t, output.String(), "Could not find template")
+}
+
+func TestValidateTemplatesAcceptsMissingPathHandledByCallback(t *testing.T) {
+	templatePath := filepath.Join(t.TempDir(), "no-such-templates")
+
+	store, _ := newValidateStore(t, "loader-missing-path-callback", templatePath)
+	store.NotFoundCallback = func(string) bool { return true }
+	require.NoError(t, store.ValidateTemplates())
+}
+
 func TestRemoteTemplates(t *testing.T) {
 	catalog := disk.NewCatalog("")
 

pkg/core/execute_options.go

@@ -86,7 +86,8 @@ func (e *Engine) ExecuteScanWithOpts(ctx context.Context, templatesList []*templ
 	}
 
 	if stringsutil.EqualFoldAny(e.options.ScanStrategy, scanstrategy.Auto.String(), "") {
-		// Last-resort fallback: runner scan planner normally resolves auto.
+		// TODO: this is only a placeholder, auto scan strategy should choose scan strategy
+		// based on no of hosts , templates , stream and other optimization parameters
 		e.options.ScanStrategy = scanstrategy.TemplateSpray.String()
 	}
 

pkg/core/plan/plan.go

@@ -1,165 +0,0 @@
-package plan
-
-import (
-	"fmt"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/types/scanstrategy"
-)
-
-// Input is the scan shape the planner uses to pick strategy and reachability.
-type Input struct {
-	Hosts       int
-	Templates   int
-	Requests    int // template×host request estimate (baseline)
-	BulkSize    int
-	Stream      bool
-	StrictProbe bool
-
-	// Reachability / probe budget (known before or after probing).
-	PortsToProbe             int
-	ConcreteNetworkTemplates int
-	// FilteredRequests, when > 0, is the post-filter request estimate the plan reports.
-	FilteredRequests int
-	// GroupCount is distinct host signatures after probing (0 = unknown).
-	GroupCount int
-
-	// TechFilter enables product/macro tag filtering after cheap HTTP fingerprinting.
-	TechFilter bool
-	// TechBoundTemplates is how many templates carry product/macro tags.
-	TechBoundTemplates int
-}
-
-// Plan is the execution plan for a single scan pass.
-type Plan struct {
-	Strategy              string
-	BuildReachability     bool
-	ProbePorts            bool
-	UseReachabilityFilter bool
-	UseTechFilter         bool
-	ExpectedRequests      int64
-	Reason                string
-}
-
-// Decide chooses spray strategy and whether to spend budget on reachability probing.
-//
-// Hard bar: never choose work that is expected to regress wall clock vs baseline spray.
-// When probe cost would dominate and savings are uncertain, reachability is skipped.
-func Decide(in Input) Plan {
-	p := Plan{
-		Strategy:         chooseStrategy(in),
-		ExpectedRequests: int64(max(in.Requests, 0)),
-	}
-
-	reachReason := "reachability=off"
-	if in.StrictProbe {
-		reach := decideReachability(in)
-		p.BuildReachability = reach.build
-		p.ProbePorts = reach.probePorts
-		p.UseReachabilityFilter = reach.useFilter
-		reachReason = reach.reason
-	}
-
-	tech := decideTechFilter(in)
-	p.UseTechFilter = tech.use
-	p.Reason = strategyReason(in, p.Strategy) + "; " + reachReason + "; " + tech.reason
-
-	if in.FilteredRequests > 0 {
-		p.ExpectedRequests = int64(in.FilteredRequests)
-	}
-	return p
-}
-
-func chooseStrategy(in Input) string {
-	if in.Stream || in.Hosts <= 0 || in.Templates <= 0 {
-		return scanstrategy.TemplateSpray.String()
-	}
-	bulk := in.BulkSize
-	if bulk <= 0 {
-		bulk = 25
-	}
-	// Host-spray only when the shape strongly favors per-host locality.
-	// Conservative default is template-spray (historical auto) so wall clock
-	// does not regress on small / mixed benches.
-	if in.Hosts <= bulk && in.Hosts <= 10 && in.Templates >= 10*in.Hosts && in.Requests >= 5000 {
-		return scanstrategy.HostSpray.String()
-	}
-	return scanstrategy.TemplateSpray.String()
-}
-
-func strategyReason(in Input, strategy string) string {
-	return fmt.Sprintf("strategy=%s hosts=%d templates=%d", strategy, in.Hosts, in.Templates)
-}
-
-type reachabilityDecision struct {
-	build      bool
-	probePorts bool
-	useFilter  bool
-	reason     string
-}
-
-func decideReachability(in Input) reachabilityDecision {
-	if in.Hosts <= 0 || in.Templates <= 0 {
-		return reachabilityDecision{reason: "reachability=skipped-empty"}
-	}
-
-	probeCost := in.PortsToProbe * in.Hosts
-	baseline := in.Requests
-	if baseline <= 0 {
-		baseline = in.Templates * in.Hosts
-	}
-
-	// No concrete network ports to probe: HTTP-only index is cheap (reuse httpx map).
-	if in.PortsToProbe == 0 || in.ConcreteNetworkTemplates == 0 {
-		return reachabilityDecision{
-			build:      true,
-			probePorts: false,
-			useFilter:  true,
-			reason:     "reachability=http-only",
-		}
-	}
-
-	// Probe budget: if dialing every port×host costs more than the scan itself
-	// and we have no signal of a mixed fleet, skip probing to match baseline time.
-	if probeCost > baseline && in.GroupCount <= 1 {
-		// GroupCount==0 (unknown) or 1 (homogeneous): do not pay for probes.
-		if probeCost > max(baseline*2, 500) {
-			return reachabilityDecision{
-				reason: fmt.Sprintf("reachability=skipped-probe-budget cost=%d baseline=%d", probeCost, baseline),
-			}
-		}
-	}
-
-	// After probing, homogeneous fleets still benefit from a cheap filter (no extra dials
-	// during spray for closed-port templates that global prune may have kept).
-	return reachabilityDecision{
-		build:      true,
-		probePorts: true,
-		useFilter:  true,
-		reason:     fmt.Sprintf("reachability=full probe_cost=%d", probeCost),
-	}
-}
-
-type techFilterDecision struct {
-	use    bool
-	reason string
-}
-
-func decideTechFilter(in Input) techFilterDecision {
-	// Opt-in only: never enable from strategy/reachability alone.
-	if !in.TechFilter {
-		return techFilterDecision{reason: "tech-filter=off"}
-	}
-	// Skip fingerprinting when nothing is tech-bound — otherwise we add
-	// probe latency with zero savings (wall-clock regression).
-	if in.TechBoundTemplates <= 0 {
-		return techFilterDecision{reason: "tech-filter=skipped-no-bound-templates"}
-	}
-	return techFilterDecision{use: true, reason: fmt.Sprintf("tech-filter=on bound=%d", in.TechBoundTemplates)}
-}
-
-// ApplyFiltered updates ExpectedRequests once post-probe estimates are known.
-func (p *Plan) ApplyFiltered(filtered int) {
-	if filtered > 0 {
-		p.ExpectedRequests = int64(filtered)
-	}
-}

pkg/core/plan/plan_test.go

@@ -1,205 +0,0 @@
-package plan
-
-import (
-	"testing"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/types/scanstrategy"
-	"github.com/stretchr/testify/require"
-)
-
-func TestDecideStrategyHostSpray(t *testing.T) {
-	p := Decide(Input{
-		Hosts:     5,
-		Templates: 200,
-		Requests:  10000,
-		BulkSize:  25,
-	})
-	require.Equal(t, scanstrategy.HostSpray.String(), p.Strategy)
-	require.Contains(t, p.Reason, "strategy=host-spray")
-}
-
-func TestDecideStrategyTemplateSpraySmallBench(t *testing.T) {
-	p := Decide(Input{
-		Hosts:     5,
-		Templates: 100,
-		Requests:  500,
-		BulkSize:  25,
-	})
-	require.Equal(t, scanstrategy.TemplateSpray.String(), p.Strategy)
-}
-
-func TestDecideStrategyTemplateSprayManyHosts(t *testing.T) {
-	p := Decide(Input{
-		Hosts:     1000,
-		Templates: 50,
-		Requests:  50000,
-		BulkSize:  25,
-	})
-	require.Equal(t, scanstrategy.TemplateSpray.String(), p.Strategy)
-}
-
-func TestDecideStrategyStreamDefaultsTemplateSpray(t *testing.T) {
-	p := Decide(Input{
-		Hosts:     5,
-		Templates: 100,
-		Stream:    true,
-		BulkSize:  25,
-	})
-	require.Equal(t, scanstrategy.TemplateSpray.String(), p.Strategy)
-}
-
-func TestDecideReachabilityOffWithoutStrictProbe(t *testing.T) {
-	p := Decide(Input{
-		Hosts:        5,
-		Templates:    60,
-		Requests:     300,
-		StrictProbe:  false,
-		PortsToProbe: 3,
-	})
-	require.False(t, p.BuildReachability)
-	require.False(t, p.UseReachabilityFilter)
-	require.Contains(t, p.Reason, "reachability=off")
-}
-
-func TestDecideReachabilityHTTPOnly(t *testing.T) {
-	p := Decide(Input{
-		Hosts:                    5,
-		Templates:                60,
-		Requests:                 300,
-		StrictProbe:              true,
-		PortsToProbe:             0,
-		ConcreteNetworkTemplates: 0,
-	})
-	require.True(t, p.BuildReachability)
-	require.False(t, p.ProbePorts)
-	require.True(t, p.UseReachabilityFilter)
-	require.Contains(t, p.Reason, "reachability=http-only")
-}
-
-func TestDecideReachabilityProbeBudgetSkip(t *testing.T) {
-	p := Decide(Input{
-		Hosts:                    100,
-		Templates:                10,
-		Requests:                 1000,
-		StrictProbe:              true,
-		PortsToProbe:             50, // cost = 5000 > 2*baseline
-		ConcreteNetworkTemplates: 5,
-		GroupCount:               0,
-	})
-	require.False(t, p.BuildReachability)
-	require.False(t, p.UseReachabilityFilter)
-	require.Contains(t, p.Reason, "reachability=skipped-probe-budget")
-}
-
-func TestDecideReachabilityProbeWithinBudget(t *testing.T) {
-	p := Decide(Input{
-		Hosts:                    100,
-		Templates:                10,
-		Requests:                 1000,
-		StrictProbe:              true,
-		PortsToProbe:             15, // cost = 1500: above baseline, within 2*baseline
-		ConcreteNetworkTemplates: 5,
-		GroupCount:               0,
-	})
-	require.True(t, p.BuildReachability)
-	require.True(t, p.ProbePorts)
-	require.True(t, p.UseReachabilityFilter)
-	require.Contains(t, p.Reason, "reachability=full")
-}
-
-func TestDecideReachabilityFull(t *testing.T) {
-	p := Decide(Input{
-		Hosts:                    5,
-		Templates:                60,
-		Requests:                 300,
-		StrictProbe:              true,
-		PortsToProbe:             3, // cost = 15 << baseline
-		ConcreteNetworkTemplates: 30,
-	})
-	require.True(t, p.BuildReachability)
-	require.True(t, p.ProbePorts)
-	require.True(t, p.UseReachabilityFilter)
-	require.Contains(t, p.Reason, "reachability=full")
-}
-
-func TestDecideReachabilityEmptyInput(t *testing.T) {
-	p := Decide(Input{StrictProbe: true, Hosts: 0, Templates: 10})
-	require.False(t, p.BuildReachability)
-	require.Contains(t, p.Reason, "reachability=skipped-empty")
-}
-
-func TestDecideReachabilityConcreteZeroUsesHTTPOnly(t *testing.T) {
-	p := Decide(Input{
-		Hosts:                    20,
-		Templates:                100,
-		Requests:                 2000,
-		StrictProbe:              true,
-		PortsToProbe:             5,
-		ConcreteNetworkTemplates: 0, // web-only set
-	})
-	require.True(t, p.BuildReachability)
-	require.False(t, p.ProbePorts)
-	require.True(t, p.UseReachabilityFilter)
-}
-
-func TestDecideHostSprayRequiresLargeRequestBudget(t *testing.T) {
-	// hosts small + many templates but requests below threshold → template-spray
-	p := Decide(Input{
-		Hosts:     5,
-		Templates: 200,
-		Requests:  4999,
-		BulkSize:  25,
-	})
-	require.Equal(t, scanstrategy.TemplateSpray.String(), p.Strategy)
-}
-
-func TestDecideApplyFilteredIgnoredWhenZero(t *testing.T) {
-	p := Decide(Input{
-		Hosts:            5,
-		Templates:        10,
-		Requests:         50,
-		StrictProbe:      false,
-		FilteredRequests: 0,
-	})
-	require.Equal(t, int64(50), p.ExpectedRequests)
-	p.ApplyFiltered(0)
-	require.Equal(t, int64(50), p.ExpectedRequests)
-	p.ApplyFiltered(12)
-	require.Equal(t, int64(12), p.ExpectedRequests)
-}
-
-func TestDecideTechFilterOffByDefault(t *testing.T) {
-	p := Decide(Input{
-		Hosts:              5,
-		Templates:          100,
-		Requests:           500,
-		TechFilter:         false,
-		TechBoundTemplates: 40,
-	})
-	require.False(t, p.UseTechFilter)
-	require.Contains(t, p.Reason, "tech-filter=off")
-}
-
-func TestDecideTechFilterSkippedWhenNoBound(t *testing.T) {
-	p := Decide(Input{
-		Hosts:              5,
-		Templates:          100,
-		Requests:           500,
-		TechFilter:         true,
-		TechBoundTemplates: 0,
-	})
-	require.False(t, p.UseTechFilter)
-	require.Contains(t, p.Reason, "tech-filter=skipped-no-bound-templates")
-}
-
-func TestDecideTechFilterOn(t *testing.T) {
-	p := Decide(Input{
-		Hosts:              5,
-		Templates:          100,
-		Requests:           500,
-		TechFilter:         true,
-		TechBoundTemplates: 40,
-	})
-	require.True(t, p.UseTechFilter)
-	require.Contains(t, p.Reason, "tech-filter=on")
-}

pkg/core/techfilter/techfilter.go

@@ -1,343 +0,0 @@
-// Package techfilter maps Wappalyzer products/categories to template tags and
-// decides whether a template can produce a finding on a fingerprinted host.
-//
-// Fail-open rules (coverage over aggressiveness):
-//   - templates with no product/macro tags always run
-//   - hosts with no fingerprints always allow every template
-//   - CDN/WAF-only fingerprints do not suppress product templates
-//   - non-HTTP templates are not filtered here (reachability owns that)
-package techfilter
-
-import (
-	"encoding/json"
-	"strings"
-	"sync"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
-	wappalyzer "github.com/projectdiscovery/wappalyzergo"
-)
-
-// genericTags are common nuclei tags that do not bind a template to a product.
-var genericTags = map[string]struct{}{
-	"cve": {}, "cnvd": {}, "edb": {}, "kev": {}, "vkev": {},
-	"misconfig": {}, "exposure": {}, "config": {}, "disclosure": {},
-	"tech": {}, "detect": {}, "fingerprint": {}, "favicon": {}, "waf": {},
-	"http": {}, "network": {}, "dns": {}, "ssl": {}, "tls": {},
-	"websocket": {}, "headless": {}, "file": {}, "code": {}, "javascript": {},
-	"workflow": {}, "intrusive": {}, "dos": {}, "fuzz": {}, "token-spray": {},
-	"osint": {}, "panel": {}, "login": {}, "auth": {}, "unauth": {},
-	"default-login": {}, "vuln": {}, "rce": {}, "lfi": {}, "sqli": {},
-	"xss": {}, "ssrf": {}, "xxe": {}, "ssti": {}, "idor": {},
-	"critical": {}, "high": {}, "medium": {}, "low": {}, "info": {},
-	"unknown": {}, "generic": {}, "misc": {}, "bench": {},
-	// Meta / source / taxonomy tags that are not products.
-	"oast": {}, "takeover": {}, "deserialization": {}, "packetstorm": {},
-	"seclists": {}, "hackerone": {}, "huntr": {}, "wp-plugin": {}, "wp-theme": {},
-	"wordpress-plugin": {}, "wordpress-theme": {}, "authenticated": {},
-	"unauthenticated": {}, "bypass": {}, "injection": {}, "traversal": {},
-	"redirect": {}, "crlf": {}, "csrf": {}, "cors": {},
-}
-
-// weakFingerprintTags alone must not suppress product-bound templates
-// (e.g. cloudflare/cdn-only fingerprints).
-var weakFingerprintTags = map[string]struct{}{
-	"cdn": {}, "waf": {}, "misc": {}, "security": {}, "caching": {},
-}
-
-// productAliases normalize common nuclei / wappalyzer spellings to one token.
-var productAliases = map[string]string{
-	"wp":                 "wordpress",
-	"httpd":              "apache",
-	"apache-http-server": "apache",
-}
-
-// categoryMacros maps Wappalyzer category names to short macro tags.
-var categoryMacros = map[string]string{
-	"cms":                   "cms",
-	"ecommerce":             "ecommerce",
-	"web servers":           "webserver",
-	"web-servers":           "webserver",
-	"web frameworks":        "web-framework",
-	"web-frameworks":        "web-framework",
-	"javascript frameworks": "js-framework",
-	"javascript-frameworks": "js-framework",
-	"javascript libraries":  "js-library",
-	"javascript-libraries":  "js-library",
-	"cdn":                   "cdn",
-	"databases":             "database",
-	"programming languages": "language",
-	"programming-languages": "language",
-	"paas":                  "paas",
-	"saas":                  "saas",
-	"security":              "security",
-	"caching":               "caching",
-	"miscellaneous":         "misc",
-}
-
-// HostProfile is the set of product + macro tags inferred for one target.
-type HostProfile struct {
-	Tags map[string]struct{} // empty / nil => unknown => fail-open
-
-	// Cats are the Wappalyzer category ids of every detected product. They
-	// record what the fingerprint actually has evidence about, which is what
-	// makes it safe to rule a template out. See AllowTags.
-	Cats map[int]struct{}
-}
-
-// HasTags reports whether fingerprinting produced at least one tag.
-func (h HostProfile) HasTags() bool {
-	return len(h.Tags) > 0
-}
-
-// HasSubstantiveTags reports whether the profile has a strong category macro
-// (cms, webserver, …). CDN/WAF-only fingerprints are not substantive.
-func (h HostProfile) HasSubstantiveTags() bool {
-	for tag := range h.Tags {
-		if isStrongMacro(tag) {
-			return true
-		}
-	}
-	return false
-}
-
-func isStrongMacro(tag string) bool {
-	if _, weak := weakFingerprintTags[tag]; weak {
-		return false
-	}
-	for _, m := range categoryMacros {
-		if tag == m {
-			return true
-		}
-	}
-	return false
-}
-
-// NormalizeProduct turns a Wappalyzer / nuclei product name into a tag token.
-func NormalizeProduct(name string) string {
-	name = strings.TrimSpace(strings.ToLower(name))
-	if name == "" {
-		return ""
-	}
-	if i := strings.IndexByte(name, ':'); i > 0 {
-		name = name[:i]
-	}
-	name = strings.ReplaceAll(name, "_", "-")
-	name = strings.Join(strings.Fields(name), "-")
-	if alias, ok := productAliases[name]; ok {
-		return alias
-	}
-	return name
-}
-
-// NormalizeCategory turns a Wappalyzer category into a macro tag.
-func NormalizeCategory(cat string) string {
-	key := strings.ToLower(strings.TrimSpace(cat))
-	if key == "" {
-		return ""
-	}
-	if m, ok := categoryMacros[key]; ok {
-		return m
-	}
-	return strings.ReplaceAll(key, " ", "-")
-}
-
-// IsGenericTag reports whether tag is not product/macro binding.
-func IsGenericTag(tag string) bool {
-	tag = strings.ToLower(strings.TrimSpace(tag))
-	if tag == "" {
-		return true
-	}
-	if _, ok := genericTags[tag]; ok {
-		return true
-	}
-	// cve2021 / cve-2021 style year tags bind no product.
-	if strings.HasPrefix(tag, "cve") {
-		rest := strings.TrimPrefix(tag, "cve")
-		rest = strings.TrimPrefix(rest, "-")
-		if rest != "" && isAllDigits(rest) {
-			return true
-		}
-	}
-	return false
-}
-
-func isAllDigits(s string) bool {
-	if s == "" {
-		return false
-	}
-	for _, r := range s {
-		if r < '0' || r > '9' {
-			return false
-		}
-	}
-	return true
-}
-
-// ProfileFromFingerprint builds a host profile from products and their categories.
-// products maps product name -> category names from Wappalyzer.
-func ProfileFromFingerprint(products map[string][]string) HostProfile {
-	tags := make(map[string]struct{})
-	observed := make(map[int]struct{})
-	byProduct := productCategories()
-	for product, cats := range products {
-		if p := NormalizeProduct(product); p != "" {
-			tags[p] = struct{}{}
-			for _, id := range byProduct[p] {
-				observed[id] = struct{}{}
-			}
-		}
-		for _, c := range cats {
-			if m := NormalizeCategory(c); m != "" {
-				tags[m] = struct{}{}
-			}
-		}
-	}
-	return HostProfile{Tags: tags, Cats: observed}
-}
-
-// TemplateProductTags returns product/macro tags that bind a template to a stack.
-// Empty means the template is generic and must always be allowed.
-func TemplateProductTags(t *templates.Template) []string {
-	if t == nil {
-		return nil
-	}
-	raw := t.Info.Tags.ToSlice()
-	out := make([]string, 0, len(raw))
-	seen := make(map[string]struct{}, len(raw))
-	known := knownFingerprintTokens()
-	for _, tag := range raw {
-		tag = NormalizeProduct(tag)
-		if tag == "" || IsGenericTag(tag) {
-			continue
-		}
-		// A tag binds only when fingerprinting can actually emit it. Anything
-		// else would never match a profile and would prune the template on
-		// every host, so it must leave the template unbound.
-		if _, ok := known[tag]; !ok {
-			continue
-		}
-		if _, ok := seen[tag]; ok {
-			continue
-		}
-		seen[tag] = struct{}{}
-		out = append(out, tag)
-	}
-	return out
-}
-
-// knownFingerprintTokens is every token a host profile can contain: the
-// normalized name of every Wappalyzer app plus every category macro.
-//
-// This is an allowlist on purpose. nuclei-templates carries thousands of tags
-// that name no product at all (discovery, wpscan, top-200, auth-bypass, ...).
-// Treating an unrecognized tag as a product binds the template to something no
-// fingerprint can ever report, which silently drops it on every host. Unknown
-// tags must therefore bind nothing.
-var knownFingerprintTokens = sync.OnceValue(func() map[string]struct{} {
-	var data struct {
-		Apps map[string]json.RawMessage `json:"apps"`
-	}
-	tokens := make(map[string]struct{}, 8192)
-	if err := json.Unmarshal([]byte(wappalyzer.GetFingerprints()), &data); err == nil {
-		for name := range data.Apps {
-			if p := NormalizeProduct(name); p != "" {
-				tokens[p] = struct{}{}
-			}
-		}
-	}
-	for _, macro := range categoryMacros {
-		tokens[macro] = struct{}{}
-	}
-	for tag := range weakFingerprintTags {
-		tokens[tag] = struct{}{}
-	}
-	return tokens
-})
-
-// IsTechBound reports whether the template is constrained to a product/macro.
-func IsTechBound(t *templates.Template) bool {
-	return len(TemplateProductTags(t)) > 0
-}
-
-// Allow reports whether template may run on a host with the given profile.
-// Fail-open when the host was not fingerprinted or the template is unbound.
-func Allow(profile HostProfile, t *templates.Template) bool {
-	return AllowTags(profile, TemplateProductTags(t))
-}
-
-// AllowTags is Allow with precomputed product tags (avoids per-pair allocation).
-func AllowTags(profile HostProfile, bound []string) bool {
-	if len(bound) == 0 {
-		return true
-	}
-	if !profile.HasTags() || !profile.HasSubstantiveTags() {
-		return true
-	}
-	for _, tag := range bound {
-		if _, ok := profile.Tags[tag]; ok {
-			return true
-		}
-	}
-
-	// No positive match. Ruling the template out now would assume the
-	// fingerprint is complete, and it never is: it is one page of one host, and
-	// a stack is routinely invisible from it (WordPress behind nginx is the
-	// ordinary case, not the exception). Absence of evidence is not evidence of
-	// absence, so a template is dropped only when the fingerprint positively
-	// contradicts it: something else was identified in the same category, and
-	// the template's own product was not. If any bound product sits in a
-	// category the fingerprint says nothing about, the template runs.
-	byProduct := productCategories()
-	for _, tag := range bound {
-		cats, ok := byProduct[tag]
-		if !ok || len(cats) == 0 {
-			return true // category macro or unknown product: no contradiction possible
-		}
-		contradicted := false
-		for _, id := range cats {
-			if _, seen := profile.Cats[id]; seen {
-				contradicted = true
-				break
-			}
-		}
-		if !contradicted {
-			return true
-		}
-	}
-	return false
-}
-
-// productCategories maps a normalized product token to its Wappalyzer category
-// ids. It is the evidence index: a category id present in a host profile means
-// fingerprinting actually identified something of that kind on that host.
-var productCategories = sync.OnceValue(func() map[string][]int {
-	var data struct {
-		Apps map[string]struct {
-			Cats []int `json:"cats"`
-		} `json:"apps"`
-	}
-	out := make(map[string][]int, 8192)
-	if err := json.Unmarshal([]byte(wappalyzer.GetFingerprints()), &data); err != nil {
-		return out
-	}
-	for name, app := range data.Apps {
-		p := NormalizeProduct(name)
-		if p == "" || len(app.Cats) == 0 {
-			continue
-		}
-		// Aliases collapse several apps onto one token; union their categories.
-		out[p] = append(out[p], app.Cats...)
-	}
-	return out
-})
-
-// CountTechBound returns how many templates carry product/macro tags.
-func CountTechBound(tpls []*templates.Template) int {
-	n := 0
-	for _, t := range tpls {
-		if IsTechBound(t) {
-			n++
-		}
-	}
-	return n
-}

pkg/core/techfilter/techfilter_test.go

@@ -1,208 +0,0 @@
-package techfilter
-
-import (
-	"testing"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/model"
-	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/stringslice"
-	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
-)
-
-func TestNormalizeProductAndCategory(t *testing.T) {
-	if got := NormalizeProduct("Nginx:1.25"); got != "nginx" {
-		t.Fatalf("NormalizeProduct = %q", got)
-	}
-	if got := NormalizeProduct("wp"); got != "wordpress" {
-		t.Fatalf("alias = %q", got)
-	}
-	if got := NormalizeCategory("Web servers"); got != "webserver" {
-		t.Fatalf("category = %q", got)
-	}
-}
-
-func TestProfileFromFingerprint(t *testing.T) {
-	p := ProfileFromFingerprint(map[string][]string{
-		"Nginx":     {"Web servers"},
-		"WordPress": {"CMS"},
-	})
-	for _, want := range []string{"nginx", "webserver", "wordpress", "cms"} {
-		if _, ok := p.Tags[want]; !ok {
-			t.Fatalf("missing tag %q in %#v", want, p.Tags)
-		}
-	}
-}
-
-func tagged(tags ...string) *templates.Template {
-	return &templates.Template{
-		Info: model.Info{Tags: stringslice.StringSlice{Value: tags}},
-	}
-}
-
-func TestAllowFailOpenAndMatch(t *testing.T) {
-	nginx := ProfileFromFingerprint(map[string][]string{"Nginx": {"Web servers"}})
-	empty := HostProfile{}
-
-	generic := tagged("misconfig", "http")
-	wp := tagged("cve", "wordpress")
-	ngx := tagged("nginx", "misconfig")
-	cms := tagged("cms")
-
-	if !Allow(nginx, generic) {
-		t.Fatal("generic must always run")
-	}
-	if !Allow(empty, wp) {
-		t.Fatal("unknown host must fail-open for bound templates")
-	}
-	// An nginx fingerprint says nothing about the CMS behind it, and WordPress
-	// behind nginx is an ordinary stack. Absence of a CMS in the profile is not
-	// evidence that there is none.
-	if !Allow(nginx, wp) {
-		t.Fatal("nginx says nothing about a CMS, so a wordpress template must run")
-	}
-	if !Allow(nginx, ngx) {
-		t.Fatal("nginx template must run on nginx host")
-	}
-	if !Allow(nginx, cms) {
-		t.Fatal("nginx says nothing about a CMS, so a cms-bound template must run")
-	}
-	wpHost := ProfileFromFingerprint(map[string][]string{"WordPress": {"CMS"}})
-	if !Allow(wpHost, cms) {
-		t.Fatal("cms macro must match wordpress host")
-	}
-	if !Allow(wpHost, wp) {
-		t.Fatal("wordpress product must match")
-	}
-}
-
-func TestMetaTagsAreNotTechBound(t *testing.T) {
-	cases := [][]string{
-		{"cve", "cve2021", "oast", "rce"},
-		{"takeover", "dns"},
-		{"hackerone", "packetstorm", "seclists"},
-		{"wp-plugin", "wordpress-plugin"},
-	}
-	for _, tags := range cases {
-		tpl := tagged(tags...)
-		if IsTechBound(tpl) {
-			t.Fatalf("tags %v must not be tech-bound, got %v", tags, TemplateProductTags(tpl))
-		}
-		cdnOnly := ProfileFromFingerprint(map[string][]string{"Cloudflare": {"CDN"}})
-		if !Allow(cdnOnly, tpl) {
-			t.Fatalf("unbound meta tags %v must run on any host", tags)
-		}
-	}
-}
-
-func TestCDNOnlyFingerprintFailOpen(t *testing.T) {
-	cdnOnly := ProfileFromFingerprint(map[string][]string{"Cloudflare": {"CDN"}})
-	if cdnOnly.HasSubstantiveTags() {
-		t.Fatalf("cdn-only profile should not be substantive: %#v", cdnOnly.Tags)
-	}
-	wp := tagged("wordpress")
-	if !Allow(cdnOnly, wp) {
-		t.Fatal("CDN-only fingerprint must fail-open for product templates")
-	}
-}
-
-func TestCountTechBound(t *testing.T) {
-	n := CountTechBound([]*templates.Template{
-		tagged("misconfig"),
-		tagged("wordpress"),
-		tagged("nginx", "http"),
-		tagged("cve2023", "oast"),
-	})
-	if n != 2 {
-		t.Fatalf("CountTechBound = %d", n)
-	}
-}
-
-// A tag only binds when fingerprinting can emit it. nuclei-templates carries
-// thousands of tags that name no product; binding to one of those prunes the
-// template on every host, because no profile can ever contain it.
-func TestOnlyKnownFingerprintTokensBind(t *testing.T) {
-	nonProducts := [][]string{
-		{"misconfig", "headers", "generic", "vuln"}, // http-missing-security-headers
-		{"tech", "discovery"},                       // tech-detect
-		{"waf", "tech", "misc", "discovery"},        // waf-detect
-		{"wpscan"},
-		{"top-100"}, {"top-200"},
-		{"auth-bypass"}, {"time-based-sqli"},
-		{"install"}, {"files"}, {"file-upload"},
-		{"phishing"}, {"osint-social"},
-		{"debug"}, {"devops"}, {"token"}, {"tokens"}, {"fpd"}, {"oss"},
-	}
-	nginx := ProfileFromFingerprint(map[string][]string{"Nginx": {"Web servers"}})
-	for _, tags := range nonProducts {
-		tpl := tagged(tags...)
-		if got := TemplateProductTags(tpl); len(got) != 0 {
-			t.Fatalf("tags %v must not bind, got %v", tags, got)
-		}
-		if !Allow(nginx, tpl) {
-			t.Fatalf("unbound template %v must run on every host", tags)
-		}
-	}
-}
-
-// The inversion must not disable the filter: real products still bind, and a
-// template for another stack is still pruned.
-func TestKnownProductsStillBind(t *testing.T) {
-	for _, tags := range [][]string{{"wordpress"}, {"apache"}, {"joomla"}, {"cms"}} {
-		if got := TemplateProductTags(tagged(tags...)); len(got) == 0 {
-			t.Fatalf("tags %v name a product and must bind", tags)
-		}
-	}
-
-	wp := ProfileFromFingerprint(map[string][]string{"WordPress": {"CMS"}})
-	if !Allow(wp, tagged("wordpress", "rce")) {
-		t.Fatal("a wordpress template must run on a wordpress profile")
-	}
-}
-
-// A template is dropped only when the fingerprint contradicts it: it identified
-// something else of the same kind and not the template's own product.
-func TestPruneOnlyOnPositiveContradiction(t *testing.T) {
-	nginx := ProfileFromFingerprint(map[string][]string{"Nginx": {"Web servers"}})
-	wpHost := ProfileFromFingerprint(map[string][]string{
-		"WordPress": {"CMS"}, "Nginx": {"Web servers"},
-	})
-
-	// Contradiction: a web server was identified and it is not apache.
-	if Allow(nginx, tagged("apache")) {
-		t.Fatal("nginx was identified, so an apache template is contradicted")
-	}
-	// Contradiction: a CMS was identified and it is not joomla.
-	if Allow(wpHost, tagged("joomla")) {
-		t.Fatal("wordpress was identified, so a joomla template is contradicted")
-	}
-	// No evidence: nothing of that kind was identified at all.
-	for _, tags := range [][]string{{"wordpress"}, {"joomla"}, {"cms"}, {"mysql"}} {
-		if !Allow(nginx, tagged(tags...)) {
-			t.Fatalf("nginx gives no evidence about %v, so it must run", tags)
-		}
-	}
-	// One uncontradicted product is enough to keep a multi-product template.
-	if !Allow(nginx, tagged("apache", "wordpress")) {
-		t.Fatal("wordpress is uncontradicted, so the template must run")
-	}
-	// A positive match always wins.
-	if !Allow(wpHost, tagged("wordpress")) {
-		t.Fatal("a positive match must run")
-	}
-}
-
-func TestKnownFingerprintTokensVocabulary(t *testing.T) {
-	tokens := knownFingerprintTokens()
-	if len(tokens) < 5000 {
-		t.Fatalf("expected the wappalyzer vocabulary, got %d tokens", len(tokens))
-	}
-	for _, want := range []string{"wordpress", "nginx", "apache", "cms", "webserver"} {
-		if _, ok := tokens[want]; !ok {
-			t.Fatalf("token %q missing from the vocabulary", want)
-		}
-	}
-	for _, bad := range []string{"discovery", "headers", "wpscan", "top-200"} {
-		if _, ok := tokens[bad]; ok {
-			t.Fatalf("token %q is not a product and must not be in the vocabulary", bad)
-		}
-	}
-}

pkg/input/formats/README.md

@@ -5,6 +5,7 @@ Formats implements support for passing a number of request source as input provi
 Currently the following formats are implemented - 
 
 - Burp Suite XML Request/Response file
+- HAR (HTTP Archive) file exported by browsers and proxies
 - Proxify JSONL output file
 - OpenAPI Specification file
 - Postman Collection file

pkg/input/formats/har/har.go

@@ -0,0 +1,137 @@
+package har
+
+import (
+	"encoding/base64"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v3/pkg/input/formats"
+	"github.com/projectdiscovery/nuclei/v3/pkg/input/types"
+	"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
+)
+
+// HARFormat is an HTTP Archive (HAR 1.2) parser, the format browsers and
+// proxies export captured traffic in.
+type HARFormat struct {
+	opts formats.InputFormatOptions
+}
+
+// New creates a new HAR parser
+func New() *HARFormat {
+	return &HARFormat{}
+}
+
+var _ formats.Format = &HARFormat{}
+
+type archive struct {
+	Log struct {
+		Entries []entry `json:"entries"`
+	} `json:"log"`
+}
+
+type entry struct {
+	Request struct {
+		Method   string   `json:"method"`
+		URL      string   `json:"url"`
+		Headers  []header `json:"headers"`
+		PostData *struct {
+			Text string `json:"text"`
+		} `json:"postData"`
+	} `json:"request"`
+	Response struct {
+		Status     int      `json:"status"`
+		StatusText string   `json:"statusText"`
+		Headers    []header `json:"headers"`
+		Content    struct {
+			Text     string `json:"text"`
+			Encoding string `json:"encoding"`
+		} `json:"content"`
+	} `json:"response"`
+}
+
+type header struct {
+	Name  string `json:"name"`
+	Value string `json:"value"`
+}
+
+// Name returns the name of the format
+func (h *HARFormat) Name() string {
+	return "har"
+}
+
+func (h *HARFormat) SetOptions(options formats.InputFormatOptions) {
+	h.opts = options
+}
+
+// Parse parses the input and calls the provided callback
+// function for each RawRequest it discovers.
+func (h *HARFormat) Parse(input io.Reader, resultsCb formats.ParseReqRespCallback, filePath string) error {
+	var har archive
+	if err := json.NewDecoder(input).Decode(&har); err != nil {
+		return errors.Wrap(err, "could not decode har file")
+	}
+
+	for _, e := range har.Log.Entries {
+		// Browsers also log data:, blob: and extension URLs, which have no
+		// host to send a request to.
+		if !strings.HasPrefix(e.Request.URL, "http://") && !strings.HasPrefix(e.Request.URL, "https://") {
+			continue
+		}
+		rawRequest, err := types.ParseRawRequestWithURL(rawRequestOf(e), e.Request.URL)
+		if err != nil {
+			gologger.Warning().Msgf("har: Could not parse request %s: %s\n", e.Request.URL, err)
+			continue
+		}
+		// Status 0 is a request that was blocked or never answered.
+		if e.Response.Status != 0 {
+			rawRequest.Response = &types.HttpResponse{Raw: rawResponseOf(e)}
+		}
+		resultsCb(rawRequest)
+	}
+	return nil
+}
+
+// rawRequestOf rebuilds the request as HTTP/1.1 with an absolute target, which
+// carries the authority that HTTP/2 captures only keep in pseudo-headers.
+func rawRequestOf(e entry) string {
+	var b strings.Builder
+	b.WriteString(e.Request.Method + " " + e.Request.URL + " HTTP/1.1\r\n")
+	writeHeaders(&b, e.Request.Headers)
+	if e.Request.PostData != nil {
+		b.WriteString(e.Request.PostData.Text)
+	}
+	return b.String()
+}
+
+func rawResponseOf(e entry) string {
+	statusText := e.Response.StatusText
+	if statusText == "" {
+		statusText = http.StatusText(e.Response.Status)
+	}
+	var b strings.Builder
+	b.WriteString("HTTP/1.1 " + strconv.Itoa(e.Response.Status) + " " + statusText + "\r\n")
+	writeHeaders(&b, e.Response.Headers)
+	body := e.Response.Content.Text
+	if e.Response.Content.Encoding == "base64" {
+		if decoded, err := base64.StdEncoding.DecodeString(body); err == nil {
+			body = string(decoded)
+		}
+	}
+	b.WriteString(body)
+	return b.String()
+}
+
+func writeHeaders(b *strings.Builder, headers []header) {
+	for _, h := range headers {
+		// HTTP/2 pseudo-headers (:authority, :path, ...) are not headers.
+		if strings.HasPrefix(h.Name, ":") {
+			continue
+		}
+		b.WriteString(h.Name + ": " + h.Value + "\r\n")
+	}
+	b.WriteString("\r\n")
+}

pkg/input/formats/har/har_test.go

@@ -0,0 +1,71 @@
+package har
+
+import (
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/input/types"
+	"github.com/stretchr/testify/require"
+)
+
+const chromeHAR = "../testdata/chrome.har"
+
+func parse(t *testing.T, input string) []*types.RequestResponse {
+	t.Helper()
+	file, err := os.Open(input)
+	require.NoError(t, err)
+	defer func() { _ = file.Close() }()
+
+	var got []*types.RequestResponse
+	require.NoError(t, New().Parse(file, func(rr *types.RequestResponse) bool {
+		got = append(got, rr)
+		return false
+	}, input))
+	return got
+}
+
+func TestHARParse(t *testing.T) {
+	got := parse(t, chromeHAR)
+
+	urls := make([]string, 0, len(got))
+	for _, rr := range got {
+		urls = append(urls, rr.URL.String())
+	}
+	require.Equal(t, []string{
+		"https://ginandjuice.shop/catalog?category=Juice",
+		"https://ginandjuice.shop/login",
+		"https://ginandjuice.shop/blocked.js",
+	}, urls, "non-http entries such as data: URLs are skipped")
+}
+
+func TestHARParseHTTP2Request(t *testing.T) {
+	rr := parse(t, chromeHAR)[0]
+
+	require.Equal(t, "GET", rr.Request.Method)
+	cookie, _ := rr.Request.Headers.Get("cookie")
+	require.Equal(t, "session=abc123", cookie)
+	for _, name := range rr.Request.Headers.GetKeys() {
+		require.False(t, strings.HasPrefix(name, ":"), "pseudo-header %s is not a real header", name)
+	}
+
+	req, err := rr.BuildRequest()
+	require.NoError(t, err)
+	require.Equal(t, "ginandjuice.shop", req.Host)
+}
+
+func TestHARParseBodyAndResponse(t *testing.T) {
+	got := parse(t, chromeHAR)
+
+	login := got[1]
+	require.Equal(t, "POST", login.Request.Method)
+	require.Equal(t, "username=carlos&password=hunter2", login.Request.Body)
+	require.NotNil(t, login.Response)
+	require.Contains(t, login.Response.Raw, "302 Found")
+	require.Contains(t, login.Response.Raw, "Location: /my-account")
+
+	require.NotNil(t, got[0].Response)
+	require.Contains(t, got[0].Response.Raw, "<h1>Juice</h1>", "base64 content is decoded")
+
+	require.Nil(t, got[2].Response, "a request that never got an answer has no response")
+}

pkg/input/formats/openapi/fuzz.go

@@ -0,0 +1,20 @@
+//go:build gofuzz
+// +build gofuzz
+
+package openapi
+
+// Fuzz exercises OpenAPI 3.0 schema parsing and request generation. The input
+// bytes are treated as a raw schema document (YAML or JSON) and fed through the
+// format's Parse, which loads the schema and generates requests from it.
+func Fuzz(data []byte) int {
+	if len(data) == 0 {
+		return 0
+	}
+	if len(data) > fuzzMaxInputSize {
+		return -1
+	}
+	if !fuzzOpenAPIParsing(data) {
+		return 0
+	}
+	return 1
+}

pkg/input/formats/openapi/fuzz_harness.go

@@ -0,0 +1,32 @@
+package openapi
+
+import (
+	"bytes"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/input/formats"
+	"github.com/projectdiscovery/nuclei/v3/pkg/input/types"
+)
+
+const fuzzMaxInputSize = 64 << 10
+
+// fuzzOpenAPIParsing feeds data to the OpenAPI format parser as a raw schema
+// document and drives request generation. It must never panic for any input;
+// it returns true when the schema parsed and at least one request was produced.
+func fuzzOpenAPIParsing(data []byte) bool {
+	if len(data) == 0 || len(data) > fuzzMaxInputSize {
+		return false
+	}
+
+	format := New()
+	format.SetOptions(formats.InputFormatOptions{})
+
+	produced := false
+	cb := func(rr *types.RequestResponse) bool {
+		produced = true
+		return true
+	}
+	if err := format.Parse(bytes.NewReader(data), cb, "fuzz.openapi"); err != nil {
+		return false
+	}
+	return produced
+}

pkg/input/formats/openapi/fuzz_harness_test.go

@@ -0,0 +1,84 @@
+package openapi
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/input/types"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFuzzOpenAPIParsingSeedCorpus(t *testing.T) {
+	entries, err := os.ReadDir("testdata/gofuzz-corpus")
+	require.NoError(t, err)
+	require.NotEmpty(t, entries)
+
+	for _, entry := range entries {
+		if entry.IsDir() {
+			continue
+		}
+
+		data, err := os.ReadFile(filepath.Join("testdata/gofuzz-corpus", entry.Name()))
+		require.NoError(t, err)
+		require.Truef(t, fuzzOpenAPIParsing(data), "seed %s should produce at least one request", entry.Name())
+	}
+}
+
+func TestFuzzOpenAPIParsingRejectsOversizeInput(t *testing.T) {
+	require.False(t, fuzzOpenAPIParsing(make([]byte, fuzzMaxInputSize+1)))
+}
+
+// Null entries decode to nil pointers and used to panic during request
+// generation. The remaining valid entries must still produce requests.
+func TestFuzzOpenAPIParsingNullEntries(t *testing.T) {
+	specs := map[string]string{
+		"null server": `
+openapi: 3.0.0
+info: {title: t, version: "1"}
+servers:
+  -
+  - {url: "http://example.com"}
+paths:
+  /users:
+    get:
+      responses: {"200": {description: ok}}
+`,
+		"null media type": `
+openapi: 3.0.0
+info: {title: t, version: "1"}
+servers: [{url: "http://example.com"}]
+paths:
+  /users:
+    post:
+      requestBody:
+        content:
+          application/xml:
+          application/json:
+            schema: {type: object, properties: {name: {type: string}}}
+      responses: {"200": {description: ok}}
+`,
+	}
+
+	for name, spec := range specs {
+		t.Run(name, func(t *testing.T) {
+			require.True(t, fuzzOpenAPIParsing([]byte(spec)))
+		})
+	}
+}
+
+func TestGenerateRequestsOnlyNullServers(t *testing.T) {
+	spec := `
+openapi: 3.0.0
+info: {title: t, version: "1"}
+servers:
+  -
+paths:
+  /users:
+    get:
+      responses: {"200": {description: ok}}
+`
+	err := New().Parse(strings.NewReader(spec), func(*types.RequestResponse) bool { return false }, "spec.yaml")
+	require.ErrorContains(t, err, "no servers found in openapi schema")
+}

pkg/input/formats/openapi/generator.go

@@ -9,6 +9,7 @@ import (
 	"net/http/httputil"
 	"net/url"
 	"os"
+	"slices"
 	"strings"
 
 	"github.com/clbanning/mxj/v2"
@@ -33,7 +34,9 @@ const (
 
 // GenerateRequestsFromSchema generates http requests from an OpenAPI 3.0 document object
 func GenerateRequestsFromSchema(schema *openapi3.T, opts formats.InputFormatOptions, callback formats.ParseReqRespCallback) error {
-	if len(schema.Servers) == 0 {
+	// Null entries in `servers:` are skipped below, so a list of only null
+	// entries has no usable server, the same as an empty list.
+	if !slices.ContainsFunc(schema.Servers, func(s *openapi3.Server) bool { return s != nil }) {
 		return errors.New("no servers found in openapi schema")
 	}
 	globalParams, globalSecurityErr := selectSecurityParameters(schema, &schema.Security, opts.Variables)
@@ -49,6 +52,11 @@ func GenerateRequestsFromSchema(schema *openapi3.T, opts formats.InputFormatOpti
 	}
 
 	for _, serverURL := range schema.Servers {
+		// A null entry in the `servers:` list decodes to a nil *Server, so skip
+		// it rather than dereferencing it.
+		if serverURL == nil {
+			continue
+		}
 		pathURL := serverURL.URL
 		// Split the server URL into baseURL and serverPath
 		u, err := url.Parse(pathURL)
@@ -268,6 +276,11 @@ func generateRequestsFromOp(opts *generateReqOptions) error {
 
 	if opts.op.RequestBody != nil {
 		for content, value := range opts.op.RequestBody.Value.Content {
+			// A null media-type entry (e.g. `content: {application/json: }`)
+			// decodes to a nil *MediaType, so skip it before dereferencing.
+			if value == nil {
+				continue
+			}
 			cloned := req.Clone(req.Context())
 
 			var val interface{}

pkg/input/formats/openapi/testdata/gofuzz-corpus/json-spec.seed

@@ -0,0 +1 @@
+{"openapi":"3.0.0","info":{"title":"t","version":"1"},"servers":[{"url":"http://example.com"}],"paths":{"/a":{"post":{"requestBody":{"content":{"application/json":{"schema":{"type":"object","properties":{"k":{"type":"string"}}}}}},"responses":{"200":{"description":"ok"}}}}}}

pkg/input/formats/openapi/testdata/gofuzz-corpus/minimal.seed

@@ -0,0 +1,7 @@
+openapi: 3.0.0
+info: {title: t, version: "1"}
+servers: [{url: "http://example.com"}]
+paths:
+  /users:
+    get:
+      responses: {"200": {description: ok}}

pkg/input/formats/openapi/testdata/gofuzz-corpus/nil-media-type.seed

@@ -0,0 +1,12 @@
+openapi: 3.0.0
+info: {title: t, version: "1"}
+servers: [{url: "http://example.com"}]
+paths:
+  /users:
+    post:
+      requestBody:
+        content:
+          application/xml:
+          application/json:
+            schema: {type: object, properties: {name: {type: string}}}
+      responses: {"200": {description: ok}}

pkg/input/formats/openapi/testdata/gofuzz-corpus/nil-server.seed

@@ -0,0 +1,9 @@
+openapi: 3.0.0
+info: {title: t, version: "1"}
+servers:
+  -
+  - {url: "http://example.com"}
+paths:
+  /users:
+    get:
+      responses: {"200": {description: ok}}

pkg/input/formats/swagger/swagger.go

@@ -3,7 +3,9 @@ package swagger
 import (
 	"fmt"
 	"io"
+	"maps"
 	"path"
+	"slices"
 
 	"github.com/getkin/kin-openapi/openapi2"
 	"github.com/getkin/kin-openapi/openapi2conv"
@@ -55,6 +57,7 @@ func (j *SwaggerFormat) Parse(input io.Reader, resultsCb formats.ParseReqRespCal
 	if err != nil {
 		return errors.Wrap(err, "could not decode openapi 2.0 schema")
 	}
+	dropNullEntries(schemav2)
 	schema, err := openapi2conv.ToV3(schemav2)
 	if err != nil {
 		return errors.Wrap(err, "could not convert openapi 2.0 schema to 3.0")
@@ -67,6 +70,32 @@ func (j *SwaggerFormat) Parse(input io.Reader, resultsCb formats.ParseReqRespCal
 	return openapi.GenerateRequestsFromSchema(schema, j.opts, resultsCb)
 }
 
+// dropNullEntries removes null entries from the maps and lists of a decoded
+// OpenAPI 2.0 document. A null value such as `paths: {/x: }` or an empty list
+// item under `parameters:` decodes to a nil pointer, which openapi2conv.ToV3
+// dereferences without checking.
+func dropNullEntries(doc *openapi2.T) {
+	maps.DeleteFunc(doc.Paths, func(_ string, v *openapi2.PathItem) bool { return v == nil })
+	maps.DeleteFunc(doc.Parameters, func(_ string, v *openapi2.Parameter) bool { return v == nil })
+	dropNullResponses(doc.Responses)
+	for _, pathItem := range doc.Paths {
+		pathItem.Parameters = slices.DeleteFunc(pathItem.Parameters, func(v *openapi2.Parameter) bool { return v == nil })
+		for _, op := range pathItem.Operations() {
+			op.Parameters = slices.DeleteFunc(op.Parameters, func(v *openapi2.Parameter) bool { return v == nil })
+			dropNullResponses(op.Responses)
+		}
+	}
+}
+
+// dropNullResponses removes null responses, and null headers of the remaining
+// responses, which openapi2conv.ToV3Response also dereferences.
+func dropNullResponses(responses map[string]*openapi2.Response) {
+	maps.DeleteFunc(responses, func(_ string, v *openapi2.Response) bool { return v == nil })
+	for _, response := range responses {
+		maps.DeleteFunc(response.Headers, func(_ string, v *openapi2.Header) bool { return v == nil })
+	}
+}
+
 func decodeYAML(data []byte, target interface{}) error {
 	var value interface{}
 	if err := yaml.Unmarshal(data, &value); err != nil {

pkg/input/formats/swagger/swagger_test.go

@@ -2,6 +2,7 @@ package swagger
 
 import (
 	"os"
+	"strings"
 	"testing"
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/types"
@@ -39,3 +40,69 @@ func TestSwaggerAPIParser(t *testing.T) {
 	}
 	require.ElementsMatch(t, gotMethodsToURLs, expectedURLs, "could not get swagger urls")
 }
+
+func TestSwaggerParseNullEntries(t *testing.T) {
+	// Each spec has a null map value or list item that decodes to a nil
+	// pointer and used to panic inside openapi2conv.ToV3.
+	specs := map[string]string{
+		"null-path-item": `
+swagger: "2.0"
+info: {title: t, version: "1"}
+host: localhost
+paths:
+  /x:
+`,
+		"null-parameter": `
+swagger: "2.0"
+info: {title: t, version: "1"}
+host: localhost
+paths:
+  /x:
+    get:
+      parameters:
+        -
+      responses: {"200": {description: ok}}
+`,
+		"null-response": `
+swagger: "2.0"
+info: {title: t, version: "1"}
+host: localhost
+paths:
+  /x:
+    get:
+      responses: {"200": }
+`,
+		"null-response-header": `
+swagger: "2.0"
+info: {title: t, version: "1"}
+host: localhost
+responses:
+  Shared:
+    description: ok
+    headers:
+      X-Shared:
+paths:
+  /x:
+    get:
+      responses:
+        "200":
+          description: ok
+          headers:
+            X-Rate-Limit:
+`,
+	}
+
+	for name, spec := range specs {
+		t.Run(name, func(t *testing.T) {
+			var count int
+			err := New().Parse(strings.NewReader(spec), func(request *types.RequestResponse) bool {
+				count++
+				return false
+			}, "spec.yaml")
+			require.NoError(t, err)
+			if name != "null-path-item" {
+				require.Equal(t, 1, count, "the operation should still produce a request")
+			}
+		})
+	}
+}

pkg/input/formats/testdata/chrome.har

@@ -0,0 +1,71 @@
+{
+  "log": {
+    "version": "1.2",
+    "creator": { "name": "WebInspector", "version": "537.36" },
+    "entries": [
+      {
+        "request": {
+          "method": "GET",
+          "url": "https://ginandjuice.shop/catalog?category=Juice",
+          "httpVersion": "http/2.0",
+          "headers": [
+            { "name": ":authority", "value": "ginandjuice.shop" },
+            { "name": ":method", "value": "GET" },
+            { "name": ":path", "value": "/catalog?category=Juice" },
+            { "name": ":scheme", "value": "https" },
+            { "name": "accept", "value": "text/html" },
+            { "name": "cookie", "value": "session=abc123" }
+          ],
+          "queryString": [{ "name": "category", "value": "Juice" }],
+          "cookies": [],
+          "headersSize": -1,
+          "bodySize": 0
+        },
+        "response": {
+          "status": 200,
+          "statusText": "",
+          "httpVersion": "http/2.0",
+          "headers": [{ "name": "content-type", "value": "text/html; charset=utf-8" }],
+          "content": { "size": 19, "mimeType": "text/html", "text": "PGgxPkp1aWNlPC9oMT4=", "encoding": "base64" }
+        }
+      },
+      {
+        "request": {
+          "method": "POST",
+          "url": "https://ginandjuice.shop/login",
+          "httpVersion": "HTTP/1.1",
+          "headers": [
+            { "name": "Host", "value": "ginandjuice.shop" },
+            { "name": "Content-Type", "value": "application/x-www-form-urlencoded" }
+          ],
+          "postData": { "mimeType": "application/x-www-form-urlencoded", "text": "username=carlos&password=hunter2" }
+        },
+        "response": {
+          "status": 302,
+          "statusText": "Found",
+          "httpVersion": "HTTP/1.1",
+          "headers": [{ "name": "Location", "value": "/my-account" }],
+          "content": { "size": 0, "mimeType": "" }
+        }
+      },
+      {
+        "request": {
+          "method": "GET",
+          "url": "data:image/png;base64,iVBORw0KGgo=",
+          "httpVersion": "",
+          "headers": []
+        },
+        "response": { "status": 200, "statusText": "", "httpVersion": "", "headers": [], "content": { "size": 0 } }
+      },
+      {
+        "request": {
+          "method": "GET",
+          "url": "https://ginandjuice.shop/blocked.js",
+          "httpVersion": "",
+          "headers": []
+        },
+        "response": { "status": 0, "statusText": "", "httpVersion": "", "headers": [], "content": { "size": 0 } }
+      }
+    ]
+  }
+}

pkg/input/provider/http/multiformat.go

@@ -10,6 +10,7 @@ import (
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/formats"
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/formats/burp"
+	"github.com/projectdiscovery/nuclei/v3/pkg/input/formats/har"
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/formats/json"
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/formats/openapi"
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/formats/raw"
@@ -142,6 +143,7 @@ func (i *HttpInputProvider) Close() {}
 // Supported Providers
 var providersList = []formats.Format{
 	burp.New(),
+	har.New(),
 	json.New(),
 	yaml.New(),
 	openapi.New(),

pkg/input/provider/list/hmap.go

@@ -89,8 +89,9 @@ func New(opts *Options) (*ListInputProvider, error) {
 		patterns:      normalize.NewFingerprinter(options.MaxURLsPerPattern),
 		ipOptions: &ipOptions{
 			ScanAllIPs: options.ScanAllIPs,
-			IPV4:       sliceutil.Contains(options.IPVersion, "4"),
-			IPV6:       sliceutil.Contains(options.IPVersion, "6"),
+			IPV4:       len(options.IPVersion) == 0 || sliceutil.Contains(options.IPVersion, "4"),
+			IPV6:       len(options.IPVersion) == 0 || sliceutil.Contains(options.IPVersion, "6"),
+			Auto:       len(options.IPVersion) == 0,
 		},
 		excludedHosts: make(map[string]struct{}),
 		profiles:      targetprofile.NewRegistry(configuration.TemplatesDirectory(options, config.DefaultConfig)),
@@ -168,6 +169,61 @@ func (i *ListInputProvider) TargetProfiles() *targetprofile.Registry {
 	return i.profiles
 }
 
+// selectedIPs returns address pins for a hostname. An empty pin delegates
+// DNS and connection fallback to fastdialer, which races IPv4 and IPv6.
+// Set and Del share this selection. With -iv omitted and without -sa, one
+// unpinned target is returned. -sa returns every selected address.
+func (i *ListInputProvider) selectedIPs(executionId, hostname string) []string {
+	if i.ipOptions == nil || (!i.ipOptions.ScanAllIPs && (i.ipOptions.Auto || !i.ipOptions.IPV6)) {
+		return []string{""}
+	}
+	dialers := protocolstate.GetDialersWithId(executionId)
+	if dialers == nil {
+		gologger.Warning().Msgf("cannot resolve %s: dialer %s not found", hostname, executionId)
+		return nil
+	}
+	dnsData, err := dialers.Fastdialer.GetDNSData(hostname)
+	if err != nil {
+		gologger.Warning().Msgf("DNS resolution failed for %s: %v", hostname, err)
+		if i.ipOptions.IPV4 {
+			return []string{""}
+		}
+		return nil
+	}
+	if len(dnsData.A) == 0 && len(dnsData.AAAA) == 0 {
+		gologger.Warning().Msgf("DNS resolution returned no IP addresses for %s", hostname)
+		if i.ipOptions.IPV4 {
+			return []string{""}
+		}
+		return nil
+	}
+	if i.ipOptions.ScanAllIPs {
+		var ips []string
+		if i.ipOptions.IPV6 {
+			ips = append(ips, dnsData.AAAA...)
+		}
+		if i.ipOptions.IPV4 {
+			ips = append(ips, dnsData.A...)
+		}
+		if len(ips) == 0 {
+			gologger.Warning().Msgf("no DNS addresses matching -iv selection for %s", hostname)
+		}
+		return ips
+	}
+	var ips []string
+	if i.ipOptions.IPV6 && len(dnsData.AAAA) > 0 {
+		ips = append(ips, dnsData.AAAA[0])
+	}
+	// Explicit -iv 4,6 scans the first IPv6 address and one unpinned target.
+	if i.ipOptions.IPV4 {
+		ips = append(ips, "")
+	}
+	if len(ips) == 0 {
+		gologger.Warning().Msgf("no IPv6 DNS address found for %s", hostname)
+	}
+	return ips
+}
+
 // Set normalizes and stores passed input values
 func (i *ListInputProvider) Set(executionId string, value string) {
 	i.set(executionId, value, nil)
@@ -202,75 +258,11 @@ func (i *ListInputProvider) set(executionId string, value string, selection *tar
 		return
 	}
 
-	if i.ipOptions.ScanAllIPs {
-		// scan all ips
-		dialers := protocolstate.GetDialersWithId(executionId)
-		if dialers == nil {
-			gologger.Error().Msgf("dialers with executionId %s not found", executionId)
-			return
-		}
-
-		dnsData, err := dialers.Fastdialer.GetDNSData(urlx.Hostname())
-		if err == nil {
-			if (len(dnsData.A) + len(dnsData.AAAA)) > 0 {
-				var ips []string
-				if i.ipOptions.IPV4 {
-					ips = append(ips, dnsData.A...)
-				}
-				if i.ipOptions.IPV6 {
-					ips = append(ips, dnsData.AAAA...)
-				}
-				for _, ip := range ips {
-					if ip == "" {
-						continue
-					}
-					metaInput := contextargs.NewMetaInput()
-					metaInput.Input = URL
-					metaInput.CustomIP = ip
-					i.setItem(metaInput, selection)
-				}
-				return
-			} else {
-				gologger.Debug().Msgf("scanAllIps: no ip's found reverting to default")
-			}
-		} else {
-			// failed to scanallips falling back to defaults
-			gologger.Debug().Msgf("scanAllIps: dns resolution failed: %v", err)
-		}
-	}
-
-	ips := []string{}
-	// only scan the target but ipv6 if it has one
-	if i.ipOptions.IPV6 {
-		dialers := protocolstate.GetDialersWithId(executionId)
-		if dialers == nil {
-			gologger.Error().Msgf("dialers with executionId %s not found", executionId)
-			return
-		}
-
-		dnsData, err := dialers.Fastdialer.GetDNSData(urlx.Hostname())
-		if err == nil && len(dnsData.AAAA) > 0 {
-			// pick/ prefer 1st
-			ips = append(ips, dnsData.AAAA[0])
-		} else {
-			gologger.Warning().Msgf("target does not have ipv6 address falling back to ipv4 %v\n", err)
-		}
-	}
-	if i.ipOptions.IPV4 {
-		// if IPV4 is enabled do not specify ip let dialer handle it
-		ips = append(ips, "")
-	}
-
-	for _, ip := range ips {
+	for _, ip := range i.selectedIPs(executionId, urlx.Hostname()) {
 		metaInput := contextargs.NewMetaInput()
-		if ip != "" {
-			metaInput.Input = URL
-			metaInput.CustomIP = ip
-			i.setItem(metaInput, selection)
-		} else {
-			metaInput.Input = URL
-			i.setItem(metaInput, selection)
-		}
+		metaInput.Input = URL
+		metaInput.CustomIP = ip
+		i.setItem(metaInput, selection)
 	}
 }
 
@@ -493,75 +485,11 @@ func (i *ListInputProvider) Del(executionId string, value string) {
 		return
 	}
 
-	if i.ipOptions.ScanAllIPs {
-		// scan all ips
-		dialers := protocolstate.GetDialersWithId(executionId)
-		if dialers == nil {
-			gologger.Error().Msgf("dialers with executionId %s not found", executionId)
-			return
-		}
-
-		dnsData, err := dialers.Fastdialer.GetDNSData(urlx.Hostname())
-		if err == nil {
-			if (len(dnsData.A) + len(dnsData.AAAA)) > 0 {
-				var ips []string
-				if i.ipOptions.IPV4 {
-					ips = append(ips, dnsData.A...)
-				}
-				if i.ipOptions.IPV6 {
-					ips = append(ips, dnsData.AAAA...)
-				}
-				for _, ip := range ips {
-					if ip == "" {
-						continue
-					}
-					metaInput := contextargs.NewMetaInput()
-					metaInput.Input = value
-					metaInput.CustomIP = ip
-					i.delItem(metaInput)
-				}
-				return
-			} else {
-				gologger.Debug().Msgf("scanAllIps: no ip's found reverting to default")
-			}
-		} else {
-			// failed to scanallips falling back to defaults
-			gologger.Debug().Msgf("scanAllIps: dns resolution failed: %v", err)
-		}
-	}
-
-	ips := []string{}
-	// only scan the target but ipv6 if it has one
-	if i.ipOptions.IPV6 {
-		dialers := protocolstate.GetDialersWithId(executionId)
-		if dialers == nil {
-			gologger.Error().Msgf("dialers with executionId %s not found", executionId)
-			return
-		}
-
-		dnsData, err := dialers.Fastdialer.GetDNSData(urlx.Hostname())
-		if err == nil && len(dnsData.AAAA) > 0 {
-			// pick/ prefer 1st
-			ips = append(ips, dnsData.AAAA[0])
-		} else {
-			gologger.Warning().Msgf("target does not have ipv6 address falling back to ipv4 %v\n", err)
-		}
-	}
-	if i.ipOptions.IPV4 {
-		// if IPV4 is enabled do not specify ip let dialer handle it
-		ips = append(ips, "")
-	}
-
-	for _, ip := range ips {
+	for _, ip := range i.selectedIPs(executionId, urlx.Hostname()) {
 		metaInput := contextargs.NewMetaInput()
-		if ip != "" {
-			metaInput.Input = URL
-			metaInput.CustomIP = ip
-			i.delItem(metaInput)
-		} else {
-			metaInput.Input = URL
-			i.delItem(metaInput)
-		}
+		metaInput.Input = URL
+		metaInput.CustomIP = ip
+		i.delItem(metaInput)
 	}
 }
 

pkg/input/provider/list/ip_version_test.go

@@ -0,0 +1,106 @@
+package list
+
+import (
+	"net"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/miekg/dns"
+	"github.com/projectdiscovery/hmap/store/hybrid"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
+	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	"github.com/stretchr/testify/require"
+)
+
+// A local deterministic DNS responder for dual-stack, A-only and AAAA-only hosts.
+type ipVersionDNSHandler struct{}
+
+func (ipVersionDNSHandler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) {
+	reply := new(dns.Msg)
+	reply.SetReply(r)
+	if len(r.Question) == 0 {
+		_ = w.WriteMsg(reply)
+		return
+	}
+	question := r.Question[0]
+	// Some resolvers try search suffixes before the absolute name. Match the
+	// test host prefix rather than answering unrelated names as dual-stack.
+	host := strings.ToLower(question.Name)
+	if strings.HasPrefix(host, "empty.") {
+		_ = w.WriteMsg(reply)
+		return
+	}
+	isDual := strings.HasPrefix(host, "dual.")
+	isV4 := strings.HasPrefix(host, "v4only.")
+	isV6 := strings.HasPrefix(host, "v6only.")
+	if question.Qtype == dns.TypeA && (isDual || isV4) {
+		reply.Answer = append(reply.Answer, &dns.A{Hdr: dns.RR_Header{Name: question.Name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60}, A: net.ParseIP("192.0.2.10")})
+	}
+	if question.Qtype == dns.TypeAAAA && (isDual || isV6) {
+		reply.Answer = append(reply.Answer, &dns.AAAA{Hdr: dns.RR_Header{Name: question.Name, Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 60}, AAAA: net.ParseIP("2001:db8::10")})
+	}
+	_ = w.WriteMsg(reply)
+}
+
+func TestIPVersionsDefaultAndExplicitSelection(t *testing.T) {
+	conn, err := net.ListenPacket("udp", "127.0.0.1:0")
+	require.NoError(t, err)
+	started := make(chan struct{})
+	server := &dns.Server{PacketConn: conn, Handler: ipVersionDNSHandler{}, NotifyStartedFunc: func() { close(started) }}
+	go func() { _ = server.ActivateAndServe() }()
+	t.Cleanup(func() { _ = server.Shutdown() })
+	select {
+	case <-started:
+	case <-time.After(5 * time.Second):
+		t.Fatal("DNS test resolver did not start")
+	}
+	opts := types.DefaultOptions()
+	opts.ExecutionId = "ip-version-regression"
+	opts.InternalResolversList = []string{conn.LocalAddr().String()}
+	require.NoError(t, protocolstate.Init(opts))
+	t.Cleanup(func() { protocolstate.Close(opts.ExecutionId) })
+	for _, tc := range []struct {
+		name, hostname            string
+		scanAll, ipv4, ipv6, auto bool
+		want                      []string
+	}{
+		{"automatic dual-stack keeps dialer fallback", "dual.test", false, true, true, true, []string{""}},
+		{"automatic IPv6-only keeps dialer fallback", "v6only.test", false, true, true, true, []string{""}},
+		{"auto IPv4 fallback", "v4only.test", false, true, true, true, []string{""}},
+		{"auto empty DNS response fallback", "empty.test", false, true, true, true, []string{""}},
+		{"all empty DNS response fallback", "empty.test", true, true, true, true, []string{""}},
+		{"all dual", "dual.test", true, true, true, true, []string{"192.0.2.10", "2001:db8::10"}},
+		{"all IPv6 only", "v6only.test", true, true, true, true, []string{"2001:db8::10"}},
+		{"all IPv4 only", "v4only.test", true, true, true, true, []string{"192.0.2.10"}},
+		{"explicit IPv4 only", "dual.test", false, true, false, false, []string{""}},
+		{"explicit IPv6 only", "dual.test", false, false, true, false, []string{"2001:db8::10"}},
+		{"explicit both", "dual.test", false, true, true, false, []string{"2001:db8::10", ""}},
+		{"all explicit IPv6", "dual.test", true, false, true, false, []string{"2001:db8::10"}},
+		{"all explicit IPv4", "dual.test", true, true, false, false, []string{"192.0.2.10"}},
+		{"IPv6 restricted on IPv4-only", "v4only.test", true, false, true, false, nil},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			hm, err := hybrid.New(hybrid.DefaultDiskOptions)
+			require.NoError(t, err)
+			input := &ListInputProvider{
+				hostMap:       hm,
+				excludedHosts: make(map[string]struct{}),
+				ipOptions:     &ipOptions{ScanAllIPs: tc.scanAll, IPV4: tc.ipv4, IPV6: tc.ipv6, Auto: tc.auto},
+			}
+			t.Cleanup(input.Close)
+			input.Set(opts.ExecutionId, "https://"+tc.hostname)
+			var got []string
+			input.Iterate(func(value *contextargs.MetaInput) bool {
+				got = append(got, value.CustomIP)
+				return true
+			})
+			require.ElementsMatch(t, tc.want, got)
+			if tc.scanAll && len(tc.want) > 0 {
+				input.Del(opts.ExecutionId, "https://"+tc.hostname)
+				require.EqualValues(t, 0, input.Count(), "Del must remove the same IPs added by Set")
+			}
+		})
+	}
+}

pkg/input/provider/list/utils.go

@@ -4,4 +4,5 @@ type ipOptions struct {
 	ScanAllIPs bool
 	IPV4       bool
 	IPV6       bool
+	Auto       bool // -iv omitted: one unpinned target unless -sa scans every address
 }

pkg/input/transform.go

@@ -16,14 +16,6 @@ import (
 // Helper is a structure for helping with input transformation
 type Helper struct {
 	InputsHTTP *hybrid.HybridMap
-	// InputsHTTPProbed is true when InputsHTTP was filled by httpx probing.
-	// False means probing was skipped (e.g. MultiFormat) and map absence must
-	// not be treated as "not HTTP".
-	InputsHTTPProbed bool
-	// StrictProbe, when set, makes web-template input resolution skip inputs that
-	// were probed and found NOT to be HTTP services (instead of falling back to
-	// a raw URL). Lossless: a non-HTTP port has no HTTP application to find.
-	StrictProbe bool
 }
 
 // NewHelper returns a new input helper instance
@@ -148,12 +140,6 @@ func (h *Helper) convertInputToType(input string, inputType inputType, defaultPo
 			if probed, ok := h.InputsHTTP.Get(input); ok {
 				return string(probed)
 			}
-			// Strict reachability: this input was probed and is not an HTTP
-			// service, so skip it for web templates rather than falling back to
-			// a raw URL. Lossless — a non-HTTP port has no HTTP app to match.
-			if h.StrictProbe {
-				return ""
-			}
 		}
 		// try to parse it as absolute url and return
 		if absUrl, err := urlutil.ParseAbsoluteURL(input, false); err == nil {

pkg/input/transform_test.go

@@ -91,32 +91,3 @@ func TestConvertInputToType(t *testing.T) {
 		require.Equal(t, test.result, result, "could not get correct result %+v", test)
 	}
 }
-
-func TestStrictProbeSkipsUnconfirmedHTTPInput(t *testing.T) {
-	hm, err := hybrid.New(hybrid.DefaultDiskOptions)
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = hm.Close() })
-
-	_ = hm.Set("ok.example", []byte("https://ok.example"))
-
-	helper := &Helper{InputsHTTP: hm, StrictProbe: true}
-
-	// confirmed host -> resolved URL
-	require.Equal(t, "https://ok.example", helper.convertInputToType("ok.example", typeURL, ""))
-	// probed but non-HTTP hosts -> skipped (no raw fallback)
-	require.Equal(t, "", helper.convertInputToType("127.0.0.1:3306", typeURL, ""))
-	require.Equal(t, "", helper.convertInputToType("zombie.example:6379", typeURL, ""))
-	// already-URL inputs are not gated by probe results
-	require.Equal(t, "https://direct.example", helper.convertInputToType("https://direct.example", typeURL, ""))
-	// non-HTTP protocols still transform normally
-	require.Equal(t, "127.0.0.1:3306", helper.convertInputToType("127.0.0.1:3306", typeHostWithOptionalPort, ""))
-}
-
-func TestStrictProbeDisabledKeepsRawFallback(t *testing.T) {
-	hm, err := hybrid.New(hybrid.DefaultDiskOptions)
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = hm.Close() })
-
-	helper := &Helper{InputsHTTP: hm, StrictProbe: false}
-	require.Equal(t, "127.0.0.1:3306", helper.convertInputToType("127.0.0.1:3306", typeURL, ""))
-}

pkg/js/compiler/pool.go

@@ -16,10 +16,12 @@ import (
 	syncutil "github.com/projectdiscovery/utils/sync"
 
 	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libbytes"
+	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libcompress"
 	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libdcerpc"
 	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libdcom"
 	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libfs"
 	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libgrpc"
+	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libhash"
 	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libhttp"
 	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libikev2"
 	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/generated/go/libkerberos"

pkg/js/compiler/stream_libs_test.go

@@ -0,0 +1,158 @@
+package compiler
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"math/rand"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/andybalholm/brotli"
+	"github.com/klauspost/compress/gzip"
+	"github.com/klauspost/compress/zlib"
+	"github.com/klauspost/compress/zstd"
+	"github.com/projectdiscovery/dsl"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
+	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	httputil "github.com/projectdiscovery/utils/http"
+	"github.com/stretchr/testify/require"
+)
+
+const streamHashScript = `
+	const net = require('nuclei/net');
+	const compress = require('nuclei/compress');
+	const hash = require('nuclei/hash');
+
+	const conn = net.Open('tcp', %q);
+	conn.Send('GET / HTTP/1.1\r\nHost: test\r\nAccept-Encoding: gzip, deflate, br, zstd\r\n\r\n');
+	let head = '';
+	while (!head.endsWith('\r\n\r\n')) head += conn.RecvString(1);
+	const header = (name) => (head.match(new RegExp('\r\n' + name + ': ([^\r]+)', 'i')) || [])[1];
+	const length = header('Content-Length');
+	const coding = [header('Content-Encoding'), header('Transfer-Encoding')].filter(Boolean).join(', ');
+
+	const d = compress.NewDecoder(coding);
+	const sha = hash.New('sha256'), mmh3 = hash.New('mmh3');
+	const take = (out) => { sha.Write(out); mmh3.Write(out); };
+	if (length !== undefined) {
+		for (let read = 0; read < parseInt(length); ) {
+			const chunk = conn.Recv(65536);
+			read += chunk.length;
+			for (let out = d.Write(chunk); out.length > 0; out = d.Read()) take(out);
+		}
+		for (let out = d.Close(); out.length > 0; out = d.Read()) take(out);
+	} else {
+		while (!d.Finished()) {
+			for (let out = d.Write(conn.Recv(65536)); out.length > 0; out = d.Read()) take(out);
+		}
+	}
+	ExportAs('sha256', sha.Sum());
+	ExportAs('mmh3', mmh3.Sum());
+	true;
+`
+
+func encodeBody(t *testing.T, encoding string, data []byte) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	var w io.WriteCloser
+	switch encoding {
+	case "gzip":
+		w = gzip.NewWriter(&buf)
+	case "deflate":
+		w = zlib.NewWriter(&buf)
+	case "br":
+		w = brotli.NewWriter(&buf)
+	case "zstd":
+		zw, err := zstd.NewWriter(&buf)
+		require.NoError(t, err)
+		w = zw
+	case "":
+		return data
+	}
+	_, err := w.Write(data)
+	require.NoError(t, err)
+	require.NoError(t, w.Close())
+	return buf.Bytes()
+}
+
+func TestStreamingHashMatchesHTTPEngine(t *testing.T) {
+	rng := rand.New(rand.NewSource(1))
+	var body []byte
+	for len(body) < 6<<20 {
+		body = append(body, "large compressed response body "...)
+		body = append(body, byte(rng.Intn(256)))
+	}
+
+	executionID := t.Name()
+	require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: executionID}))
+	t.Cleanup(func() { protocolstate.Close(executionID) })
+
+	for _, tc := range []struct {
+		encoding string
+		chunked  bool
+	}{
+		{"gzip", false},
+		{"gzip", true},
+		{"deflate", false},
+		{"br", true},
+		{"zstd", true},
+		{"", true},
+	} {
+		t.Run(fmt.Sprintf("%s chunked=%v", tc.encoding, tc.chunked), func(t *testing.T) {
+			encoded := encodeBody(t, tc.encoding, body)
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				if tc.encoding != "" {
+					w.Header().Set("Content-Encoding", tc.encoding)
+				}
+				if !tc.chunked {
+					w.Header().Set("Content-Length", strconv.Itoa(len(encoded)))
+					_, _ = w.Write(encoded)
+					return
+				}
+				// Flushing before the handler returns makes the body chunked.
+				for rest := encoded; len(rest) > 0; {
+					n := min(len(rest), 50000)
+					_, _ = w.Write(rest[:n])
+					w.(http.Flusher).Flush()
+					rest = rest[n:]
+				}
+			}))
+			defer server.Close()
+
+			client := &http.Client{Transport: &http.Transport{DisableCompression: true}}
+			resp, err := client.Get(server.URL)
+			require.NoError(t, err)
+			if tc.chunked {
+				require.Equal(t, []string{"chunked"}, resp.TransferEncoding)
+			}
+			chain := httputil.NewResponseChain(resp, 64<<20)
+			defer chain.Close()
+			require.NoError(t, chain.Fill())
+			require.Equal(t, len(body), chain.Body().Len(), "the engine must decode the whole body")
+			engineBody := map[string]interface{}{"body": chain.BodyString()}
+			wantSha, err := dsl.EvalExpr("sha256(body)", engineBody)
+			require.NoError(t, err)
+			wantMmh3, err := dsl.EvalExpr("mmh3(body)", engineBody)
+			require.NoError(t, err)
+
+			serverURL, err := url.Parse(server.URL)
+			require.NoError(t, err)
+			script := fmt.Sprintf(streamHashScript, serverURL.Host)
+			compiled, err := SourceAutoMode(script, false)
+			require.NoError(t, err)
+			result, err := New().ExecuteWithOptions(t.Context(), compiled, NewExecuteArgs(), &ExecuteOptions{
+				ExecutionId:     executionID,
+				Source:          &script,
+				TimeoutVariants: &types.Timeouts{JsCompilerExecutionTimeout: 20 * time.Second},
+			})
+			require.NoError(t, err)
+			require.Equal(t, fmt.Sprint(wantSha), result["sha256"])
+			require.Equal(t, fmt.Sprint(wantMmh3), result["mmh3"])
+		})
+	}
+}

pkg/js/generated/go/libcompress/compress.go

@@ -0,0 +1,30 @@
+package compress
+
+import (
+	lib_compress "github.com/projectdiscovery/nuclei/v3/pkg/js/libs/compress"
+
+	"github.com/projectdiscovery/goja"
+	"github.com/projectdiscovery/nuclei/v3/pkg/js/gojs"
+)
+
+var (
+	module = gojs.NewGojaModule("nuclei/compress")
+)
+
+func init() {
+	module.Set(
+		gojs.Objects{
+			// Functions
+			"NewDecoder": lib_compress.NewDecoder,
+
+			// Var and consts
+
+			// Objects / Classes
+			"Decoder": gojs.GetClassConstructor[lib_compress.Decoder](&lib_compress.Decoder{}),
+		},
+	).Register()
+}
+
+func Enable(runtime *goja.Runtime) {
+	module.Enable(runtime)
+}

pkg/js/generated/go/libhash/hash.go

@@ -0,0 +1,30 @@
+package hash
+
+import (
+	lib_hash "github.com/projectdiscovery/nuclei/v3/pkg/js/libs/hash"
+
+	"github.com/projectdiscovery/goja"
+	"github.com/projectdiscovery/nuclei/v3/pkg/js/gojs"
+)
+
+var (
+	module = gojs.NewGojaModule("nuclei/hash")
+)
+
+func init() {
+	module.Set(
+		gojs.Objects{
+			// Functions
+			"New": lib_hash.New,
+
+			// Var and consts
+
+			// Objects / Classes
+			"Hash": gojs.GetClassConstructor[lib_hash.Hash](&lib_hash.Hash{}),
+		},
+	).Register()
+}
+
+func Enable(runtime *goja.Runtime) {
+	module.Enable(runtime)
+}

pkg/js/generated/ts/compress.ts

@@ -0,0 +1,102 @@
+
+
+/**
+ * NewDecoder returns a decoder for gzip, deflate, br, zstd, chunked or
+ * identity. List several comma-separated, in the order they were applied.
+ * @example
+ * ```javascript
+ * const compress = require('nuclei/compress');
+ * const d = compress.NewDecoder('br');
+ * // Content-Encoding: gzip sent with Transfer-Encoding: chunked
+ * const c = compress.NewDecoder('gzip, chunked');
+ * ```
+ */
+export function NewDecoder(encoding: string): Decoder | null {
+    return null;
+}
+
+
+
+/**
+ * Decoder decodes a compressed stream written to it in chunks. Each call
+ * returns up to about 1 MB; call Read until it returns nothing before the
+ * next Write.
+ * @example
+ * ```javascript
+ * const compress = require('nuclei/compress');
+ * const hash = require('nuclei/hash');
+ * const d = compress.NewDecoder('gzip');
+ * const h = hash.New('sha256');
+ * for (const chunk of chunks) {
+ *   for (let out = d.Write(chunk); out.length > 0; out = d.Read()) h.Write(out);
+ * }
+ * for (let out = d.Close(); out.length > 0; out = d.Read()) h.Write(out);
+ * log(h.Sum());
+ * ```
+ */
+export class Decoder {
+    
+
+    // Constructor of Decoder
+    constructor() {}
+    /**
+    * Write decodes the next chunk and returns up to about 1 MB of output.
+    * @example
+    * ```javascript
+    * const compress = require('nuclei/compress');
+    * const d = compress.NewDecoder('gzip');
+    * const out = d.Write(chunk);
+    * ```
+    */
+    public Write(chunk: Uint8Array): Uint8Array | null {
+        return null;
+    }
+    
+
+    /**
+    * Read returns more decoded bytes, or none once the decoder needs input.
+    * @example
+    * ```javascript
+    * const compress = require('nuclei/compress');
+    * const d = compress.NewDecoder('gzip');
+    * for (let out = d.Write(chunk); out.length > 0; out = d.Read()) log(out.length);
+    * ```
+    */
+    public Read(): Uint8Array | null {
+        return null;
+    }
+    
+
+    /**
+    * Close ends the input and returns the last decoded bytes. A truncated
+    * stream errors once the bytes before the cut have been returned.
+    * @example
+    * ```javascript
+    * const compress = require('nuclei/compress');
+    * const d = compress.NewDecoder('gzip');
+    * d.Write(chunk);
+    * for (let out = d.Close(); out.length > 0; out = d.Read()) log(out.length);
+    * ```
+    */
+    public Close(): Uint8Array | null {
+        return null;
+    }
+    
+
+    /**
+    * Finished reports whether a stream that marks its own end, such as a
+    * chunked body, has ended and all its output was read.
+    * @example
+    * ```javascript
+    * const compress = require('nuclei/compress');
+    * const d = compress.NewDecoder('gzip, chunked');
+    * while (!d.Finished()) for (let out = d.Write(conn.Recv(65536)); out.length > 0; out = d.Read()) log(out.length);
+    * ```
+    */
+    public Finished(): boolean {
+        return false;
+    }
+    
+
+}
+

pkg/js/generated/ts/hash.ts

@@ -0,0 +1,77 @@
+
+
+/**
+ * New returns a running hash for md5, sha1, sha256, sha512 or mmh3. Sum
+ * matches the DSL function of the same name.
+ * @example
+ * ```javascript
+ * const hash = require('nuclei/hash');
+ * const h = hash.New('sha256');
+ * ```
+ */
+export function New(algorithm: string): Hash | null {
+    return null;
+}
+
+
+
+/**
+ * Hash is a running hash.
+ * @example
+ * ```javascript
+ * const hash = require('nuclei/hash');
+ * const h = hash.New('sha256');
+ * h.WriteString('hello');
+ * log(h.Sum());
+ * ```
+ */
+export class Hash {
+    
+
+    // Constructor of Hash
+    constructor() {}
+    /**
+    * Write adds data to the hash.
+    * @example
+    * ```javascript
+    * const hash = require('nuclei/hash');
+    * const h = hash.New('sha256');
+    * h.Write([104, 105]);
+    * ```
+    */
+    public Write(data: Uint8Array): void {
+        return;
+    }
+    
+
+    /**
+    * WriteString adds the string data to the hash.
+    * @example
+    * ```javascript
+    * const hash = require('nuclei/hash');
+    * const h = hash.New('sha256');
+    * h.WriteString('hi');
+    * ```
+    */
+    public WriteString(data: string): void {
+        return;
+    }
+    
+
+    /**
+    * Sum returns the hash of the data written so far, formatted as the DSL does.
+    * @example
+    * ```javascript
+    * const hash = require('nuclei/hash');
+    * const h = hash.New('mmh3');
+    * h.WriteString('hello');
+    * log(h.Sum());
+    * ```
+    */
+    public Sum(): string | null {
+        return null;
+    }
+    
+
+}
+

pkg/js/generated/ts/index.ts

@@ -1,9 +1,11 @@
 export * as bytes from './bytes';
+export * as compress from './compress';
 export * as dcom from './dcom';
 export * as dcerpc from './dcerpc';
 export * as fs from './fs';
 export * as goconsole from './goconsole';
 export * as grpc from './grpc';
+export * as hash from './hash';
 export * as http from './http';
 export * as ikev2 from './ikev2';
 export * as kerberos from './kerberos';

pkg/js/libs/compress/decoder.go

@@ -0,0 +1,303 @@
+// Package compress exposes streaming decompression to nuclei JavaScript templates.
+package compress
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http/httputil"
+	"slices"
+	"strings"
+
+	"github.com/dsnet/compress/brotli"
+	"github.com/klauspost/compress/gzip"
+	"github.com/klauspost/compress/zlib"
+	"github.com/klauspost/compress/zstd"
+)
+
+// maxDecodedChunk caps the decoded bytes one call hands back.
+const maxDecodedChunk = 1 << 20
+
+type pauseReason int
+
+const (
+	needInput pauseReason = iota
+	outputFull
+	finished
+)
+
+type (
+	// Decoder decodes a compressed stream written to it in chunks. Each call
+	// returns up to about 1 MB; call Read until it returns nothing before the
+	// next Write.
+	// @example
+	// ```javascript
+	// const compress = require('nuclei/compress');
+	// const hash = require('nuclei/hash');
+	// const d = compress.NewDecoder('gzip');
+	// const h = hash.New('sha256');
+	// for (const chunk of chunks) {
+	//   for (let out = d.Write(chunk); out.length > 0; out = d.Read()) h.Write(out);
+	// }
+	// for (let out = d.Close(); out.length > 0; out = d.Read()) h.Write(out);
+	// log(h.Sum());
+	// ```
+	Decoder struct {
+		ctx context.Context
+		// The decode goroutine and the template's calls take turns over these
+		// channels, so no field is accessed by both at once.
+		resume chan []byte
+		paused chan pauseReason
+		done   chan struct{}
+		out    []byte
+
+		state      pauseReason
+		inputEnded bool
+
+		pending []byte
+		eof     bool
+		err     error
+	}
+)
+
+// NewDecoder returns a decoder for gzip, deflate, br, zstd, chunked or
+// identity. List several comma-separated, in the order they were applied.
+// @example
+// ```javascript
+// const compress = require('nuclei/compress');
+// const d = compress.NewDecoder('br');
+// // Content-Encoding: gzip sent with Transfer-Encoding: chunked
+// const c = compress.NewDecoder('gzip, chunked');
+// ```
+func NewDecoder(ctx context.Context, encoding string) (*Decoder, error) {
+	var opens []func(io.Reader) (io.Reader, func(), error)
+	for name := range strings.SplitSeq(encoding, ",") {
+		switch name = strings.ToLower(strings.TrimSpace(name)); name {
+		case "gzip":
+			opens = append(opens, func(r io.Reader) (io.Reader, func(), error) {
+				zr, err := gzip.NewReader(r)
+				return zr, func() {}, err
+			})
+		case "deflate":
+			opens = append(opens, func(r io.Reader) (io.Reader, func(), error) {
+				zr, err := zlib.NewReader(r)
+				return zr, func() {}, err
+			})
+		case "br":
+			opens = append(opens, func(r io.Reader) (io.Reader, func(), error) {
+				br, err := brotli.NewReader(r, nil)
+				return br, func() {}, err
+			})
+		case "zstd":
+			opens = append(opens, func(r io.Reader) (io.Reader, func(), error) {
+				// One goroutine so no background workers outlive the stream.
+				zr, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(1))
+				if err != nil {
+					return nil, nil, err
+				}
+				return zr, zr.Close, nil
+			})
+		case "chunked":
+			opens = append(opens, func(r io.Reader) (io.Reader, func(), error) {
+				return httputil.NewChunkedReader(r), func() {}, nil
+			})
+		case "", "identity":
+		default:
+			return nil, fmt.Errorf("unsupported encoding %q, expected gzip, deflate, br, zstd, chunked or identity", name)
+		}
+	}
+	slices.Reverse(opens)
+	open := func(r io.Reader) (io.Reader, func(), error) {
+		var closers []func()
+		closeAll := func() {
+			for _, c := range slices.Backward(closers) {
+				c()
+			}
+		}
+		for _, o := range opens {
+			next, c, err := o(r)
+			if err != nil {
+				closeAll()
+				return nil, nil, err
+			}
+			r = next
+			closers = append(closers, c)
+		}
+		return r, closeAll, nil
+	}
+	d := &Decoder{
+		ctx:    ctx,
+		resume: make(chan []byte),
+		paused: make(chan pauseReason),
+		done:   make(chan struct{}),
+	}
+	go d.run(open)
+	d.wait()
+	return d, nil
+}
+
+// Write decodes the next chunk and returns up to about 1 MB of output.
+// @example
+// ```javascript
+// const compress = require('nuclei/compress');
+// const d = compress.NewDecoder('gzip');
+// const out = d.Write(chunk);
+// ```
+func (d *Decoder) Write(chunk []byte) ([]byte, error) {
+	if d.done == nil {
+		return nil, errNotCreatedByNewDecoder
+	}
+	if d.inputEnded {
+		return nil, errors.New("decoder is closed")
+	}
+	if d.state == outputFull {
+		return nil, errors.New("decoded bytes are waiting: call Read until it returns no bytes before writing more")
+	}
+	if d.state == needInput && len(chunk) > 0 {
+		// The decode goroutine may still hold the chunk after Write returns.
+		d.resumeWith(bytes.Clone(chunk))
+	}
+	return d.result()
+}
+
+// Read returns more decoded bytes, or none once the decoder needs input.
+// @example
+// ```javascript
+// const compress = require('nuclei/compress');
+// const d = compress.NewDecoder('gzip');
+// for (let out = d.Write(chunk); out.length > 0; out = d.Read()) log(out.length);
+// ```
+func (d *Decoder) Read() ([]byte, error) {
+	if d.done == nil {
+		return nil, errNotCreatedByNewDecoder
+	}
+	d.advance()
+	return d.result()
+}
+
+// Close ends the input and returns the last decoded bytes. A truncated
+// stream errors once the bytes before the cut have been returned.
+// @example
+// ```javascript
+// const compress = require('nuclei/compress');
+// const d = compress.NewDecoder('gzip');
+// d.Write(chunk);
+// for (let out = d.Close(); out.length > 0; out = d.Read()) log(out.length);
+// ```
+func (d *Decoder) Close() ([]byte, error) {
+	if d.done == nil {
+		return nil, errNotCreatedByNewDecoder
+	}
+	d.inputEnded = true
+	d.advance()
+	return d.result()
+}
+
+// Finished reports whether a stream that marks its own end, such as a
+// chunked body, has ended and all its output was read.
+// @example
+// ```javascript
+// const compress = require('nuclei/compress');
+// const d = compress.NewDecoder('gzip, chunked');
+// while (!d.Finished()) for (let out = d.Write(conn.Recv(65536)); out.length > 0; out = d.Read()) log(out.length);
+// ```
+func (d *Decoder) Finished() bool {
+	return d.state == finished && len(d.out) == 0
+}
+
+var errNotCreatedByNewDecoder = errors.New("decoder was not created with compress.NewDecoder")
+
+func (d *Decoder) advance() {
+	for len(d.out) == 0 && (d.state == outputFull || d.state == needInput && d.inputEnded) {
+		d.resumeWith(nil)
+	}
+}
+
+func (d *Decoder) result() ([]byte, error) {
+	out := d.out
+	d.out = nil
+	if len(out) == 0 && d.state == finished {
+		return nil, d.err
+	}
+	return out, nil
+}
+
+func (d *Decoder) resumeWith(chunk []byte) {
+	select {
+	case d.resume <- chunk:
+		d.wait()
+	case <-d.done:
+		d.state = finished
+	}
+}
+
+func (d *Decoder) wait() {
+	select {
+	case d.state = <-d.paused:
+	case <-d.done:
+		d.state = finished
+	}
+}
+
+func (d *Decoder) run(open func(io.Reader) (io.Reader, func(), error)) {
+	defer close(d.done)
+	r, closeReader, err := open(decoderInput{d})
+	if err != nil {
+		d.err = err
+		return
+	}
+	defer closeReader()
+	_, err = io.Copy(decoderOutput{d}, r)
+	d.err = err
+}
+
+func (d *Decoder) pause(reason pauseReason) ([]byte, error) {
+	select {
+	case d.paused <- reason:
+	case <-d.ctx.Done():
+		return nil, context.Cause(d.ctx)
+	}
+	select {
+	case chunk := <-d.resume:
+		return chunk, nil
+	case <-d.ctx.Done():
+		return nil, context.Cause(d.ctx)
+	}
+}
+
+type decoderInput struct{ d *Decoder }
+
+func (in decoderInput) Read(p []byte) (int, error) {
+	d := in.d
+	for len(d.pending) == 0 {
+		if d.eof {
+			return 0, io.EOF
+		}
+		chunk, err := d.pause(needInput)
+		if err != nil {
+			return 0, err
+		}
+		if chunk == nil {
+			d.eof = true
+		}
+		d.pending = chunk
+	}
+	n := copy(p, d.pending)
+	d.pending = d.pending[n:]
+	return n, nil
+}
+
+type decoderOutput struct{ d *Decoder }
+
+func (out decoderOutput) Write(p []byte) (int, error) {
+	d := out.d
+	d.out = append(d.out, p...)
+	if len(d.out) >= maxDecodedChunk {
+		if _, err := d.pause(outputFull); err != nil {
+			return 0, err
+		}
+	}
+	return len(p), nil
+}

pkg/js/libs/compress/decoder_test.go

@@ -0,0 +1,192 @@
+package compress
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"math/rand"
+	"net/http/httputil"
+	"testing"
+	"time"
+
+	"github.com/andybalholm/brotli"
+	"github.com/klauspost/compress/gzip"
+	"github.com/klauspost/compress/zlib"
+	"github.com/klauspost/compress/zstd"
+	"github.com/stretchr/testify/require"
+)
+
+func compressed(t *testing.T, encoding string, data []byte) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	var w io.WriteCloser
+	switch encoding {
+	case "gzip":
+		w = gzip.NewWriter(&buf)
+	case "deflate":
+		w = zlib.NewWriter(&buf)
+	case "br":
+		w = brotli.NewWriter(&buf)
+	case "zstd":
+		zw, err := zstd.NewWriter(&buf)
+		require.NoError(t, err)
+		w = zw
+	case "identity":
+		return data
+	}
+	_, err := w.Write(data)
+	require.NoError(t, err)
+	require.NoError(t, w.Close())
+	return buf.Bytes()
+}
+
+func decodeInChunks(t *testing.T, d *Decoder, input []byte) ([]byte, error) {
+	t.Helper()
+	rng := rand.New(rand.NewSource(1))
+	var decoded []byte
+	collect := func(out []byte, err error) bool {
+		require.LessOrEqual(t, len(out), maxDecodedChunk+64<<10)
+		decoded = append(decoded, out...)
+		return err == nil && len(out) > 0
+	}
+	drain := func(out []byte, err error) error {
+		for collect(out, err) {
+			out, err = d.Read()
+		}
+		return err
+	}
+	for len(input) > 0 {
+		n := min(len(input), 1+rng.Intn(70000))
+		out, err := d.Write(input[:n])
+		if err := drain(out, err); err != nil {
+			return decoded, err
+		}
+		input = input[n:]
+	}
+	out, err := d.Close()
+	return decoded, drain(out, err)
+}
+
+func TestDecoderMatchesWholeBodyDecode(t *testing.T) {
+	data := bytes.Repeat([]byte("nuclei streaming decoder "), 400000)
+	for _, encoding := range []string{"gzip", "deflate", "br", "zstd", "identity"} {
+		t.Run(encoding, func(t *testing.T) {
+			d, err := NewDecoder(t.Context(), encoding)
+			require.NoError(t, err)
+			decoded, err := decodeInChunks(t, d, compressed(t, encoding, data))
+			require.NoError(t, err)
+			require.Equal(t, data, decoded)
+		})
+	}
+}
+
+func chunked(t *testing.T, data []byte) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	w := httputil.NewChunkedWriter(&buf)
+	for len(data) > 0 {
+		n := min(len(data), 8000)
+		_, err := w.Write(data[:n])
+		require.NoError(t, err)
+		data = data[n:]
+	}
+	require.NoError(t, w.Close())
+	buf.WriteString("\r\n")
+	return buf.Bytes()
+}
+
+func TestDecoderDecodesChunkedTransfer(t *testing.T) {
+	data := bytes.Repeat([]byte("nuclei chunked decoder "), 400000)
+	for _, encoding := range []string{"gzip", "deflate", "br", "zstd", "identity"} {
+		t.Run(encoding, func(t *testing.T) {
+			d, err := NewDecoder(t.Context(), encoding+", chunked")
+			require.NoError(t, err)
+			decoded, err := decodeInChunks(t, d, chunked(t, compressed(t, encoding, data)))
+			require.NoError(t, err)
+			require.Equal(t, data, decoded)
+		})
+	}
+}
+
+func TestDecoderFinishesAtLastChunk(t *testing.T) {
+	data := bytes.Repeat([]byte("finished "), 1000)
+	d, err := NewDecoder(t.Context(), "gzip, chunked")
+	require.NoError(t, err)
+	input := chunked(t, compressed(t, "gzip", data))
+
+	half := len(input) / 2
+	_, err = d.Write(input[:half])
+	require.NoError(t, err)
+	require.False(t, d.Finished())
+
+	out, err := d.Write(input[half:])
+	require.NoError(t, err)
+	var decoded []byte
+	for ; len(out) > 0; out, err = d.Read() {
+		decoded = append(decoded, out...)
+	}
+	require.NoError(t, err)
+	require.Equal(t, data, decoded)
+	require.True(t, d.Finished(), "decoder must finish without Close once the last chunk arrives")
+}
+
+func TestDecoderAppliesCodingsInReverse(t *testing.T) {
+	data := bytes.Repeat([]byte("two codings "), 100000)
+	input := compressed(t, "br", compressed(t, "gzip", data))
+	d, err := NewDecoder(t.Context(), "gzip, br")
+	require.NoError(t, err)
+	decoded, err := decodeInChunks(t, d, input)
+	require.NoError(t, err)
+	require.Equal(t, data, decoded)
+}
+
+func TestDecoderReportsTruncatedStream(t *testing.T) {
+	data := bytes.Repeat([]byte("truncated "), 200000)
+	for _, encoding := range []string{"gzip", "deflate", "br", "zstd", "identity, chunked"} {
+		t.Run(encoding, func(t *testing.T) {
+			var input []byte
+			if encoding == "identity, chunked" {
+				input = chunked(t, data)
+			} else {
+				input = compressed(t, encoding, data)
+			}
+			d, err := NewDecoder(t.Context(), encoding)
+			require.NoError(t, err)
+			decoded, err := decodeInChunks(t, d, input[:len(input)/2])
+			require.Error(t, err)
+			require.True(t, bytes.HasPrefix(data, decoded), "bytes before the cut must still be handed back")
+		})
+	}
+}
+
+func TestDecoderStopsWhenContextEnds(t *testing.T) {
+	ctx, cancel := context.WithCancel(t.Context())
+	d, err := NewDecoder(ctx, "gzip")
+	require.NoError(t, err)
+	_, err = d.Write(compressed(t, "gzip", []byte("abandoned"))[:5])
+	require.NoError(t, err)
+
+	cancel()
+	select {
+	case <-d.done:
+	case <-time.After(time.Second):
+		t.Fatal("decode goroutine outlived its context")
+	}
+}
+
+func TestNewDecoderRejectsUnknownEncoding(t *testing.T) {
+	_, err := NewDecoder(t.Context(), "compress")
+	require.Error(t, err)
+	_, err = NewDecoder(t.Context(), "gzip, compress")
+	require.Error(t, err)
+}
+
+func TestDecoderNotCreatedByNewDecoderErrors(t *testing.T) {
+	var d Decoder
+	_, err := d.Write([]byte("x"))
+	require.ErrorIs(t, err, errNotCreatedByNewDecoder)
+	_, err = d.Read()
+	require.ErrorIs(t, err, errNotCreatedByNewDecoder)
+	_, err = d.Close()
+	require.ErrorIs(t, err, errNotCreatedByNewDecoder)
+}

pkg/js/libs/hash/hash.go

@@ -0,0 +1,100 @@
+// Package hash exposes incremental hashes to nuclei JavaScript templates.
+package hash
+
+import (
+	"crypto/md5"
+	"crypto/sha1"
+	"crypto/sha256"
+	"crypto/sha512"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"hash"
+	"strings"
+
+	"github.com/spaolacci/murmur3"
+)
+
+type (
+	// Hash is a running hash.
+	// @example
+	// ```javascript
+	// const hash = require('nuclei/hash');
+	// const h = hash.New('sha256');
+	// h.WriteString('hello');
+	// log(h.Sum());
+	// ```
+	Hash struct {
+		h    hash.Hash
+		mmh3 bool
+	}
+)
+
+// New returns a running hash for md5, sha1, sha256, sha512 or mmh3. Sum
+// matches the DSL function of the same name.
+// @example
+// ```javascript
+// const hash = require('nuclei/hash');
+// const h = hash.New('sha256');
+// ```
+func New(algorithm string) (*Hash, error) {
+	switch strings.ToLower(algorithm) {
+	case "md5":
+		return &Hash{h: md5.New()}, nil
+	case "sha1":
+		return &Hash{h: sha1.New()}, nil
+	case "sha256":
+		return &Hash{h: sha256.New()}, nil
+	case "sha512":
+		return &Hash{h: sha512.New()}, nil
+	case "mmh3":
+		return &Hash{h: murmur3.New32WithSeed(0), mmh3: true}, nil
+	}
+	return nil, fmt.Errorf("unsupported hash algorithm %q, expected md5, sha1, sha256, sha512 or mmh3", algorithm)
+}
+
+// Write adds data to the hash.
+// @example
+// ```javascript
+// const hash = require('nuclei/hash');
+// const h = hash.New('sha256');
+// h.Write([104, 105]);
+// ```
+func (h *Hash) Write(data []byte) error {
+	if h.h == nil {
+		return errNotCreatedByNew
+	}
+	_, _ = h.h.Write(data)
+	return nil
+}
+
+// WriteString adds the string data to the hash.
+// @example
+// ```javascript
+// const hash = require('nuclei/hash');
+// const h = hash.New('sha256');
+// h.WriteString('hi');
+// ```
+func (h *Hash) WriteString(data string) error {
+	return h.Write([]byte(data))
+}
+
+// Sum returns the hash of the data written so far, formatted as the DSL does.
+// @example
+// ```javascript
+// const hash = require('nuclei/hash');
+// const h = hash.New('mmh3');
+// h.WriteString('hello');
+// log(h.Sum());
+// ```
+func (h *Hash) Sum() (string, error) {
+	if h.h == nil {
+		return "", errNotCreatedByNew
+	}
+	if h.mmh3 {
+		return fmt.Sprintf("%d", int32(h.h.(hash.Hash32).Sum32())), nil
+	}
+	return hex.EncodeToString(h.h.Sum(nil)), nil
+}
+
+var errNotCreatedByNew = errors.New("hash was not created with hash.New")

pkg/js/libs/hash/hash_test.go

@@ -0,0 +1,43 @@
+package hash
+
+import (
+	"testing"
+
+	"github.com/projectdiscovery/dsl"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSumMatchesDSLAcrossChunks(t *testing.T) {
+	data := make([]byte, 3<<20+7)
+	for i := range data {
+		data[i] = byte(i * 31)
+	}
+	for _, algorithm := range []string{"md5", "sha1", "sha256", "sha512", "mmh3"} {
+		t.Run(algorithm, func(t *testing.T) {
+			h, err := New(algorithm)
+			require.NoError(t, err)
+			for chunk := data; len(chunk) > 0; {
+				n := min(len(chunk), 65521)
+				require.NoError(t, h.Write(chunk[:n]))
+				chunk = chunk[n:]
+			}
+			want, err := dsl.EvalExpr(algorithm+"(body)", map[string]interface{}{"body": string(data)})
+			require.NoError(t, err)
+			sum, err := h.Sum()
+			require.NoError(t, err)
+			require.Equal(t, want, sum)
+		})
+	}
+}
+
+func TestNewRejectsUnknownAlgorithm(t *testing.T) {
+	_, err := New("crc32")
+	require.Error(t, err)
+}
+
+func TestHashNotCreatedByNewErrors(t *testing.T) {
+	var h Hash
+	require.ErrorIs(t, h.WriteString("x"), errNotCreatedByNew)
+	_, err := h.Sum()
+	require.ErrorIs(t, err, errNotCreatedByNew)
+}

pkg/js/libs/mysql/mysql_fingerprint_client_test.go

@@ -2,7 +2,11 @@ package mysql
 
 import (
 	"context"
+	"fmt"
 	"net"
+	"os"
+	"path/filepath"
+	"runtime"
 	"strings"
 	"sync/atomic"
 	"testing"
@@ -169,6 +173,84 @@ func TestFingerprintMySQLDeniedByNetworkPolicy(t *testing.T) {
 	require.Contains(t, err.Error(), "127.0.0.1")
 }
 
+func TestConnectWithDSNDeniedByNetworkPolicy(t *testing.T) {
+	t.Parallel()
+
+	executionID := "mysql-dsn-deny-" + strings.NewReplacer("/", "-", " ", "-").Replace(t.Name())
+	require.NoError(t, protocolstate.Init(&types.Options{
+		ExecutionId:                executionID,
+		RestrictLocalNetworkAccess: true,
+	}))
+	t.Cleanup(func() { protocolstate.Close(executionID) })
+	ctx := context.WithValue(context.Background(), "executionId", executionID) //nolint:staticcheck
+
+	var accepted atomic.Int32
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = ln.Close() })
+	go func() {
+		for {
+			conn, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			accepted.Add(1)
+			_ = conn.Close()
+		}
+	}()
+
+	port := ln.Addr().(*net.TCPAddr).Port
+	dsn := fmt.Sprintf("vuser:vpass@tcp(127.0.0.1:%d)/vdb", port)
+	ok, err := (&MySQLClient{}).ConnectWithDSN(ctx, dsn)
+	require.Error(t, err)
+	require.False(t, ok)
+	require.Contains(t, err.Error(), "127.0.0.1")
+	require.Zero(t, accepted.Load())
+}
+
+func TestConnectWithDSNUnixDeniedByNetworkPolicy(t *testing.T) {
+	t.Parallel()
+
+	executionID := "mysql-dsn-unix-deny-" + strings.NewReplacer("/", "-", " ", "-").Replace(t.Name())
+	require.NoError(t, protocolstate.Init(&types.Options{
+		ExecutionId:                executionID,
+		RestrictLocalNetworkAccess: true,
+	}))
+	t.Cleanup(func() { protocolstate.Close(executionID) })
+	ctx := context.WithValue(context.Background(), "executionId", executionID) //nolint:staticcheck
+
+	// t.TempDir can exceed the unix socket path limit on macOS, so use a short dir.
+	dir, err := os.MkdirTemp("", "nuclei-mysql")
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = os.RemoveAll(dir) })
+	sock := filepath.Join(dir, "lna.sock")
+	ln, err := net.Listen("unix", sock)
+	if err != nil && runtime.GOOS == "windows" {
+		t.Skipf("unix sockets not available: %v", err)
+	}
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = ln.Close() })
+
+	var accepted atomic.Int32
+	go func() {
+		for {
+			conn, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			accepted.Add(1)
+			_ = conn.Close()
+		}
+	}()
+
+	dsn := fmt.Sprintf("vuser:vpass@unix(%s)/vdb", sock)
+	ok, err := (&MySQLClient{}).ConnectWithDSN(ctx, dsn)
+	require.Error(t, err)
+	require.False(t, ok)
+	require.Contains(t, err.Error(), "127.0.0.1")
+	require.Zero(t, accepted.Load())
+}
+
 func TestFingerprintMySQLVersionNeverEmptyOnHandshakeFixtures(t *testing.T) {
 	t.Parallel()
 

pkg/js/libs/mysql/mysql_private.go

@@ -76,24 +76,57 @@ func BuildDSN(opts MySQLOptions) (string, error) {
 	return dsn.String(), nil
 }
 
-// sandboxDSN enforces the local file access sandbox on a MySQL DSN. The
-// driver's allowAllFiles option lets a malicious server read any local file
-// off the host via LOAD DATA LOCAL INFILE, so it is only honored when -lfa is
-// enabled, mirroring the fs.ReadFile restriction.
-func sandboxDSN(dsn string, lfaAllowed bool) (string, error) {
+// sandboxDSN enforces local file and network policy on a MySQL DSN.
+// allowAllFiles is only honored when -lfa is enabled. tcp, tcp4 and tcp6
+// DSNs are rewritten to nucleitcp so the driver uses the fastdialer
+// registered in protocolstate. Unix sockets are treated as loopback for -lna.
+// Other networks are rejected.
+func sandboxDSN(executionId, dsn string) (string, error) {
 	cfg, err := mysql.ParseDSN(dsn)
 	if err != nil {
 		return "", err
 	}
-	if cfg.AllowAllFiles && !lfaAllowed {
+	opts := &types.Options{ExecutionId: executionId}
+	if cfg.AllowAllFiles && !protocolstate.IsLfaAllowed(opts) {
 		cfg.AllowAllFiles = false
 	}
+
+	netName := cfg.Net
+	if netName == "" {
+		netName = "tcp"
+	}
+	switch netName {
+	case "unix":
+		// IP-based policy never sees a socket path; gate on loopback like ldapi.
+		const unixPolicyHost = "127.0.0.1"
+		if !protocolstate.IsHostAllowed(executionId, unixPolicyHost) {
+			return "", protocolstate.ErrHostDenied.Msgf(unixPolicyHost)
+		}
+	case "tcp", "tcp4", "tcp6", "nucleitcp":
+		// Only nucleitcp has a registered dialer; the others would use the
+		// driver's plain net.Dialer and skip the dial-time policy check.
+		cfg.Net = "nucleitcp"
+		host := dsnPolicyHost(cfg.Addr)
+		if !protocolstate.IsHostAllowed(executionId, host) {
+			return "", protocolstate.ErrHostDenied.Msgf(host)
+		}
+	default:
+		return "", fmt.Errorf("unsupported mysql network %q", netName)
+	}
 	return cfg.FormatDSN(), nil
 }
 
+func dsnPolicyHost(addr string) string {
+	host, _, err := net.SplitHostPort(addr)
+	if err == nil && host != "" {
+		return host
+	}
+	return addr
+}
+
 // openDB opens a sandboxed MySQL connection from dsn.
 func openDB(executionId, dsn string) (*sql.DB, error) {
-	dsn, err := sandboxDSN(dsn, protocolstate.IsLfaAllowed(&types.Options{ExecutionId: executionId}))
+	dsn, err := sandboxDSN(executionId, dsn)
 	if err != nil {
 		return nil, err
 	}

pkg/js/libs/mysql/mysql_private_test.go

@@ -5,11 +5,14 @@ import (
 
 	"github.com/go-sql-driver/mysql"
 	"github.com/stretchr/testify/require"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
+	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 )
 
 func TestSandboxDSN(t *testing.T) {
 	t.Run("strips allowAllFiles when lfa disabled", func(t *testing.T) {
-		got, err := sandboxDSN("root:x@nucleitcp(127.0.0.1:3306)/?allowAllFiles=true", false)
+		got, err := sandboxDSN("", "root:x@nucleitcp(127.0.0.1:3306)/?allowAllFiles=true")
 		require.NoError(t, err)
 
 		cfg, err := mysql.ParseDSN(got)
@@ -18,7 +21,14 @@ func TestSandboxDSN(t *testing.T) {
 	})
 
 	t.Run("keeps allowAllFiles when lfa enabled", func(t *testing.T) {
-		got, err := sandboxDSN("root:x@nucleitcp(127.0.0.1:3306)/?allowAllFiles=true", true)
+		executionID := "mysql-lfa-" + t.Name()
+		require.NoError(t, protocolstate.Init(&types.Options{
+			ExecutionId:          executionID,
+			AllowLocalFileAccess: true,
+		}))
+		t.Cleanup(func() { protocolstate.Close(executionID) })
+
+		got, err := sandboxDSN(executionID, "root:x@nucleitcp(127.0.0.1:3306)/?allowAllFiles=true")
 		require.NoError(t, err)
 
 		cfg, err := mysql.ParseDSN(got)
@@ -27,7 +37,7 @@ func TestSandboxDSN(t *testing.T) {
 	})
 
 	t.Run("leaves dsn without allowAllFiles untouched", func(t *testing.T) {
-		got, err := sandboxDSN("root:x@nucleitcp(127.0.0.1:3306)/", false)
+		got, err := sandboxDSN("", "root:x@nucleitcp(127.0.0.1:3306)/")
 		require.NoError(t, err)
 
 		cfg, err := mysql.ParseDSN(got)
@@ -35,8 +45,80 @@ func TestSandboxDSN(t *testing.T) {
 		require.False(t, cfg.AllowAllFiles)
 	})
 
+	t.Run("rewrites tcp to nucleitcp", func(t *testing.T) {
+		got, err := sandboxDSN("", "root:x@tcp(acme.com:3306)/")
+		require.NoError(t, err)
+
+		cfg, err := mysql.ParseDSN(got)
+		require.NoError(t, err)
+		require.Equal(t, "nucleitcp", cfg.Net)
+		require.Equal(t, "acme.com:3306", cfg.Addr)
+	})
+
+	t.Run("rewrites tcp4 and tcp6 to nucleitcp", func(t *testing.T) {
+		for _, netName := range []string{"tcp4", "tcp6"} {
+			got, err := sandboxDSN("", "root:x@"+netName+"(acme.com:3306)/")
+			require.NoError(t, err)
+
+			cfg, err := mysql.ParseDSN(got)
+			require.NoError(t, err)
+			require.Equal(t, "nucleitcp", cfg.Net)
+			require.Equal(t, "acme.com:3306", cfg.Addr)
+		}
+	})
+
+	t.Run("rejects unsupported network", func(t *testing.T) {
+		_, err := sandboxDSN("", "root:x@udp(acme.com:3306)/")
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "udp")
+	})
+
 	t.Run("errors on invalid dsn", func(t *testing.T) {
-		_, err := sandboxDSN("::not-a-dsn::", false)
+		_, err := sandboxDSN("", "::not-a-dsn::")
 		require.Error(t, err)
 	})
+
+	t.Run("denies loopback tcp when lna enabled", func(t *testing.T) {
+		executionID := "mysql-lna-tcp-" + t.Name()
+		require.NoError(t, protocolstate.Init(&types.Options{
+			ExecutionId:                executionID,
+			RestrictLocalNetworkAccess: true,
+		}))
+		t.Cleanup(func() { protocolstate.Close(executionID) })
+
+		_, err := sandboxDSN(executionID, "vuser:vpass@tcp(127.0.0.1:3306)/vdb")
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "127.0.0.1")
+	})
+
+	t.Run("denies loopback variants when lna enabled", func(t *testing.T) {
+		executionID := "mysql-lna-variants-" + t.Name()
+		require.NoError(t, protocolstate.Init(&types.Options{
+			ExecutionId:                executionID,
+			RestrictLocalNetworkAccess: true,
+		}))
+		t.Cleanup(func() { protocolstate.Close(executionID) })
+
+		for _, dsn := range []string{
+			"vuser:vpass@tcp4(127.0.0.1:3306)/vdb",
+			"vuser:vpass@tcp6([::1]:3306)/vdb",
+			"vuser:vpass@tcp([::ffff:127.0.0.1]:3306)/vdb",
+		} {
+			_, err := sandboxDSN(executionID, dsn)
+			require.Error(t, err, dsn)
+		}
+	})
+
+	t.Run("denies unix socket when lna enabled", func(t *testing.T) {
+		executionID := "mysql-lna-unix-" + t.Name()
+		require.NoError(t, protocolstate.Init(&types.Options{
+			ExecutionId:                executionID,
+			RestrictLocalNetworkAccess: true,
+		}))
+		t.Cleanup(func() { protocolstate.Close(executionID) })
+
+		_, err := sandboxDSN(executionID, "vuser:vpass@unix(/tmp/nuclei-mysql-lna.sock)/vdb")
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "127.0.0.1")
+	})
 }

pkg/js/libs/redis/memo.redis.go

@@ -3,7 +3,6 @@ package redis
 
 import (
 	"context"
-
 	"errors"
 
 	"fmt"

pkg/js/libs/redis/redis.go

@@ -3,13 +3,12 @@ package redis
 import (
 	"bytes"
 	"context"
-	"errors"
 	"fmt"
-	"io"
 	"net"
 	"strconv"
 	"time"
 
+	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/redis/go-redis/v9"
 
@@ -269,9 +268,10 @@ var errReadLimit = fmt.Errorf("redis: reply exceeds %d bytes", maxReadBytes)
 // maxReadBytes.
 func newClient(addr, password string) *redis.Client {
 	opts := &redis.Options{
-		Addr:     addr,
-		Password: password,
-		DB:       0,
+		Addr:          addr,
+		Password:      password,
+		DB:            0,
+		DialerRetries: dialAttempts,
 	}
 	dial := redis.NewDialer(opts)
 	opts.Dialer = func(ctx context.Context, network, addr string) (net.Conn, error) {
@@ -285,138 +285,134 @@ func newClient(addr, password string) *redis.Client {
 	return redis.NewClient(opts)
 }
 
-// limitedConn fails a read once remaining is spent, and withholds a RESP length line whose
-// declared size is over the cap so go-redis never allocates it.
+// maxHeaderLen fits a type byte, any int64, and "\r". Redis never sends a longer length header.
+const maxHeaderLen = 32
+
+// limitedConn fails a read once remaining is spent, and ends a read before the "\n" of a RESP
+// length header whose declared size is over the cap, so go-redis never gets the line to allocate
+// from.
+//
+// It follows the reply's framing as the bytes pass: a bulk payload is skipped by its declared
+// length, so only header lines are checked. A payload line such as "*abc" is data.
 type limitedConn struct {
 	net.Conn
 	remaining int64
-	pending   []byte
-	partial   []byte
-	err       error
-	scratch   [32 << 10]byte
+	// payload is how many bytes of the current bulk payload, "\r\n" included, are still to come.
+	payload int64
+	// header is the start of the current header line, without its "\n".
+	header   []byte
+	overlong bool
+	err      error
 }
 
 func (c *limitedConn) Read(b []byte) (int, error) {
-	if len(b) == 0 {
+	if c.err != nil {
+		return 0, c.err
+	}
+	if c.remaining <= 0 {
+		return 0, errReadLimit
+	}
+	if int64(len(b)) > c.remaining {
+		b = b[:c.remaining]
+	}
+	n, err := c.Conn.Read(b)
+	c.remaining -= int64(n)
+	// Errors from the connection, deadlines included, are returned as they come and not kept: a
+	// read deadline belongs to one call. go-redis peeks for push notifications with a few
+	// microseconds of deadline, and keeping that timeout would fail the command that follows.
+	kept, scanErr := c.scan(b[:n])
+	if scanErr == nil {
+		return n, err
+	}
+	c.err = scanErr
+	if kept == 0 {
+		return 0, scanErr
+	}
+	return kept, nil
+}
+
+// scan follows the RESP framing of p and returns how many of its bytes may be returned.
+func (c *limitedConn) scan(p []byte) (int, error) {
+	for i := 0; i < len(p); {
+		if c.payload > 0 {
+			skip := min(c.payload, int64(len(p)-i))
+			c.payload -= skip
+			i += int(skip)
+			continue
+		}
+		end := bytes.IndexByte(p[i:], '\n')
+		if end < 0 {
+			c.keepHeader(p[i:])
+			return len(p), nil
+		}
+		c.keepHeader(p[i : i+end])
+		payload, err := declaredPayload(c.header, c.overlong)
+		c.header, c.overlong = c.header[:0], false
+		if err != nil {
+			return i + end, err
+		}
+		c.payload = payload
+		i += end + 1
+	}
+	return len(p), nil
+}
+
+func (c *limitedConn) keepHeader(p []byte) {
+	if room := maxHeaderLen - len(c.header); len(p) > room {
+		p = p[:room]
+		c.overlong = true
+	}
+	c.header = append(c.header, p...)
+}
+
+// declaredPayload returns how many payload bytes, "\r\n" included, follow a RESP header line
+// given without its "\n". It returns errReadLimit when go-redis would allocate more than the cap
+// from the line: the declared byte length for bulk strings, one interface per declared element
+// for arrays, sets, maps, attributes, and pushes.
+func declaredPayload(line []byte, overlong bool) (int64, error) {
+	if len(line) == 0 {
 		return 0, nil
 	}
-	if len(c.pending) == 0 {
-		if err := c.fill(); err != nil {
-			return 0, err
-		}
-	}
-	n := copy(b, c.pending)
-	c.pending = c.pending[n:]
-	return n, nil
-}
-
-func (c *limitedConn) fill() error {
-	if len(c.pending) > 0 {
-		return nil
-	}
-	if c.err != nil {
-		return c.err
-	}
-	for len(c.pending) == 0 {
-		if c.remaining <= 0 {
-			c.err = errReadLimit
-			return c.err
-		}
-		want := int64(len(c.scratch))
-		if want > c.remaining {
-			want = c.remaining
-		}
-		n, err := c.Conn.Read(c.scratch[:want])
-		if n > 0 {
-			c.remaining -= int64(n)
-			if aerr := c.accept(c.scratch[:n]); aerr != nil {
-				c.err = aerr
-				if len(c.pending) > 0 {
-					return nil
-				}
-				return aerr
-			}
-		}
-		if err != nil {
-			// A read deadline belongs to one call. go-redis peeks for push notifications
-			// with a few microseconds of deadline, and on Windows that peek always runs.
-			// Keeping the timeout fails the command that follows.
-			if timeoutErr(err) {
-				if len(c.pending) > 0 {
-					return nil
-				}
-				return err
-			}
-			c.err = err
-			if len(c.pending) > 0 {
-				return nil
-			}
-			return err
-		}
-		if n == 0 {
-			c.err = io.ErrUnexpectedEOF
-			return c.err
-		}
-	}
-	return nil
-}
-
-func (c *limitedConn) accept(p []byte) error {
-	c.partial = append(c.partial, p...)
-	for {
-		i := bytes.IndexByte(c.partial, '\n')
-		if i < 0 {
-			if int64(len(c.partial)) > maxReadBytes {
-				c.partial = nil
-				return errReadLimit
-			}
-			return nil
-		}
-		line := c.partial[:i+1]
-		if lengthTooLarge(line) {
-			c.partial = nil
-			return errReadLimit
-		}
-		copied := make([]byte, len(line))
-		copy(copied, line)
-		c.pending = append(c.pending, copied...)
-		rest := make([]byte, len(c.partial)-(i+1))
-		copy(rest, c.partial[i+1:])
-		c.partial = rest
-	}
-}
-
-// lengthTooLarge reports whether line is a RESP length header go-redis would allocate from.
-// Bulk strings allocate the declared byte length. Arrays, sets, maps, attributes, and pushes
-// allocate one interface per declared element.
-func lengthTooLarge(line []byte) bool {
-	if len(line) < 3 {
-		return false
-	}
 	var limit int64
+	bulk := false
 	switch line[0] {
 	case '$', '=', '!':
-		limit = maxReadBytes
+		limit, bulk = maxReadBytes, true
 	case '*', '~', '>', '%', '|':
 		limit = maxAggregate
 	default:
-		return false
+		return 0, nil
 	}
-	body := line[1 : len(line)-1]
-	if len(body) > 0 && body[len(body)-1] == '\r' {
-		body = body[:len(body)-1]
+	if overlong {
+		return 0, errReadLimit
 	}
+	body := bytes.TrimSuffix(line[1:], []byte("\r"))
 	if len(body) == 0 || body[0] == '-' {
-		return false
+		return 0, nil
 	}
 	n, err := strconv.ParseInt(string(body), 10, 64)
-	if err != nil {
-		return true
+	if err != nil || n > limit {
+		return 0, errReadLimit
 	}
-	return n > limit
+	if !bulk {
+		return 0, nil
+	}
+	return n + 2, nil
 }
 
-func timeoutErr(err error) bool {
-	var netErr net.Error
-	return errors.As(err, &netErr) && netErr.Timeout()
+// dialAttempts replaces go-redis's default of 5. The extra attempts run in a goroutine that
+// ignores the caller's context, so they kept dialing a filtered port for ~25s after the
+// template had given up on it.
+const dialAttempts = 1
+
+func init() {
+	redis.SetLogger(warningLogger{})
+}
+
+// warningLogger hands go-redis's logs to gologger as warnings. Its default logger writes them
+// straight to stderr, where a failed dial against the scanned host reads as an error.
+type warningLogger struct{}
+
+func (warningLogger) Printf(_ context.Context, format string, v ...interface{}) {
+	gologger.Warning().Msgf(format, v...)
 }

pkg/js/libs/redis/redis_dial_test.go

@@ -0,0 +1,81 @@
+package redis
+
+import (
+	"context"
+	"net"
+	"runtime"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/gologger/levels"
+	"github.com/projectdiscovery/gologger/writer"
+	"github.com/stretchr/testify/require"
+)
+
+// go-redis retries a failed dial 4 times, 100ms apart, on each of a command's 4 attempts: a closed
+// port took at least 1.6s to fail.
+func TestGetServerInfoFailsFastOnAClosedPort(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("windows takes about a second to report a refused localhost dial")
+	}
+
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	addr, ok := listener.Addr().(*net.TCPAddr)
+	require.True(t, ok)
+	require.NoError(t, listener.Close())
+
+	start := time.Now()
+	_, err = getServerInfo(context.Background(), "redis-closed-port", "127.0.0.1", addr.Port)
+	require.Error(t, err)
+	require.Less(t, time.Since(start), time.Second)
+}
+
+// go-redis's default logger wrote each failed dial straight to stderr, past nuclei's logger and
+// its level.
+func TestGoRedisLogsAreGologgerWarnings(t *testing.T) {
+	logs := &recordingWriter{}
+	gologger.DefaultLogger.SetWriter(logs)
+	gologger.DefaultLogger.SetMaxLevel(levels.LevelWarning)
+	t.Cleanup(func() {
+		// gologger's own defaults: it has no getter to save the previous ones.
+		gologger.DefaultLogger.SetWriter(writer.NewCLI())
+		gologger.DefaultLogger.SetMaxLevel(levels.LevelInfo)
+	})
+
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	addr, ok := listener.Addr().(*net.TCPAddr)
+	require.True(t, ok)
+	require.NoError(t, listener.Close())
+
+	_, err = getServerInfo(context.Background(), "redis-log-level", "127.0.0.1", addr.Port)
+	require.Error(t, err)
+
+	warnings := logs.at(levels.LevelWarning)
+	require.NotEmpty(t, warnings)
+	require.Contains(t, warnings[0], "failed to dial")
+}
+
+type recordingWriter struct {
+	mu    sync.Mutex
+	lines map[levels.Level][]string
+}
+
+func (w *recordingWriter) Write(data []byte, level levels.Level) {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	if w.lines == nil {
+		w.lines = make(map[levels.Level][]string)
+	}
+	w.lines[level] = append(w.lines[level], strings.TrimSpace(string(data)))
+}
+
+func (w *recordingWriter) at(level levels.Level) []string {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	return w.lines[level]
+}

pkg/js/libs/redis/redis_test.go

@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"net"
 	"runtime"
 	"strconv"
@@ -90,37 +91,58 @@ func TestGetServerInfoAuthBoundsAReplyWithoutNewline(t *testing.T) {
 }
 
 func TestGetServerInfoAuthReadsAnInfoReply(t *testing.T) {
-	listener, err := net.Listen("tcp", "127.0.0.1:0")
-	require.NoError(t, err)
-	t.Cleanup(func() {
-		_ = listener.Close()
-	})
+	tests := []struct {
+		name string
+		info string
+	}{
+		{name: "info", info: "# Server\r\nredis_version:7.2.4\r\n"},
+		// A bulk payload's lines are data, even when they start like a RESP length header.
+		{name: "header-like lines", info: "# Server\r\n*abc\r\n$ 5 dollars\r\n%x\r\n$99999999999\r\n"},
+		// Each line used to cost a copy of what was left of the read buffer.
+		{name: "many short lines", info: strings.Repeat("\n", 1<<20)},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			listener, err := net.Listen("tcp", "127.0.0.1:0")
+			require.NoError(t, err)
+			t.Cleanup(func() {
+				_ = listener.Close()
+			})
 
-	const info = "# Server\r\nredis_version:7.2.4\r\n"
-	go func() {
-		for {
-			conn, err := listener.Accept()
-			if err != nil {
-				return
-			}
-			go serveRESP(conn, info)
-		}
-	}()
+			go func() {
+				for {
+					conn, err := listener.Accept()
+					if err != nil {
+						return
+					}
+					go serveRESP(conn, tt.info)
+				}
+			}()
 
-	executionID := "redis-" + strings.ReplaceAll(t.Name(), "/", "-")
-	require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: executionID}))
-	t.Cleanup(func() {
-		protocolstate.Close(executionID)
-	})
+			executionID := "redis-" + strings.ReplaceAll(t.Name(), "/", "-")
+			require.NoError(t, protocolstate.Init(&types.Options{ExecutionId: executionID}))
+			t.Cleanup(func() {
+				protocolstate.Close(executionID)
+			})
 
-	addr, ok := listener.Addr().(*net.TCPAddr)
-	require.True(t, ok)
+			addr, ok := listener.Addr().(*net.TCPAddr)
+			require.True(t, ok)
 
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	got, err := getServerInfoAuth(ctx, executionID, "127.0.0.1", addr.Port, "")
-	require.NoError(t, err)
-	require.Equal(t, info, got)
+			var before runtime.MemStats
+			runtime.ReadMemStats(&before)
+
+			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+			defer cancel()
+			got, err := getServerInfoAuth(ctx, executionID, "127.0.0.1", addr.Port, "")
+
+			var after runtime.MemStats
+			runtime.ReadMemStats(&after)
+			require.NoError(t, err)
+			require.Equal(t, tt.info, got)
+			growth := int64(after.TotalAlloc - before.TotalAlloc)
+			require.Less(t, growth, int64(64<<20), "allocated %d MiB reading one reply", growth>>20)
+		})
+	}
 }
 
 // serveRESP answers HELLO as a Redis without RESP3 does, so the client falls back to RESP2, and
@@ -182,26 +204,85 @@ func readRESPCommand(rd *bufio.Reader) (string, error) {
 	return command, nil
 }
 
-func TestLengthTooLarge(t *testing.T) {
+func TestDeclaredPayload(t *testing.T) {
 	tests := []struct {
-		line string
-		want bool
+		line     string
+		overlong bool
+		payload  int64
+		tooLarge bool
 	}{
-		{line: fmt.Sprintf("$%d\r\n", maxReadBytes), want: false},
-		{line: fmt.Sprintf("$%d\r\n", maxReadBytes+1), want: true},
-		{line: "$-1\r\n", want: false},
-		{line: "+OK\r\n", want: false},
-		{line: fmt.Sprintf("*%d\r\n", maxAggregate), want: false},
-		{line: fmt.Sprintf("*%d\r\n", maxAggregate+1), want: true},
-		{line: "$999999999999999999999\r\n", want: true},
+		{line: fmt.Sprintf("$%d\r", maxReadBytes), payload: maxReadBytes + 2},
+		{line: fmt.Sprintf("$%d\r", maxReadBytes+1), tooLarge: true},
+		{line: "=10\r", payload: 12},
+		{line: "$-1\r"},
+		{line: "+OK\r"},
+		{line: "+OK\r", overlong: true},
+		{line: fmt.Sprintf("*%d\r", maxAggregate)},
+		{line: fmt.Sprintf("*%d\r", maxAggregate+1), tooLarge: true},
+		{line: "$999999999999999999999\r", tooLarge: true},
+		{line: "*abc\r", tooLarge: true},
+		{line: "$000000000000000000000000000000", overlong: true, tooLarge: true},
 	}
 	for _, tt := range tests {
 		t.Run(tt.line, func(t *testing.T) {
-			require.Equal(t, tt.want, lengthTooLarge([]byte(tt.line)))
+			payload, err := declaredPayload([]byte(tt.line), tt.overlong)
+			if tt.tooLarge {
+				require.ErrorIs(t, err, errReadLimit)
+				return
+			}
+			require.NoError(t, err)
+			require.Equal(t, tt.payload, payload)
 		})
 	}
 }
 
+// The check that rejects a length header has to land on the "\n" when the header arrives over
+// several reads, and payload bytes have to pass whatever they contain.
+func TestLimitedConnFollowsFramingAcrossReads(t *testing.T) {
+	const ok = "*2\r\n$5\r\n*abc\n\r\n$-1\r\n+OK\r\n"
+	tooLarge := fmt.Sprintf("$%d\r\n", maxReadBytes+1)
+
+	tests := []struct {
+		name    string
+		stream  string
+		want    string
+		wantErr error
+	}{
+		{name: "payload passes", stream: ok, want: ok},
+		{name: "header rejected", stream: ok + tooLarge + "x", want: ok + tooLarge[:len(tooLarge)-1], wantErr: errReadLimit},
+	}
+	for _, tt := range tests {
+		for _, size := range []int{1, 3, 64} {
+			t.Run(fmt.Sprintf("%s/%d", tt.name, size), func(t *testing.T) {
+				lc := &limitedConn{Conn: &chunkedConn{data: []byte(tt.stream), size: size}, remaining: maxReadBytes}
+				got, err := io.ReadAll(lc)
+				require.Equal(t, tt.want, string(got))
+				if tt.wantErr != nil {
+					require.ErrorIs(t, err, tt.wantErr)
+					return
+				}
+				require.NoError(t, err)
+			})
+		}
+	}
+}
+
+// chunkedConn returns data at most size bytes per read, then io.EOF.
+type chunkedConn struct {
+	net.Conn
+	data []byte
+	size int
+}
+
+func (c *chunkedConn) Read(b []byte) (int, error) {
+	if len(c.data) == 0 {
+		return 0, io.EOF
+	}
+	n := copy(b[:min(len(b), c.size)], c.data)
+	c.data = c.data[n:]
+	return n, nil
+}
+
 // A short RESP length header used to make go-redis allocate the declared size before reading
 // any of the payload, so the byte cap never applied.
 func TestGetServerInfoAuthRejectsAHugeDeclaredLength(t *testing.T) {

pkg/loader/workflow/workflow_loader.go

@@ -16,6 +16,8 @@ import (
 type workflowLoader struct {
 	pathFilter *filter.PathFilter
 	tagFilter  *templates.TagFilter
+	// the exclusions of tagFilter other than severities, see filters
+	exclusions *templates.TagFilter
 	options    *protocols.ExecutorOptions
 
 	// every workflow step that selects templates by tag searches the whole
@@ -45,12 +47,29 @@ func NewLoader(options *protocols.ExecutorOptions) (model.WorkflowLoader, error)
 	if err != nil {
 		return nil, err
 	}
+	// IncludeTags is not an inclusion filter: it lifts the exclusion of the
+	// same tags
+	exclusions, err := templates.NewTagFilter(&templates.TagFilterConfig{
+		ExcludeTags:      options.Options.ExcludeTags,
+		IncludeTags:      options.Options.IncludeTags,
+		ExcludeIds:       options.Options.ExcludeIds,
+		ExcludeProtocols: options.Options.ExcludeProtocols,
+	})
+	if err != nil {
+		return nil, err
+	}
 	pathFilter := filter.NewPathFilter(&filter.PathFilterConfig{
 		IncludedTemplates: options.Options.IncludeTemplates,
 		ExcludedTemplates: options.Options.ExcludedTemplates,
 	}, options.Catalog)
 
-	loader := &workflowLoader{pathFilter: pathFilter, tagFilter: tagFilter, options: options, parsed: make(map[string]*templates.Template)}
+	loader := &workflowLoader{
+		pathFilter: pathFilter,
+		tagFilter:  tagFilter,
+		exclusions: exclusions,
+		options:    options,
+		parsed:     make(map[string]*templates.Template),
+	}
 	loader.templatesDirectoryPaths = sync.OnceValue(loader.listTemplatesDirectory)
 	return loader, nil
 }
@@ -85,20 +104,34 @@ func (w *workflowLoader) parseTemplate(path string) *templates.Template {
 	return template
 }
 
-func (w *workflowLoader) GetTemplatePathsByTags(templateTags []string) []string {
+func (w *workflowLoader) GetTemplatePathsByTags(templateTags []string, noValidate bool) []string {
 	var loadedTemplates []string
 	for _, path := range w.templatesDirectoryPaths() {
+		// the step's tags select the templates and the filters then decide
+		// which of them load, as for the templates of a template step
 		template := w.parseTemplate(path)
-		if template == nil {
+		if template == nil || !selectedByTags(template, templateTags) {
 			continue
 		}
-		if loaded, _ := templates.MatchLoadFilters(template, path, w.tagFilter, templateTags); loaded {
+		if matched, _ := templates.MatchLoadFilters(template, path, w.filters(noValidate), nil); matched {
 			loadedTemplates = append(loadedTemplates, path)
 		}
 	}
 	return loadedTemplates
 }
 
+// selectedByTags reports whether a workflow step that selects templates by
+// tags selects template, which it does when template has one of tags.
+// Workflows are never selected, as a workflow cannot run as a step.
+func selectedByTags(template *templates.Template, tags []string) bool {
+	if len(template.Workflows) > 0 {
+		return false
+	}
+	return slices.ContainsFunc(template.Info.Tags.ToSlice(), func(tag string) bool {
+		return slices.Contains(tags, tag)
+	})
+}
+
 func (w *workflowLoader) GetTemplatePaths(templatesList []string, noValidate bool) []string {
 	includedTemplates, errs := w.options.Catalog.GetTemplatesPath(templatesList)
 	for template, err := range errs {
@@ -108,12 +141,27 @@ func (w *workflowLoader) GetTemplatePaths(templatesList []string, noValidate boo
 
 	loadedTemplates := make([]string, 0, len(templatesPathMap))
 	for templatePath := range templatesPathMap {
-		matched, err := w.options.Parser.LoadTemplate(templatePath, w.tagFilter, nil, w.options.Catalog)
+		matched, err := w.options.Parser.LoadTemplate(templatePath, w.filters(noValidate), nil, w.options.Catalog)
 		if err != nil && !matched {
 			gologger.Warning().Msg(err.Error())
-		} else if matched || noValidate {
+		} else if matched {
 			loadedTemplates = append(loadedTemplates, templatePath)
 		}
 	}
 	return loadedTemplates
 }
+
+// filters returns the filters that decide which templates a workflow step
+// loads. With noValidate, set for a step whose subtemplates loaded, only the
+// exclusions apply: the step must load for its subtemplates to run, but an
+// excluded template never runs. Excluded severities do not apply either: a
+// step with subtemplates does not report its own results, so excluding its
+// severity would only keep its subtemplates from running. A late
+// out-of-band (interactsh) match of the step is the exception: it is
+// reported whatever its severity, as with -s.
+func (w *workflowLoader) filters(noValidate bool) *templates.TagFilter {
+	if noValidate {
+		return w.exclusions
+	}
+	return w.tagFilter
+}

pkg/loader/workflow/workflow_loader_test.go

@@ -1,6 +1,8 @@
 package workflow
 
 import (
+	"maps"
+	"slices"
 	"testing"
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog"
@@ -8,8 +10,12 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
 	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/stringslice"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/dns"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
+	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	"github.com/projectdiscovery/nuclei/v3/pkg/utils/stats"
+	"github.com/projectdiscovery/nuclei/v3/pkg/workflows"
 	"github.com/stretchr/testify/require"
 )
 
@@ -63,9 +69,87 @@ func TestGetTemplatePathsByTagsReadsTemplatesDirectoryOnce(t *testing.T) {
 	require.NoError(t, err)
 	walksBeforeLookups := cat.walks
 
-	require.Equal(t, []string{"wordpress.yaml"}, loader.GetTemplatePathsByTags([]string{"wordpress"}))
-	require.Equal(t, []string{"jira.yaml"}, loader.GetTemplatePathsByTags([]string{"jira"}))
+	require.Equal(t, []string{"wordpress.yaml"}, loader.GetTemplatePathsByTags([]string{"wordpress"}, false))
+	require.Equal(t, []string{"jira.yaml"}, loader.GetTemplatePathsByTags([]string{"jira"}, false))
 
 	require.Equal(t, 1, cat.walks-walksBeforeLookups, "templates directory walked per lookup")
 	require.Equal(t, map[string]int{"wordpress.yaml": 1, "jira.yaml": 1}, parser.parses, "templates parsed per lookup")
 }
+
+func TestGetTemplatePathsByTagsNoValidateAppliesExclusionsButSeverity(t *testing.T) {
+	template := func(id string, tags ...string) *templates.Template {
+		return &templates.Template{ID: id, Info: model.Info{
+			Name:           id,
+			Authors:        stringslice.StringSlice{Value: "author"},
+			Tags:           stringslice.StringSlice{Value: tags},
+			SeverityHolder: severity.Holder{Severity: severity.Info},
+		}}
+	}
+	excludedSeverity := template("excluded-severity", "step")
+	excludedSeverity.Info.SeverityHolder.Severity = severity.Low
+	excludedProtocol := template("excluded-protocol", "step")
+	excludedProtocol.RequestsDNS = []*dns.Request{{}}
+	workflow := template("workflow", "step")
+	workflow.Workflows = []*workflows.WorkflowTemplate{{Template: "selected.yaml"}}
+	parser := &countingParser{
+		templates: map[string]*templates.Template{
+			"selected.yaml":          template("selected", "step"),
+			"included.yaml":          template("included", "step", "included"),
+			"excluded-tag.yaml":      template("excluded-tag", "step", "excluded"),
+			"excluded-id.yaml":       template("excluded-id", "step"),
+			"excluded-severity.yaml": excludedSeverity,
+			"excluded-protocol.yaml": excludedProtocol,
+			"untagged.yaml":          template("untagged", "other"),
+			"workflow.yaml":          workflow,
+		},
+		parses: make(map[string]int),
+	}
+	options := types.DefaultOptions()
+	options.IncludeIds = []string{"untagged"}
+	options.ExcludeTags = []string{"excluded", "included"}
+	options.IncludeTags = []string{"included"}
+	options.ExcludeIds = []string{"excluded-id"}
+	options.ExcludeSeverities = severity.Severities{severity.Low}
+	options.ExcludeProtocols = templateTypes.ProtocolTypes{templateTypes.DNSProtocol}
+	cat := &countingCatalog{paths: slices.Collect(maps.Keys(parser.templates))}
+	loader, err := NewLoader(&protocols.ExecutorOptions{Options: options, Catalog: cat, Parser: parser})
+	require.NoError(t, err)
+
+	require.Empty(t, loader.GetTemplatePathsByTags([]string{"step"}, false))
+	loaded := loader.GetTemplatePathsByTags([]string{"step"}, true)
+	require.ElementsMatch(t, []string{"selected.yaml", "included.yaml", "excluded-severity.yaml"}, loaded)
+}
+
+func TestGetTemplatePathsByTagsCountsSyntaxWarningsOfLoadedTemplates(t *testing.T) {
+	// a template without severity has a syntax warning
+	unrated := &templates.Template{ID: "unrated", Info: model.Info{
+		Name:    "unrated",
+		Authors: stringslice.StringSlice{Value: "author"},
+		Tags:    stringslice.StringSlice{Value: "step"},
+	}}
+	tests := []struct {
+		name     string
+		ids      []string
+		warnings int64
+	}{
+		{name: "loaded", warnings: 1},
+		{name: "filtered out", ids: []string{"other"}, warnings: 0},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			parser := &countingParser{
+				templates: map[string]*templates.Template{"unrated.yaml": unrated},
+				parses:    make(map[string]int),
+			}
+			options := types.DefaultOptions()
+			options.IncludeIds = tc.ids
+			cat := &countingCatalog{paths: []string{"unrated.yaml"}}
+			loader, err := NewLoader(&protocols.ExecutorOptions{Options: options, Catalog: cat, Parser: parser})
+			require.NoError(t, err)
+
+			before := stats.GetValue(templates.TemplateSyntaxWarningStats)
+			loader.GetTemplatePathsByTags([]string{"step"}, false)
+			require.Equal(t, tc.warnings, stats.GetValue(templates.TemplateSyntaxWarningStats)-before)
+		})
+	}
+}

pkg/model/model.go

@@ -1,9 +1,15 @@
 package model
 
 import (
+	"slices"
+	"strings"
+
 	"github.com/invopop/jsonschema"
 	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
 	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/stringslice"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/marker"
+	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
 )
 
 type schemaMetadata struct {
@@ -47,15 +53,28 @@ type Info struct {
 	//
 	//   You can go in-depth here on what the template actually does.
 	//
+	//   Description, impact and remediation may hold {{placeholders}} filled for
+	//   each finding: named extractors, payloads, variables and DSL helpers
+	//   (network helpers excluded), and the result fields host, port, ip, url,
+	//   scheme, path, matched-at, matcher-name and extractor-name, which win over
+	//   values of the same name. Use join(", ", name) for an extractor with
+	//   several values, and brackets for a hyphenated name inside a helper:
+	//   to_upper([matcher-name]). Unknown placeholders stay as written, and so
+	//   does a field whose expression fails. Global matcher templates match on
+	//   other templates' responses, so only the result fields fill their text.
+	//
 	// examples:
 	//   - value: "\"Bower is a package manager which stores package information in the bower.json file\""
 	//   - value: "\"Subversion ALM for the enterprise before 8.8.2 allows reflected XSS at multiple locations\""
+	//   - value: "\"Redis {{version}} on {{host}}:{{port}} accepts the default password\""
 	Description string `json:"description,omitempty" yaml:"description,omitempty" jsonschema:"title=description of the template,description=In-depth explanation on what the template does,type=string,example=Bower is a package manager which stores package information in the bower.json file"`
 	// description: |
 	//   Impact of the template.
 	//
 	//   You can go in-depth here on impact of the template.
 	//
+	//   Supports the same placeholders as description.
+	//
 	// examples:
 	//   - value: "\"Successful exploitation of this vulnerability could allow an attacker to execute arbitrary SQL queries, potentially leading to unauthorized access, data leakage, or data manipulation.\""
 	//   - value: "\"Successful exploitation of this vulnerability could allow an attacker to execute arbitrary script code in the context of the victim's browser, potentially leading to session hijacking, defacement, or theft of sensitive information.\""
@@ -89,11 +108,70 @@ type Info struct {
 	//
 	//   You can go in-depth here on how to mitigate the problem found by this template.
 	//
+	//   Supports the same placeholders as description.
+	//
 	// examples:
 	//   - value: "\"Change the default administrative username and password of Apache ActiveMQ by editing the file jetty-realm.properties\""
 	Remediation string `json:"remediation,omitempty" yaml:"remediation,omitempty" jsonschema:"title=remediation steps for the template,description=In-depth explanation on how to fix the issues found by the template,example=Change the default administrative username and password of Apache ActiveMQ by editing the file jetty-realm.properties,type=string"`
 }
 
+// MarshalJSON encodes the info with every map nested in Metadata keyed by strings.
+func (i Info) MarshalJSON() ([]byte, error) {
+	// info has the fields of Info but not this method, so marshaling it does not recurse.
+	type info Info
+	encoded := info(i)
+	metadata, err := types.ToJSONMap(i.Metadata)
+	if err != nil {
+		return nil, err
+	}
+	encoded.Metadata = metadata
+	return json.Marshal(encoded)
+}
+
+// TextField is a free-text field of the info, labeled as reports show it.
+type TextField struct {
+	Label string
+	Value *string
+}
+
+// PlaceholderFields returns the free-text fields that may hold {{placeholders}}
+// filled for each finding. Rendering, redaction and the sarif export all walk
+// this one list, so a field added here is handled by every one of them.
+func (i *Info) PlaceholderFields() []TextField {
+	return []TextField{
+		{Label: "Description", Value: &i.Description},
+		{Label: "Impact", Value: &i.Impact},
+		{Label: "Remediation", Value: &i.Remediation},
+	}
+}
+
+// HasPlaceholders reports whether the field holds a {{...}} or §...§ marker
+// the renderer could fill.
+func (f TextField) HasPlaceholders() bool {
+	return strings.Contains(*f.Value, marker.ParenthesisOpen) || strings.Contains(*f.Value, marker.General)
+}
+
+// HasPlaceholders reports whether any placeholder field holds a marker.
+func (i *Info) HasPlaceholders() bool {
+	return slices.ContainsFunc(i.PlaceholderFields(), TextField.HasPlaceholders)
+}
+
+// FilledFields returns the placeholder fields whose text differs from written,
+// the info as the template wrote it. A nil written means nothing was filled.
+func (i *Info) FilledFields(written *Info) []TextField {
+	if written == nil {
+		return nil
+	}
+	writtenFields := written.PlaceholderFields()
+	var filled []TextField
+	for n, field := range i.PlaceholderFields() {
+		if *field.Value != *writtenFields[n].Value {
+			filled = append(filled, field)
+		}
+	}
+	return filled
+}
+
 // JSONSchemaProperty returns the JSON schema property for the Info object.
 func (i Info) JSONSchemaExtend(base *jsonschema.Schema) {
 	// since we are re-using a stringslice and rawStringSlice everywhere, we can extend/edit the schema here

pkg/model/model_test.go

@@ -35,6 +35,34 @@ func TestInfoJsonMarshal(t *testing.T) {
 	require.Equal(t, expected, string(result))
 }
 
+func TestInfoJsonMarshalNestedYAMLMetadata(t *testing.T) {
+	var info Info
+	err := yaml.Unmarshal([]byte(`
+name: Nested metadata
+author: pdteam
+tags: misc
+severity: info
+metadata:
+  labels:
+    en: Hello
+    1: one
+  items:
+    - name: first
+      tags:
+        fr: Bonjour
+  plain: value
+`), &info)
+	require.NoError(t, err)
+	require.IsType(t, map[interface{}]interface{}{}, info.Metadata["labels"])
+
+	for _, value := range []interface{}{info, &info} {
+		result, err := json.Marshal(value)
+		require.NoError(t, err)
+		require.JSONEq(t, `{"name":"Nested metadata","author":["pdteam"],"tags":["misc"],"severity":"info","metadata":{"labels":{"en":"Hello","1":"one"},"items":[{"name":"first","tags":{"fr":"Bonjour"}}],"plain":"value"}}`, string(result))
+	}
+	require.IsType(t, map[interface{}]interface{}{}, info.Metadata["labels"], "marshaling must not modify the info")
+}
+
 func TestInfoYamlMarshal(t *testing.T) {
 	info := Info{
 		Name:           "Test Template Name",
@@ -135,3 +163,21 @@ func TestUnmarshal(t *testing.T) {
 	info2 := assertUnmarshalledTemplateInfo(t, yamlPayload2)
 	require.Equal(t, info1, info2)
 }
+
+func TestInfoFilledFields(t *testing.T) {
+	written := Info{Description: "{{host}} is exposed", Impact: "Data loss"}
+	filled := written
+	filled.Description = "example.com is exposed"
+
+	require.Nil(t, filled.FilledFields(nil))
+	fields := filled.FilledFields(&written)
+	require.Len(t, fields, 1)
+	require.Equal(t, "Description", fields[0].Label)
+	require.Equal(t, "example.com is exposed", *fields[0].Value)
+}
+
+func TestInfoHasPlaceholders(t *testing.T) {
+	require.False(t, (&Info{Description: "Redis is exposed"}).HasPlaceholders())
+	require.True(t, (&Info{Impact: "{{host}} is exposed"}).HasPlaceholders())
+	require.True(t, (&Info{Remediation: "rotate §password§"}).HasPlaceholders())
+}

pkg/model/workflow_loader.go

@@ -4,9 +4,13 @@ package model
 
 // WorkflowLoader is a loader interface required for workflow initialization.
 type WorkflowLoader interface {
-	// GetTemplatePathsByTags returns a list of template paths based on the provided tags from the templates directory
-	GetTemplatePathsByTags(tags []string) []string
+	// GetTemplatePathsByTags returns the paths of the templates in the
+	// templates directory that have one of tags. noValidate is as in
+	// GetTemplatePaths.
+	GetTemplatePathsByTags(tags []string, noValidate bool) []string
 
-	// GetTemplatePaths takes a list of templates and returns paths for them
+	// GetTemplatePaths takes a list of templates and returns paths for them.
+	// With noValidate, only the exclusion filters apply, except excluded
+	// severities: excluded tags, IDs, protocols and paths.
 	GetTemplatePaths(templatesList []string, noValidate bool) []string
 }

pkg/operators/common/dsl/network_policy_test.go

@@ -3,6 +3,8 @@ package dsl
 import (
 	"fmt"
 	"net"
+	"os"
+	"path/filepath"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -44,6 +46,12 @@ func networkTestDNS(t *testing.T, address string) (string, *atomic.Int32) {
 
 func networkTestOptions(t *testing.T, options *types.Options) *types.Options {
 	t.Helper()
+	// fastdialer adds the nameservers of the system resolver file before the
+	// stub resolvers and picks them round robin, so the stub answer would
+	// depend on the host. An empty resolver file keeps only the stub.
+	resolverFile := filepath.Join(t.TempDir(), "resolv.conf")
+	require.NoError(t, os.WriteFile(resolverFile, nil, 0o600))
+	t.Setenv("RESOLVERS_PATH", resolverFile)
 	if options.ExecutionId == "" {
 		options.ExecutionId = t.Name()
 	}

pkg/operators/common/llm/llm.go

@@ -21,6 +21,22 @@ type Client interface {
 	Complete(ctx context.Context, prompt string, asJSON bool) (string, error)
 }
 
+// Judge is implemented by clients whose provider classifies natively. Its
+// confidence is a calibrated probability rather than a number the model
+// reports about itself, and the inputs travel apart from the question, so
+// matchers use it over Complete when it is available.
+type Judge interface {
+	Judge(ctx context.Context, question string, labels, inputs []string) (Verdict, error)
+}
+
+// Verdict is the label a Judge chose. Model is the exact model version that
+// answered, when the provider reports one.
+type Verdict struct {
+	Label      string
+	Confidence float64
+	Model      string
+}
+
 // TruncateApproxTokens trims input to roughly maxTokens using the common
 // 4-chars-per-token approximation. Exact counting is provider-specific; this
 // only needs to keep a large body from blowing the context window.

pkg/operators/extractors/compile_test.go

@@ -130,3 +130,12 @@ func TestCompileExtractorsIgnoresXPathOnOtherTypes(t *testing.T) {
 	}
 	require.NoError(t, e.CompileExtractors())
 }
+
+func TestCompileExtractorsXPathUndeclaredVariable(t *testing.T) {
+	// Regression test: this expression used to compile and then panic
+	// with a nil pointer dereference inside antchfx/xpath when the
+	// first response was evaluated. It must fail at compile time.
+	var operator Extractor
+	require.NoError(t, yaml.Unmarshal([]byte("type: xpath\nxpath:\n  - \"******$A\"\n"), &operator))
+	require.ErrorContains(t, operator.CompileExtractors(), "undeclared variable")
+}

pkg/operators/matchers/compile_test.go

@@ -53,3 +53,12 @@ func TestCompileMatchersAllowsEmptyStringValue(t *testing.T) {
 	matched, _ := m.MatchWords("body", nil)
 	require.True(t, matched)
 }
+
+func TestCompileMatchersDSLTrailingBackslash(t *testing.T) {
+	// Regression test: a DSL expression whose string literal ends in a
+	// backslash used to panic the govaluate lexer (index out of range)
+	// while a template was loading. It must surface as a compile error.
+	var operator Matcher
+	require.NoError(t, yaml.Unmarshal([]byte("type: dsl\ndsl:\n  - \"body == 'x\\\\'\"\n"), &operator))
+	require.ErrorContains(t, operator.CompileMatchers(), "Unclosed string literal")
+}

pkg/operators/matchers/match_llm.go

@@ -65,26 +65,21 @@ func (matcher *Matcher) MatchLLMWithAudit(corpus string, inputs []string, values
 		return false, nil, nil
 	}
 
-	prompt := matcher.buildLLMPrompt(matcher.truncate(corpus, inputs), values)
-	answer, err := client.Complete(context.Background(), prompt, true)
-	if err != nil {
+	responses := matcher.truncate(corpus, inputs)
+	var (
+		verdict llmVerdict
+		audit   *LLMAudit
+		ok      bool
+	)
+	if judge, isJudge := client.(llmclient.Judge); isJudge {
+		verdict, audit, ok = matcher.judgeLLM(judge, responses, values)
+	} else {
+		verdict, audit, ok = matcher.completeLLM(client, responses, values)
+	}
+	if !ok {
 		return false, nil, nil
 	}
 
-	var verdict llmVerdict
-	if err := json.Unmarshal([]byte(answer), &verdict); err != nil {
-		return false, nil, nil
-	}
-
-	audit := &LLMAudit{
-		PromptHash: hashPrompt(prompt),
-		Verdict:    strings.TrimSpace(verdict.Verdict),
-		Confidence: verdict.Confidence,
-	}
-	if namer, ok := client.(modelNamer); ok {
-		audit.Model = namer.Model()
-	}
-
 	expect := matcher.Expect
 	if expect == "" {
 		expect = "yes"
@@ -107,9 +102,55 @@ func (matcher *Matcher) MatchLLMWithAudit(corpus string, inputs []string, values
 		return false, nil, audit
 	}
 
+	if verdict.Evidence == "" {
+		return true, nil, audit
+	}
 	return true, []string{verdict.Evidence}, audit
 }
 
+// completeLLM asks a chat model for the verdict as JSON.
+func (matcher *Matcher) completeLLM(client LLMClient, inputs []string, values map[string]interface{}) (llmVerdict, *LLMAudit, bool) {
+	prompt := matcher.buildLLMPrompt(inputs, values)
+	answer, err := client.Complete(context.Background(), prompt, true)
+	if err != nil {
+		return llmVerdict{}, nil, false
+	}
+
+	var verdict llmVerdict
+	if err := json.Unmarshal([]byte(answer), &verdict); err != nil {
+		return llmVerdict{}, nil, false
+	}
+
+	audit := &LLMAudit{
+		PromptHash: hashPrompt(prompt),
+		Verdict:    strings.TrimSpace(verdict.Verdict),
+		Confidence: verdict.Confidence,
+	}
+	if namer, ok := client.(modelNamer); ok {
+		audit.Model = namer.Model()
+	}
+	return verdict, audit, true
+}
+
+// judgeLLM asks a native classifier. It needs no prompt framing: the question
+// and the responses are separate fields of the request, and the answer is a
+// probability over the allowed verdicts, not text. It quotes no evidence.
+func (matcher *Matcher) judgeLLM(judge llmclient.Judge, inputs []string, values map[string]interface{}) (llmVerdict, *LLMAudit, bool) {
+	question := interpolate(matcher.Prompt, values)
+	answer, err := judge.Judge(context.Background(), question, matcher.Options, inputs)
+	if err != nil {
+		return llmVerdict{}, nil, false
+	}
+
+	audit := &LLMAudit{
+		Model:      answer.Model,
+		PromptHash: hashPrompt(strings.Join(append([]string{question, strings.Join(matcher.Options, ",")}, inputs...), "\x00")),
+		Verdict:    answer.Label,
+		Confidence: answer.Confidence,
+	}
+	return llmVerdict{Verdict: answer.Label, Confidence: answer.Confidence}, audit, true
+}
+
 // buildLLMPrompt wraps the author's question with the output contract and the
 // response under a random boundary (see FrameResponse).
 //