mirror of
https://github.com/projectdiscovery/nuclei.git
synced 2026-11-08 10:10:18 -10:00
Compare commits
9
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Mzack9999
|
5a1f299efe | ||
|
Mzack9999
|
710ac6ecf1 | ||
|
Mzack9999
|
61a2d10919 | ||
|
Mzack9999
|
21057b7e8a | ||
|
Mzack9999
|
65323e52c4 | ||
|
Mzack9999
|
6c20d4daf7 | ||
|
Mzack9999
|
53a8e4c109 | ||
|
Mzack9999
|
bd1b13b726 | ||
|
Mzack9999
|
d43e31f200 |
No files matched your search
@@ -295,19 +295,21 @@ UNCOVER:
|
||||
-ur, -uncover-ratelimit int override ratelimit of engines with unknown ratelimit (default 60 req/min) (default 60)
|
||||
|
||||
RATE-LIMIT:
|
||||
-rl, -rate-limit int maximum number of requests to send per second (default 150)
|
||||
-rld, -rate-limit-duration value maximum number of requests to send per second (default 1s)
|
||||
-host-backoff slow down a host that starts blocking (429, repeated 403s, connection failures)
|
||||
-host-backoff-max value maximum delay applied to a host that is blocking (default 30s)
|
||||
-per-host-rate-limit enable per-host rate limiting (global rate limit becomes unlimited when enabled)
|
||||
-rlm, -rate-limit-minute int maximum number of requests to send per minute (DEPRECATED)
|
||||
-bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25)
|
||||
-c, -concurrency int maximum number of templates to be executed in parallel (default 25)
|
||||
-hbs, -headless-bulk-size int maximum number of headless hosts to be analyzed in parallel per template (default 10)
|
||||
-headc, -headless-concurrency int maximum number of headless templates to be executed in parallel (default 10)
|
||||
-jsc, -js-concurrency int maximum number of javascript runtimes to be executed in parallel (default 120)
|
||||
-pc, -payload-concurrency int max payload concurrency for each template (default 25)
|
||||
-prc, -probe-concurrency int http probe concurrency with httpx (default 50)
|
||||
-rl, -rate-limit int maximum number of requests to send per second (default 150)
|
||||
-rld, -rate-limit-duration value maximum number of requests to send per second (default 1s)
|
||||
-disable-adaptive-concurrency do not adapt in-flight requests per host:port to its latency and errors
|
||||
-disable-service-placement run js templates that only speak non-http protocols (smb, ssh, mysql, ...) even on ports that answer http
|
||||
-host-backoff slow down a host that starts blocking (429, repeated 403s, connection failures)
|
||||
-host-backoff-max value maximum delay applied to a host that is blocking (default 30s)
|
||||
-per-host-rate-limit enable per-host rate limiting (global rate limit becomes unlimited when enabled)
|
||||
-rlm, -rate-limit-minute int maximum number of requests to send per minute (DEPRECATED)
|
||||
-bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25)
|
||||
-c, -concurrency int maximum number of templates to be executed in parallel (default 25, or 100 with adaptive concurrency)
|
||||
-hbs, -headless-bulk-size int maximum number of headless hosts to be analyzed in parallel per template (default 10)
|
||||
-headc, -headless-concurrency int maximum number of headless templates to be executed in parallel (default 10)
|
||||
-jsc, -js-concurrency int maximum number of javascript runtimes to be executed in parallel (default 120)
|
||||
-pc, -payload-concurrency int max payload concurrency for each template (default 25)
|
||||
-prc, -probe-concurrency int http probe concurrency with httpx (default 50)
|
||||
-tlc, -template-loading-concurrency int maximum number of concurrent template loading operations (default 50)
|
||||
|
||||
OPTIMIZATIONS:
|
||||
@@ -323,6 +325,7 @@ OPTIMIZATIONS:
|
||||
-stream stream mode - start elaborating without sorting the input
|
||||
-ss, -scan-strategy value strategy to use while scanning(auto/host-spray/template-spray) (default auto)
|
||||
-irt, -input-read-timeout value timeout on input read (default 3m0s)
|
||||
-response-cache reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
|
||||
-nh, -no-httpx disable httpx probing for non-url input
|
||||
-preflight-portscan run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
|
||||
-no-stdin disable stdin processing
|
||||
|
||||
+16
-11
@@ -292,17 +292,21 @@ UNCOVER:
|
||||
-ur, -uncover-ratelimit int override ratelimit of engines with unknown ratelimit (default 60 req/min) (default 60)
|
||||
|
||||
RATE-LIMIT:
|
||||
-rl, -rate-limit int maximum number of requests to send per second (default 150)
|
||||
-rld, -rate-limit-duration value maximum number of requests to send per second (default 1s)
|
||||
-per-host-rate-limit enable per-host rate limiting (global rate limit becomes unlimited when enabled)
|
||||
-rlm, -rate-limit-minute int maximum number of requests to send per minute (DEPRECATED)
|
||||
-bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25)
|
||||
-c, -concurrency int maximum number of templates to be executed in parallel (default 25)
|
||||
-hbs, -headless-bulk-size int maximum number of headless hosts to be analyzed in parallel per template (default 10)
|
||||
-headc, -headless-concurrency int maximum number of headless templates to be executed in parallel (default 10)
|
||||
-jsc, -js-concurrency int maximum number of javascript runtimes to be executed in parallel (default 120)
|
||||
-pc, -payload-concurrency int max payload concurrency for each template (default 25)
|
||||
-prc, -probe-concurrency int http probe concurrency with httpx (default 50)
|
||||
-rl, -rate-limit int maximum number of requests to send per second (default 150)
|
||||
-rld, -rate-limit-duration value maximum number of requests to send per second (default 1s)
|
||||
-disable-adaptive-concurrency do not adapt in-flight requests per host:port to its latency and errors
|
||||
-disable-service-placement run js templates that only speak non-http protocols (smb, ssh, mysql, ...) even on ports that answer http
|
||||
-host-backoff slow down a host that starts blocking (429, repeated 403s, connection failures)
|
||||
-host-backoff-max value maximum delay applied to a host that is blocking (default 30s)
|
||||
-per-host-rate-limit enable per-host rate limiting (global rate limit becomes unlimited when enabled)
|
||||
-rlm, -rate-limit-minute int maximum number of requests to send per minute (DEPRECATED)
|
||||
-bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25)
|
||||
-c, -concurrency int maximum number of templates to be executed in parallel (default 25, or 100 with adaptive concurrency)
|
||||
-hbs, -headless-bulk-size int maximum number of headless hosts to be analyzed in parallel per template (default 10)
|
||||
-headc, -headless-concurrency int maximum number of headless templates to be executed in parallel (default 10)
|
||||
-jsc, -js-concurrency int maximum number of javascript runtimes to be executed in parallel (default 120)
|
||||
-pc, -payload-concurrency int max payload concurrency for each template (default 25)
|
||||
-prc, -probe-concurrency int http probe concurrency with httpx (default 50)
|
||||
-tlc, -template-loading-concurrency int maximum number of concurrent template loading operations (default 50)
|
||||
|
||||
OPTIMIZATIONS:
|
||||
@@ -318,6 +322,7 @@ OPTIMIZATIONS:
|
||||
-stream stream mode - start elaborating without sorting the input
|
||||
-ss, -scan-strategy value strategy to use while scanning(auto/host-spray/template-spray) (default auto)
|
||||
-irt, -input-read-timeout value timeout on input read (default 3m0s)
|
||||
-response-cache reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
|
||||
-nh, -no-httpx disable httpx probing for non-url input
|
||||
-preflight-portscan run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
|
||||
-no-stdin disable stdin processing
|
||||
|
||||
+16
-11
@@ -292,17 +292,21 @@ UNCOVER:
|
||||
-ur, -uncover-ratelimit int override ratelimit of engines with unknown ratelimit (default 60 req/min) (default 60)
|
||||
|
||||
RATE-LIMIT:
|
||||
-rl, -rate-limit int maximum number of requests to send per second (default 150)
|
||||
-rld, -rate-limit-duration value maximum number of requests to send per second (default 1s)
|
||||
-per-host-rate-limit enable per-host rate limiting (global rate limit becomes unlimited when enabled)
|
||||
-rlm, -rate-limit-minute int maximum number of requests to send per minute (DEPRECATED)
|
||||
-bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25)
|
||||
-c, -concurrency int maximum number of templates to be executed in parallel (default 25)
|
||||
-hbs, -headless-bulk-size int maximum number of headless hosts to be analyzed in parallel per template (default 10)
|
||||
-headc, -headless-concurrency int maximum number of headless templates to be executed in parallel (default 10)
|
||||
-jsc, -js-concurrency int maximum number of javascript runtimes to be executed in parallel (default 120)
|
||||
-pc, -payload-concurrency int max payload concurrency for each template (default 25)
|
||||
-prc, -probe-concurrency int http probe concurrency with httpx (default 50)
|
||||
-rl, -rate-limit int maximum number of requests to send per second (default 150)
|
||||
-rld, -rate-limit-duration value maximum number of requests to send per second (default 1s)
|
||||
-disable-adaptive-concurrency do not adapt in-flight requests per host:port to its latency and errors
|
||||
-disable-service-placement run js templates that only speak non-http protocols (smb, ssh, mysql, ...) even on ports that answer http
|
||||
-host-backoff slow down a host that starts blocking (429, repeated 403s, connection failures)
|
||||
-host-backoff-max value maximum delay applied to a host that is blocking (default 30s)
|
||||
-per-host-rate-limit enable per-host rate limiting (global rate limit becomes unlimited when enabled)
|
||||
-rlm, -rate-limit-minute int maximum number of requests to send per minute (DEPRECATED)
|
||||
-bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25)
|
||||
-c, -concurrency int maximum number of templates to be executed in parallel (default 25, or 100 with adaptive concurrency)
|
||||
-hbs, -headless-bulk-size int maximum number of headless hosts to be analyzed in parallel per template (default 10)
|
||||
-headc, -headless-concurrency int maximum number of headless templates to be executed in parallel (default 10)
|
||||
-jsc, -js-concurrency int maximum number of javascript runtimes to be executed in parallel (default 120)
|
||||
-pc, -payload-concurrency int max payload concurrency for each template (default 25)
|
||||
-prc, -probe-concurrency int http probe concurrency with httpx (default 50)
|
||||
-tlc, -template-loading-concurrency int maximum number of concurrent template loading operations (default 50)
|
||||
|
||||
OPTIMIZATIONS:
|
||||
@@ -318,6 +322,7 @@ OPTIMIZATIONS:
|
||||
-stream stream mode - start elaborating without sorting the input
|
||||
-ss, -scan-strategy value strategy to use while scanning(auto/host-spray/template-spray) (default auto)
|
||||
-irt, -input-read-timeout value timeout on input read (default 3m0s)
|
||||
-response-cache reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
|
||||
-nh, -no-httpx disable httpx probing for non-url input
|
||||
-preflight-portscan run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
|
||||
-no-stdin disable stdin processing
|
||||
|
||||
+16
-13
@@ -310,19 +310,21 @@ UNCOVER:
|
||||
-ur, -uncover-ratelimit int override ratelimit of engines with unknown ratelimit (default 60 req/min) (default 60)
|
||||
|
||||
RATE-LIMIT:
|
||||
-rl, -rate-limit int maximum number of requests to send per second (default 150)
|
||||
-rld, -rate-limit-duration value maximum number of requests to send per second (default 1s)
|
||||
-host-backoff slow down a host that starts blocking (429, repeated 403s, connection failures)
|
||||
-host-backoff-max value maximum delay applied to a host that is blocking (default 30s)
|
||||
-per-host-rate-limit enable per-host rate limiting (global rate limit becomes unlimited when enabled)
|
||||
-rlm, -rate-limit-minute int maximum number of requests to send per minute (DEPRECATED)
|
||||
-bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25)
|
||||
-c, -concurrency int maximum number of templates to be executed in parallel (default 25)
|
||||
-hbs, -headless-bulk-size int maximum number of headless hosts to be analyzed in parallel per template (default 10)
|
||||
-headc, -headless-concurrency int maximum number of headless templates to be executed in parallel (default 10)
|
||||
-jsc, -js-concurrency int maximum number of javascript runtimes to be executed in parallel (default 120)
|
||||
-pc, -payload-concurrency int max payload concurrency for each template (default 25)
|
||||
-prc, -probe-concurrency int http probe concurrency with httpx (default 50)
|
||||
-rl, -rate-limit int maximum number of requests to send per second (default 150)
|
||||
-rld, -rate-limit-duration value maximum number of requests to send per second (default 1s)
|
||||
-disable-adaptive-concurrency do not adapt in-flight requests per host:port to its latency and errors
|
||||
-disable-service-placement run js templates that only speak non-http protocols (smb, ssh, mysql, ...) even on ports that answer http
|
||||
-host-backoff slow down a host that starts blocking (429, repeated 403s, connection failures)
|
||||
-host-backoff-max value maximum delay applied to a host that is blocking (default 30s)
|
||||
-per-host-rate-limit enable per-host rate limiting (global rate limit becomes unlimited when enabled)
|
||||
-rlm, -rate-limit-minute int maximum number of requests to send per minute (DEPRECATED)
|
||||
-bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25)
|
||||
-c, -concurrency int maximum number of templates to be executed in parallel (default 25, or 100 with adaptive concurrency)
|
||||
-hbs, -headless-bulk-size int maximum number of headless hosts to be analyzed in parallel per template (default 10)
|
||||
-headc, -headless-concurrency int maximum number of headless templates to be executed in parallel (default 10)
|
||||
-jsc, -js-concurrency int maximum number of javascript runtimes to be executed in parallel (default 120)
|
||||
-pc, -payload-concurrency int max payload concurrency for each template (default 25)
|
||||
-prc, -probe-concurrency int http probe concurrency with httpx (default 50)
|
||||
-tlc, -template-loading-concurrency int maximum number of concurrent template loading operations (default 50)
|
||||
|
||||
OPTIMIZATIONS:
|
||||
@@ -338,6 +340,7 @@ OPTIMIZATIONS:
|
||||
-stream stream mode - start elaborating without sorting the input
|
||||
-ss, -scan-strategy value strategy to use while scanning(auto/host-spray/template-spray) (default auto)
|
||||
-irt, -input-read-timeout value timeout on input read (default 3m0s)
|
||||
-response-cache reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
|
||||
-nh, -no-httpx disable httpx probing for non-url input
|
||||
-preflight-portscan run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
|
||||
-no-stdin disable stdin processing
|
||||
|
||||
+16
-11
@@ -292,17 +292,21 @@ UNCOVER:
|
||||
-ur, -uncover-ratelimit int override ratelimit of engines with unknown ratelimit (default 60 req/min) (default 60)
|
||||
|
||||
RATE-LIMIT:
|
||||
-rl, -rate-limit int maximum number of requests to send per second (default 150)
|
||||
-rld, -rate-limit-duration value maximum number of requests to send per second (default 1s)
|
||||
-per-host-rate-limit enable per-host rate limiting (global rate limit becomes unlimited when enabled)
|
||||
-rlm, -rate-limit-minute int maximum number of requests to send per minute (DEPRECATED)
|
||||
-bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25)
|
||||
-c, -concurrency int maximum number of templates to be executed in parallel (default 25)
|
||||
-hbs, -headless-bulk-size int maximum number of headless hosts to be analyzed in parallel per template (default 10)
|
||||
-headc, -headless-concurrency int maximum number of headless templates to be executed in parallel (default 10)
|
||||
-jsc, -js-concurrency int maximum number of javascript runtimes to be executed in parallel (default 120)
|
||||
-pc, -payload-concurrency int max payload concurrency for each template (default 25)
|
||||
-prc, -probe-concurrency int http probe concurrency with httpx (default 50)
|
||||
-rl, -rate-limit int maximum number of requests to send per second (default 150)
|
||||
-rld, -rate-limit-duration value maximum number of requests to send per second (default 1s)
|
||||
-disable-adaptive-concurrency do not adapt in-flight requests per host:port to its latency and errors
|
||||
-disable-service-placement run js templates that only speak non-http protocols (smb, ssh, mysql, ...) even on ports that answer http
|
||||
-host-backoff slow down a host that starts blocking (429, repeated 403s, connection failures)
|
||||
-host-backoff-max value maximum delay applied to a host that is blocking (default 30s)
|
||||
-per-host-rate-limit enable per-host rate limiting (global rate limit becomes unlimited when enabled)
|
||||
-rlm, -rate-limit-minute int maximum number of requests to send per minute (DEPRECATED)
|
||||
-bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25)
|
||||
-c, -concurrency int maximum number of templates to be executed in parallel (default 25, or 100 with adaptive concurrency)
|
||||
-hbs, -headless-bulk-size int maximum number of headless hosts to be analyzed in parallel per template (default 10)
|
||||
-headc, -headless-concurrency int maximum number of headless templates to be executed in parallel (default 10)
|
||||
-jsc, -js-concurrency int maximum number of javascript runtimes to be executed in parallel (default 120)
|
||||
-pc, -payload-concurrency int max payload concurrency for each template (default 25)
|
||||
-prc, -probe-concurrency int http probe concurrency with httpx (default 50)
|
||||
-tlc, -template-loading-concurrency int maximum number of concurrent template loading operations (default 50)
|
||||
|
||||
OPTIMIZATIONS:
|
||||
@@ -318,6 +322,7 @@ OPTIMIZATIONS:
|
||||
-stream stream mode - start elaborating without sorting the input
|
||||
-ss, -scan-strategy value strategy to use while scanning(auto/host-spray/template-spray) (default auto)
|
||||
-irt, -input-read-timeout value timeout on input read (default 3m0s)
|
||||
-response-cache reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
|
||||
-nh, -no-httpx disable httpx probing for non-url input
|
||||
-preflight-portscan run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
|
||||
-no-stdin disable stdin processing
|
||||
|
||||
+16
-11
@@ -292,17 +292,21 @@ UNCOVER:
|
||||
-ur, -uncover-ratelimit int override ratelimit of engines with unknown ratelimit (default 60 req/min) (default 60)
|
||||
|
||||
RATE-LIMIT:
|
||||
-rl, -rate-limit int maximum number of requests to send per second (default 150)
|
||||
-rld, -rate-limit-duration value maximum number of requests to send per second (default 1s)
|
||||
-per-host-rate-limit enable per-host rate limiting (global rate limit becomes unlimited when enabled)
|
||||
-rlm, -rate-limit-minute int maximum number of requests to send per minute (DEPRECATED)
|
||||
-bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25)
|
||||
-c, -concurrency int maximum number of templates to be executed in parallel (default 25)
|
||||
-hbs, -headless-bulk-size int maximum number of headless hosts to be analyzed in parallel per template (default 10)
|
||||
-headc, -headless-concurrency int maximum number of headless templates to be executed in parallel (default 10)
|
||||
-jsc, -js-concurrency int maximum number of javascript runtimes to be executed in parallel (default 120)
|
||||
-pc, -payload-concurrency int max payload concurrency for each template (default 25)
|
||||
-prc, -probe-concurrency int http probe concurrency with httpx (default 50)
|
||||
-rl, -rate-limit int maximum number of requests to send per second (default 150)
|
||||
-rld, -rate-limit-duration value maximum number of requests to send per second (default 1s)
|
||||
-disable-adaptive-concurrency do not adapt in-flight requests per host:port to its latency and errors
|
||||
-disable-service-placement run js templates that only speak non-http protocols (smb, ssh, mysql, ...) even on ports that answer http
|
||||
-host-backoff slow down a host that starts blocking (429, repeated 403s, connection failures)
|
||||
-host-backoff-max value maximum delay applied to a host that is blocking (default 30s)
|
||||
-per-host-rate-limit enable per-host rate limiting (global rate limit becomes unlimited when enabled)
|
||||
-rlm, -rate-limit-minute int maximum number of requests to send per minute (DEPRECATED)
|
||||
-bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25)
|
||||
-c, -concurrency int maximum number of templates to be executed in parallel (default 25, or 100 with adaptive concurrency)
|
||||
-hbs, -headless-bulk-size int maximum number of headless hosts to be analyzed in parallel per template (default 10)
|
||||
-headc, -headless-concurrency int maximum number of headless templates to be executed in parallel (default 10)
|
||||
-jsc, -js-concurrency int maximum number of javascript runtimes to be executed in parallel (default 120)
|
||||
-pc, -payload-concurrency int max payload concurrency for each template (default 25)
|
||||
-prc, -probe-concurrency int http probe concurrency with httpx (default 50)
|
||||
-tlc, -template-loading-concurrency int maximum number of concurrent template loading operations (default 50)
|
||||
|
||||
OPTIMIZATIONS:
|
||||
@@ -318,6 +322,7 @@ OPTIMIZATIONS:
|
||||
-stream stream mode - start elaborating without sorting the input
|
||||
-ss, -scan-strategy value strategy to use while scanning(auto/host-spray/template-spray) (default auto)
|
||||
-irt, -input-read-timeout value timeout on input read (default 3m0s)
|
||||
-response-cache reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
|
||||
-nh, -no-httpx disable httpx probing for non-url input
|
||||
-preflight-portscan run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
|
||||
-no-stdin disable stdin processing
|
||||
|
||||
+16
-11
@@ -292,17 +292,21 @@ UNCOVER:
|
||||
-ur, -uncover-ratelimit int override ratelimit of engines with unknown ratelimit (default 60 req/min) (default 60)
|
||||
|
||||
RATE-LIMIT:
|
||||
-rl, -rate-limit int maximum number of requests to send per second (default 150)
|
||||
-rld, -rate-limit-duration value maximum number of requests to send per second (default 1s)
|
||||
-per-host-rate-limit enable per-host rate limiting (global rate limit becomes unlimited when enabled)
|
||||
-rlm, -rate-limit-minute int maximum number of requests to send per minute (DEPRECATED)
|
||||
-bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25)
|
||||
-c, -concurrency int maximum number of templates to be executed in parallel (default 25)
|
||||
-hbs, -headless-bulk-size int maximum number of headless hosts to be analyzed in parallel per template (default 10)
|
||||
-headc, -headless-concurrency int maximum number of headless templates to be executed in parallel (default 10)
|
||||
-jsc, -js-concurrency int maximum number of javascript runtimes to be executed in parallel (default 120)
|
||||
-pc, -payload-concurrency int max payload concurrency for each template (default 25)
|
||||
-prc, -probe-concurrency int http probe concurrency with httpx (default 50)
|
||||
-rl, -rate-limit int maximum number of requests to send per second (default 150)
|
||||
-rld, -rate-limit-duration value maximum number of requests to send per second (default 1s)
|
||||
-disable-adaptive-concurrency do not adapt in-flight requests per host:port to its latency and errors
|
||||
-disable-service-placement run js templates that only speak non-http protocols (smb, ssh, mysql, ...) even on ports that answer http
|
||||
-host-backoff slow down a host that starts blocking (429, repeated 403s, connection failures)
|
||||
-host-backoff-max value maximum delay applied to a host that is blocking (default 30s)
|
||||
-per-host-rate-limit enable per-host rate limiting (global rate limit becomes unlimited when enabled)
|
||||
-rlm, -rate-limit-minute int maximum number of requests to send per minute (DEPRECATED)
|
||||
-bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25)
|
||||
-c, -concurrency int maximum number of templates to be executed in parallel (default 25, or 100 with adaptive concurrency)
|
||||
-hbs, -headless-bulk-size int maximum number of headless hosts to be analyzed in parallel per template (default 10)
|
||||
-headc, -headless-concurrency int maximum number of headless templates to be executed in parallel (default 10)
|
||||
-jsc, -js-concurrency int maximum number of javascript runtimes to be executed in parallel (default 120)
|
||||
-pc, -payload-concurrency int max payload concurrency for each template (default 25)
|
||||
-prc, -probe-concurrency int http probe concurrency with httpx (default 50)
|
||||
-tlc, -template-loading-concurrency int maximum number of concurrent template loading operations (default 50)
|
||||
|
||||
OPTIMIZATIONS:
|
||||
@@ -318,6 +322,7 @@ OPTIMIZATIONS:
|
||||
-stream stream mode - start elaborating without sorting the input
|
||||
-ss, -scan-strategy value strategy to use while scanning(auto/host-spray/template-spray) (default auto)
|
||||
-irt, -input-read-timeout value timeout on input read (default 3m0s)
|
||||
-response-cache reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
|
||||
-nh, -no-httpx disable httpx probing for non-url input
|
||||
-preflight-portscan run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
|
||||
-no-stdin disable stdin processing
|
||||
|
||||
+16
-11
@@ -292,17 +292,21 @@ UNCOVER:
|
||||
-ur, -uncover-ratelimit int override ratelimit of engines with unknown ratelimit (default 60 req/min) (default 60)
|
||||
|
||||
RATE-LIMIT:
|
||||
-rl, -rate-limit int maximum number of requests to send per second (default 150)
|
||||
-rld, -rate-limit-duration value maximum number of requests to send per second (default 1s)
|
||||
-per-host-rate-limit enable per-host rate limiting (global rate limit becomes unlimited when enabled)
|
||||
-rlm, -rate-limit-minute int maximum number of requests to send per minute (DEPRECATED)
|
||||
-bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25)
|
||||
-c, -concurrency int maximum number of templates to be executed in parallel (default 25)
|
||||
-hbs, -headless-bulk-size int maximum number of headless hosts to be analyzed in parallel per template (default 10)
|
||||
-headc, -headless-concurrency int maximum number of headless templates to be executed in parallel (default 10)
|
||||
-jsc, -js-concurrency int maximum number of javascript runtimes to be executed in parallel (default 120)
|
||||
-pc, -payload-concurrency int max payload concurrency for each template (default 25)
|
||||
-prc, -probe-concurrency int http probe concurrency with httpx (default 50)
|
||||
-rl, -rate-limit int maximum number of requests to send per second (default 150)
|
||||
-rld, -rate-limit-duration value maximum number of requests to send per second (default 1s)
|
||||
-disable-adaptive-concurrency do not adapt in-flight requests per host:port to its latency and errors
|
||||
-disable-service-placement run js templates that only speak non-http protocols (smb, ssh, mysql, ...) even on ports that answer http
|
||||
-host-backoff slow down a host that starts blocking (429, repeated 403s, connection failures)
|
||||
-host-backoff-max value maximum delay applied to a host that is blocking (default 30s)
|
||||
-per-host-rate-limit enable per-host rate limiting (global rate limit becomes unlimited when enabled)
|
||||
-rlm, -rate-limit-minute int maximum number of requests to send per minute (DEPRECATED)
|
||||
-bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25)
|
||||
-c, -concurrency int maximum number of templates to be executed in parallel (default 25, or 100 with adaptive concurrency)
|
||||
-hbs, -headless-bulk-size int maximum number of headless hosts to be analyzed in parallel per template (default 10)
|
||||
-headc, -headless-concurrency int maximum number of headless templates to be executed in parallel (default 10)
|
||||
-jsc, -js-concurrency int maximum number of javascript runtimes to be executed in parallel (default 120)
|
||||
-pc, -payload-concurrency int max payload concurrency for each template (default 25)
|
||||
-prc, -probe-concurrency int http probe concurrency with httpx (default 50)
|
||||
-tlc, -template-loading-concurrency int maximum number of concurrent template loading operations (default 50)
|
||||
|
||||
OPTIMIZATIONS:
|
||||
@@ -318,6 +322,7 @@ OPTIMIZATIONS:
|
||||
-stream stream mode - start elaborating without sorting the input
|
||||
-ss, -scan-strategy value strategy to use while scanning(auto/host-spray/template-spray) (default auto)
|
||||
-irt, -input-read-timeout value timeout on input read (default 3m0s)
|
||||
-response-cache reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
|
||||
-nh, -no-httpx disable httpx probing for non-url input
|
||||
-preflight-portscan run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
|
||||
-no-stdin disable stdin processing
|
||||
|
||||
+16
-11
@@ -292,17 +292,21 @@ UNCOVER:
|
||||
-ur, -uncover-ratelimit int override ratelimit of engines with unknown ratelimit (default 60 req/min) (default 60)
|
||||
|
||||
RATE-LIMIT:
|
||||
-rl, -rate-limit int maximum number of requests to send per second (default 150)
|
||||
-rld, -rate-limit-duration value maximum number of requests to send per second (default 1s)
|
||||
-per-host-rate-limit enable per-host rate limiting (global rate limit becomes unlimited when enabled)
|
||||
-rlm, -rate-limit-minute int maximum number of requests to send per minute (DEPRECATED)
|
||||
-bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25)
|
||||
-c, -concurrency int maximum number of templates to be executed in parallel (default 25)
|
||||
-hbs, -headless-bulk-size int maximum number of headless hosts to be analyzed in parallel per template (default 10)
|
||||
-headc, -headless-concurrency int maximum number of headless templates to be executed in parallel (default 10)
|
||||
-jsc, -js-concurrency int maximum number of javascript runtimes to be executed in parallel (default 120)
|
||||
-pc, -payload-concurrency int max payload concurrency for each template (default 25)
|
||||
-prc, -probe-concurrency int http probe concurrency with httpx (default 50)
|
||||
-rl, -rate-limit int maximum number of requests to send per second (default 150)
|
||||
-rld, -rate-limit-duration value maximum number of requests to send per second (default 1s)
|
||||
-disable-adaptive-concurrency do not adapt in-flight requests per host:port to its latency and errors
|
||||
-disable-service-placement run js templates that only speak non-http protocols (smb, ssh, mysql, ...) even on ports that answer http
|
||||
-host-backoff slow down a host that starts blocking (429, repeated 403s, connection failures)
|
||||
-host-backoff-max value maximum delay applied to a host that is blocking (default 30s)
|
||||
-per-host-rate-limit enable per-host rate limiting (global rate limit becomes unlimited when enabled)
|
||||
-rlm, -rate-limit-minute int maximum number of requests to send per minute (DEPRECATED)
|
||||
-bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25)
|
||||
-c, -concurrency int maximum number of templates to be executed in parallel (default 25, or 100 with adaptive concurrency)
|
||||
-hbs, -headless-bulk-size int maximum number of headless hosts to be analyzed in parallel per template (default 10)
|
||||
-headc, -headless-concurrency int maximum number of headless templates to be executed in parallel (default 10)
|
||||
-jsc, -js-concurrency int maximum number of javascript runtimes to be executed in parallel (default 120)
|
||||
-pc, -payload-concurrency int max payload concurrency for each template (default 25)
|
||||
-prc, -probe-concurrency int http probe concurrency with httpx (default 50)
|
||||
-tlc, -template-loading-concurrency int maximum number of concurrent template loading operations (default 50)
|
||||
|
||||
OPTIMIZATIONS:
|
||||
@@ -318,6 +322,7 @@ OPTIMIZATIONS:
|
||||
-stream stream mode - start elaborating without sorting the input
|
||||
-ss, -scan-strategy value strategy to use while scanning(auto/host-spray/template-spray) (default auto)
|
||||
-irt, -input-read-timeout value timeout on input read (default 3m0s)
|
||||
-response-cache reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)
|
||||
-nh, -no-httpx disable httpx probing for non-url input
|
||||
-preflight-portscan run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)
|
||||
-no-stdin disable stdin processing
|
||||
|
||||
+11
-1
@@ -436,12 +436,14 @@ on extensive configurability, massive extensibility and ease of use.`)
|
||||
flagSet.CreateGroup("rate-limit", "Rate-Limit",
|
||||
flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "maximum number of requests to send per second"),
|
||||
flagSet.DurationVarP(&options.RateLimitDuration, "rate-limit-duration", "rld", time.Second, "maximum number of requests to send per second"),
|
||||
flagSet.BoolVar(&options.DisableAdaptiveConcurrency, "disable-adaptive-concurrency", false, "do not adapt in-flight requests per host:port to its latency and errors"),
|
||||
flagSet.BoolVar(&options.DisableServicePlacement, "disable-service-placement", false, "run js templates that only speak non-http protocols (smb, ssh, mysql, ...) even on ports that answer http"),
|
||||
flagSet.BoolVar(&options.HostBackoff, "host-backoff", false, "slow down a host that starts blocking (429, repeated 403s, connection failures)"),
|
||||
flagSet.DurationVar(&options.HostBackoffMax, "host-backoff-max", 30*time.Second, "maximum delay applied to a host that is blocking"),
|
||||
flagSet.BoolVar(&options.PerHostRateLimit, "per-host-rate-limit", false, "enable per-host rate limiting (global rate limit becomes unlimited when enabled)"),
|
||||
flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute (DEPRECATED)"),
|
||||
flagSet.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "maximum number of hosts to be analyzed in parallel per template"),
|
||||
flagSet.IntVarP(&options.TemplateThreads, "concurrency", "c", 25, "maximum number of templates to be executed in parallel"),
|
||||
flagSet.IntVarP(&options.TemplateThreads, "concurrency", "c", 0, "maximum number of templates to be executed in parallel (default 25, or 100 with adaptive concurrency)"),
|
||||
flagSet.IntVarP(&options.HeadlessBulkSize, "headless-bulk-size", "hbs", 10, "maximum number of headless hosts to be analyzed in parallel per template"),
|
||||
flagSet.IntVarP(&options.HeadlessTemplateThreads, "headless-concurrency", "headc", 10, "maximum number of headless templates to be executed in parallel"),
|
||||
flagSet.IntVarP(&options.JsConcurrency, "js-concurrency", "jsc", 120, "maximum number of javascript runtimes to be executed in parallel"),
|
||||
@@ -466,6 +468,7 @@ on extensive configurability, massive extensibility and ease of use.`)
|
||||
scanstrategy.TemplateSpray.String(): goflags.EnumVariable(2),
|
||||
}),
|
||||
flagSet.DurationVarP(&options.InputReadTimeout, "input-read-timeout", "irt", time.Duration(3*time.Minute), "timeout on input read"),
|
||||
flagSet.BoolVar(&options.ResponseCache, "response-cache", false, "reuse identical safe GET/HEAD responses within a scan (lossless: skips requests with auth, cookies, body or custom headers)"),
|
||||
flagSet.BoolVarP(&options.DisableHTTPProbe, "no-httpx", "nh", false, "disable httpx probing for non-url input"),
|
||||
flagSet.BoolVar(&options.PreflightPortScan, "preflight-portscan", false, "run preflight resolve + TCP portscan and filter targets before scanning (disabled by default)"),
|
||||
flagSet.BoolVar(&options.DisableStdin, "no-stdin", false, "disable stdin processing"),
|
||||
@@ -575,6 +578,13 @@ Additional documentation is available at: https://docs.nuclei.sh/getting-started
|
||||
}
|
||||
pdcpAuth = cliOnlyDynamicValue(flagSet, "auth", pdcpAuth)
|
||||
|
||||
// -c defaults to 0, meaning "not set" on the command line and in every
|
||||
// config file. With adaptive concurrency each endpoint's own limit decides
|
||||
// what a target receives, so a higher template concurrency is safe and keeps
|
||||
// endpoints busy. Any explicit value wins; without the endpoint limit the
|
||||
// historical default stays.
|
||||
options.ResolveTemplateThreads()
|
||||
|
||||
if disableUpdateCheck {
|
||||
config.DefaultConfig.DisableUpdateCheck()
|
||||
}
|
||||
|
||||
@@ -718,8 +718,12 @@ func (r *Runner) RunEnumeration() error {
|
||||
if r.options.ShouldUseHostError() {
|
||||
maxHostError := r.options.MaxHostError
|
||||
if r.options.TemplateThreads > maxHostError {
|
||||
r.Logger.Warning().Msg("The concurrency value is higher than max-host-error")
|
||||
r.Logger.Info().Msgf("Adjusting max-host-error to the concurrency value: %d", r.options.TemplateThreads)
|
||||
// The automatic -c of adaptive concurrency is expected to exceed the
|
||||
// default threshold; only an explicit value is worth a warning.
|
||||
if !r.options.TemplateThreadsAuto {
|
||||
r.Logger.Warning().Msg("The concurrency value is higher than max-host-error")
|
||||
r.Logger.Info().Msgf("Adjusting max-host-error to the concurrency value: %d", r.options.TemplateThreads)
|
||||
}
|
||||
|
||||
maxHostError = r.options.TemplateThreads
|
||||
}
|
||||
@@ -731,6 +735,8 @@ func (r *Runner) RunEnumeration() error {
|
||||
executorOpts.HostErrorsCache = cache
|
||||
}
|
||||
|
||||
executorOpts.AttachScanFeatures()
|
||||
|
||||
if r.options.HostBackoff {
|
||||
executorOpts.HostBackoff = hostbackoff.New(hostbackoff.Config{Max: r.options.HostBackoffMax})
|
||||
}
|
||||
@@ -928,6 +934,11 @@ func (r *Runner) RunEnumeration() error {
|
||||
}
|
||||
}
|
||||
|
||||
if opts := executorOpts; opts != nil && opts.HTTPResponseCache != nil {
|
||||
hits, misses, stores := opts.HTTPResponseCache.Stats()
|
||||
r.Logger.Info().Msgf("http-response-cache: hits=%d misses=%d stores=%d", hits, misses, stores)
|
||||
}
|
||||
|
||||
// todo: error propagation without canonical straight error check is required by cloud?
|
||||
// use safe dereferencing to avoid potential panics in case of previous unchecked errors
|
||||
if v := ptrutil.Safe(results); !v.Load() {
|
||||
|
||||
@@ -86,7 +86,7 @@ func New(options *Options) (*DASTServer, error) {
|
||||
}
|
||||
// Disable bulk mode and single threaded execution
|
||||
// by auto adjusting in case of default values
|
||||
if options.NucleiExecutorOptions.Options.BulkSize == 25 && options.NucleiExecutorOptions.Options.TemplateThreads == 25 {
|
||||
if opts := options.NucleiExecutorOptions.Options; opts.BulkSize == 25 && (opts.TemplateThreads == 25 || opts.TemplateThreadsAuto) {
|
||||
options.NucleiExecutorOptions.Options.BulkSize = 1
|
||||
options.NucleiExecutorOptions.Options.TemplateThreads = 1
|
||||
}
|
||||
|
||||
@@ -161,6 +161,25 @@ func WithConcurrency(opts Concurrency) NucleiSDKOptions {
|
||||
}
|
||||
}
|
||||
|
||||
// DisableServicePlacement runs JS templates that only speak non-HTTP protocols
|
||||
// (SMB, SSH, MySQL, ...) even on ports that answer HTTP. Placement is on by
|
||||
// default.
|
||||
func DisableServicePlacement() NucleiSDKOptions {
|
||||
return func(e *NucleiEngine) error {
|
||||
e.opts.DisableServicePlacement = true
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// DisableAdaptiveConcurrency turns off the per-endpoint limit that adapts
|
||||
// in-flight requests to each endpoint's latency and errors. On by default.
|
||||
func DisableAdaptiveConcurrency() NucleiSDKOptions {
|
||||
return func(e *NucleiEngine) error {
|
||||
e.opts.DisableAdaptiveConcurrency = true
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithResponseReadSize sets the maximum size of response to read in bytes.
|
||||
// A value of 0 means no limit. Recommended values: 1MB (1048576) to 10MB (10485760).
|
||||
func WithResponseReadSize(responseReadSize int) NucleiSDKOptions {
|
||||
@@ -173,6 +192,16 @@ func WithResponseReadSize(responseReadSize int) NucleiSDKOptions {
|
||||
}
|
||||
}
|
||||
|
||||
// EnableResponseCache reuses identical safe GET/HEAD responses within one scan.
|
||||
// It is off by default: a template that depends on state or timing could see a
|
||||
// different result.
|
||||
func EnableResponseCache() NucleiSDKOptions {
|
||||
return func(e *NucleiEngine) error {
|
||||
e.opts.ResponseCache = true
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithGlobalRateLimit sets global rate (i.e all hosts combined) limit options
|
||||
// Deprecated: will be removed in favour of WithGlobalRateLimitCtx in next release
|
||||
func WithGlobalRateLimit(maxTokens int, duration time.Duration) NucleiSDKOptions {
|
||||
|
||||
@@ -52,6 +52,7 @@ func createEphemeralObjects(ctx context.Context, base *NucleiEngine, opts *types
|
||||
if opts.ShouldUseHostError() && base.hostErrCache != nil {
|
||||
u.executerOpts.HostErrorsCache = base.hostErrCache
|
||||
}
|
||||
u.executerOpts.AttachScanFeatures()
|
||||
if opts.RateLimitMinute > 0 {
|
||||
opts.RateLimit = opts.RateLimitMinute
|
||||
opts.RateLimitDuration = time.Minute
|
||||
|
||||
@@ -327,6 +327,10 @@ func (e *NucleiEngine) ExecuteCallbackWithCtx(ctx context.Context, callback ...f
|
||||
return ErrNoTemplatesAvailable
|
||||
}
|
||||
|
||||
// Per-scan state is reset in place, so a response cached by an earlier scan
|
||||
// of this engine is never served to this one.
|
||||
e.executerOpts.ResetScanState()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
|
||||
@@ -292,6 +292,7 @@ func (e *NucleiEngine) init(ctx context.Context) error {
|
||||
if e.opts.ShouldUseHostError() && e.hostErrCache != nil {
|
||||
e.executerOpts.HostErrorsCache = e.hostErrCache
|
||||
}
|
||||
e.executerOpts.AttachScanFeatures()
|
||||
if len(e.opts.SecretsFile) > 0 {
|
||||
authTmplStore, err := runner.GetAuthTmplStore(e.opts, e.catalog, e.executerOpts)
|
||||
if err != nil {
|
||||
|
||||
@@ -324,9 +324,22 @@ func (i *ListInputProvider) initializeInputSources(opts *Options) error {
|
||||
|
||||
// Handle stdin
|
||||
if options.Stdin {
|
||||
if err := i.scanInputFromReader(
|
||||
options.ExecutionId,
|
||||
readerutil.TimeoutReader{Reader: os.Stdin, Timeout: time.Duration(options.InputReadTimeout)}); err != nil {
|
||||
var stdin io.Reader = readerutil.TimeoutReader{Reader: os.Stdin, Timeout: time.Duration(options.InputReadTimeout)}
|
||||
if len(options.Targets) > 0 || options.TargetsFilePath != "" {
|
||||
// stdin is not a terminal, so nuclei reads targets from it as well.
|
||||
// When it stays silent, say so instead of hanging without output:
|
||||
// CI jobs and subprocesses often inherit a stdin that never closes.
|
||||
logger := options.Logger
|
||||
if logger == nil {
|
||||
logger = gologger.DefaultLogger
|
||||
}
|
||||
notice := newNoticeReader(stdin, stdinNoticeDelay, func() {
|
||||
logger.Info().Msgf("Waiting for targets on stdin (up to %s, -input-read-timeout); pass -no-stdin if no targets are piped", time.Duration(options.InputReadTimeout))
|
||||
})
|
||||
defer notice.stop()
|
||||
stdin = notice
|
||||
}
|
||||
if err := i.scanInputFromReader(options.ExecutionId, stdin); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
package list
|
||||
|
||||
import (
|
||||
"io"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// stdinNoticeDelay is how long stdin may stay silent before nuclei says it is
|
||||
// waiting, when targets were also given on the command line.
|
||||
const stdinNoticeDelay = 2 * time.Second
|
||||
|
||||
// noticeReader calls notify once if no byte arrives within delay. It never
|
||||
// changes what is read: stdin is still consumed until EOF or the read timeout,
|
||||
// so targets from a slow producer (`subfinder | nuclei -u extra`) are kept.
|
||||
type noticeReader struct {
|
||||
r io.Reader
|
||||
got atomic.Bool
|
||||
timer *time.Timer
|
||||
stopMu sync.Once
|
||||
}
|
||||
|
||||
func newNoticeReader(r io.Reader, delay time.Duration, notify func()) *noticeReader {
|
||||
n := ¬iceReader{r: r}
|
||||
n.timer = time.AfterFunc(delay, func() {
|
||||
if !n.got.Load() {
|
||||
notify()
|
||||
}
|
||||
})
|
||||
return n
|
||||
}
|
||||
|
||||
func (n *noticeReader) Read(p []byte) (int, error) {
|
||||
c, err := n.r.Read(p)
|
||||
if c > 0 || err != nil {
|
||||
n.got.Store(true)
|
||||
n.stop()
|
||||
}
|
||||
return c, err
|
||||
}
|
||||
|
||||
func (n *noticeReader) stop() {
|
||||
n.stopMu.Do(func() { n.timer.Stop() })
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
package list
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"io"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// waitFor fails the test if ch does not deliver within d.
|
||||
func waitFor(t *testing.T, ch <-chan struct{}, d time.Duration, what string) {
|
||||
t.Helper()
|
||||
select {
|
||||
case <-ch:
|
||||
case <-time.After(d):
|
||||
t.Fatalf("timed out waiting for %s", what)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoticeOnSilentStdin(t *testing.T) {
|
||||
pr, pw := io.Pipe()
|
||||
defer func() { _ = pw.Close() }()
|
||||
notified := make(chan struct{}, 1)
|
||||
n := newNoticeReader(pr, 10*time.Millisecond, func() { notified <- struct{}{} })
|
||||
go func() { buf := make([]byte, 8); _, _ = n.Read(buf) }()
|
||||
waitFor(t, notified, 5*time.Second, "the notice on a silent stdin")
|
||||
}
|
||||
|
||||
func TestNoNoticeWhenDataArrives(t *testing.T) {
|
||||
notified := make(chan struct{}, 1)
|
||||
// A reader that already holds data: the first Read delivers bytes, which
|
||||
// stops the timer before it can fire.
|
||||
n := newNoticeReader(stringsReader("a.example\n"), time.Hour, func() { notified <- struct{}{} })
|
||||
if _, err := io.ReadAll(n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
n.stop()
|
||||
select {
|
||||
case <-notified:
|
||||
t.Fatal("must not warn when stdin delivers data")
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// Targets that arrive after the notice are still read: the notice never
|
||||
// changes what is scanned. The writer starts only after the notice fired.
|
||||
func TestLateTargetsStillRead(t *testing.T) {
|
||||
pr, pw := io.Pipe()
|
||||
notified := make(chan struct{}, 1)
|
||||
n := newNoticeReader(pr, 10*time.Millisecond, func() { notified <- struct{}{} })
|
||||
go func() {
|
||||
defer func() { _ = pw.Close() }()
|
||||
select {
|
||||
case <-notified:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Error("timed out waiting for the notice before the late data")
|
||||
return
|
||||
}
|
||||
_, _ = pw.Write([]byte("a.example\nb.example\n"))
|
||||
}()
|
||||
var lines []string
|
||||
sc := bufio.NewScanner(n)
|
||||
for sc.Scan() {
|
||||
lines = append(lines, sc.Text())
|
||||
}
|
||||
if len(lines) != 2 {
|
||||
t.Fatalf("late targets lost: %v", lines)
|
||||
}
|
||||
}
|
||||
|
||||
type stringReader struct {
|
||||
s string
|
||||
i int
|
||||
}
|
||||
|
||||
func stringsReader(s string) *stringReader { return &stringReader{s: s} }
|
||||
|
||||
func (r *stringReader) Read(p []byte) (int, error) {
|
||||
if r.i >= len(r.s) {
|
||||
return 0, io.EOF
|
||||
}
|
||||
n := copy(p, r.s[r.i:])
|
||||
r.i += n
|
||||
return n, nil
|
||||
}
|
||||
@@ -0,0 +1,307 @@
|
||||
// Package hostlimit bounds how many requests are in flight to one endpoint
|
||||
// (host:port) at a time, and adapts that bound to what the endpoint can take.
|
||||
//
|
||||
// A fixed template concurrency cannot be right for every target. Measured on a
|
||||
// WordPress server: 50 concurrent templates were fast and lossless, 100 lost 13
|
||||
// of 17 findings to timeouts without any visible error, and 10 were twice as
|
||||
// slow. On an nginx server the same 100 was harmless. The safe concurrency is a
|
||||
// property of the target, so it is measured while the scan runs.
|
||||
//
|
||||
// The control rule is the one TCP congestion control and adaptive concurrency
|
||||
// limiters use: grow the limit slowly while the endpoint stays healthy, cut it
|
||||
// at once when it shows strain. Healthy means the median latency of ordinary
|
||||
// requests stays close to the endpoint's own baseline and no overload errors
|
||||
// (timeouts, resets, 429, 503) appear. Requests that are slow by design, such as
|
||||
// time-based checks with a custom timeout, are admitted and counted, but they do
|
||||
// not feed the latency signal.
|
||||
package hostlimit
|
||||
|
||||
import (
|
||||
"context"
|
||||
stderrors "errors"
|
||||
"io"
|
||||
"net"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Config tunes the limiter. Zero fields take the defaults.
|
||||
type Config struct {
|
||||
// Initial is the limit an endpoint starts with.
|
||||
Initial int
|
||||
// Min and Max bound the limit.
|
||||
Min, Max int
|
||||
// Tolerance is how much the window median latency may exceed the baseline
|
||||
// before the endpoint counts as strained (2 = twice the baseline).
|
||||
Tolerance float64
|
||||
// Window is how many latency samples one control decision uses.
|
||||
Window int
|
||||
}
|
||||
|
||||
func (c *Config) defaults() {
|
||||
if c.Initial <= 0 {
|
||||
c.Initial = 16
|
||||
}
|
||||
if c.Min <= 0 {
|
||||
c.Min = 2
|
||||
}
|
||||
if c.Max <= 0 {
|
||||
c.Max = 256
|
||||
}
|
||||
if c.Max < c.Min {
|
||||
c.Max = c.Min
|
||||
}
|
||||
if c.Initial < c.Min {
|
||||
c.Initial = c.Min
|
||||
}
|
||||
if c.Initial > c.Max {
|
||||
c.Initial = c.Max
|
||||
}
|
||||
if c.Tolerance <= 1 {
|
||||
c.Tolerance = 2
|
||||
}
|
||||
if c.Window <= 0 {
|
||||
c.Window = 16
|
||||
}
|
||||
}
|
||||
|
||||
// Limiter holds one adaptive limit per endpoint for a scan.
|
||||
type Limiter struct {
|
||||
cfg Config
|
||||
mu sync.Mutex
|
||||
endpoints map[string]*endpoint
|
||||
}
|
||||
|
||||
// New returns a limiter with cfg, unset fields defaulted.
|
||||
func New(cfg Config) *Limiter {
|
||||
cfg.defaults()
|
||||
return &Limiter{cfg: cfg, endpoints: make(map[string]*endpoint)}
|
||||
}
|
||||
|
||||
// Outcome describes how one admitted request ended.
|
||||
type Outcome struct {
|
||||
// Overload is set when the endpoint showed strain: a timeout, a reset, a
|
||||
// refused connection on an endpoint that was answering, or a 429/503.
|
||||
Overload bool
|
||||
// Sample is set when the latency is a fair health signal (an ordinary
|
||||
// request with the default timeout).
|
||||
Sample bool
|
||||
// Latency is the time from admission to the response headers.
|
||||
Latency time.Duration
|
||||
}
|
||||
|
||||
// Ticket is one admission. Release it exactly once.
|
||||
type Ticket struct {
|
||||
e *endpoint
|
||||
start time.Time
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
// Acquire waits until the endpoint has room, or ctx ends.
|
||||
func (l *Limiter) Acquire(ctx context.Context, key string) (*Ticket, error) {
|
||||
if l == nil || key == "" {
|
||||
return &Ticket{}, nil
|
||||
}
|
||||
e := l.endpoint(key)
|
||||
if err := e.acquire(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Ticket{e: e, start: time.Now()}, nil
|
||||
}
|
||||
|
||||
// Since returns the time elapsed since admission.
|
||||
func (t *Ticket) Since() time.Duration {
|
||||
if t == nil || t.e == nil {
|
||||
return 0
|
||||
}
|
||||
return time.Since(t.start)
|
||||
}
|
||||
|
||||
// Release returns the slot and feeds the outcome to the endpoint's control loop.
|
||||
func (t *Ticket) Release(o Outcome) {
|
||||
if t == nil || t.e == nil {
|
||||
return
|
||||
}
|
||||
t.once.Do(func() { t.e.release(o) })
|
||||
}
|
||||
|
||||
// Limit reports the current limit of an endpoint (tests and stats).
|
||||
func (l *Limiter) Limit(key string) int {
|
||||
if l == nil {
|
||||
return 0
|
||||
}
|
||||
l.mu.Lock()
|
||||
e, ok := l.endpoints[key]
|
||||
l.mu.Unlock()
|
||||
if !ok {
|
||||
return l.cfg.Initial
|
||||
}
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
return int(e.limit)
|
||||
}
|
||||
|
||||
func (l *Limiter) endpoint(key string) *endpoint {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
e, ok := l.endpoints[key]
|
||||
if !ok {
|
||||
e = &endpoint{cfg: &l.cfg, limit: float64(l.cfg.Initial)}
|
||||
l.endpoints[key] = e
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
type endpoint struct {
|
||||
cfg *Config
|
||||
mu sync.Mutex
|
||||
|
||||
limit float64
|
||||
inflight int
|
||||
waiters []chan struct{}
|
||||
|
||||
samples []time.Duration
|
||||
overloads int
|
||||
peakUse int
|
||||
baseline time.Duration
|
||||
}
|
||||
|
||||
func (e *endpoint) acquire(ctx context.Context) error {
|
||||
e.mu.Lock()
|
||||
if e.inflight < int(e.limit) && len(e.waiters) == 0 {
|
||||
e.inflight++
|
||||
e.notePeak()
|
||||
e.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
ch := make(chan struct{})
|
||||
e.waiters = append(e.waiters, ch)
|
||||
e.mu.Unlock()
|
||||
|
||||
select {
|
||||
case <-ch:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
e.mu.Lock()
|
||||
for i, w := range e.waiters {
|
||||
if w == ch {
|
||||
e.waiters = append(e.waiters[:i], e.waiters[i+1:]...)
|
||||
e.mu.Unlock()
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
e.mu.Unlock()
|
||||
// The slot was handed over while ctx ended; give it back.
|
||||
e.release(Outcome{})
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// notePeak records how much of the limit the endpoint actually used in the
|
||||
// current window. Must hold e.mu.
|
||||
func (e *endpoint) notePeak() {
|
||||
if e.inflight > e.peakUse {
|
||||
e.peakUse = e.inflight
|
||||
}
|
||||
}
|
||||
|
||||
func (e *endpoint) release(o Outcome) {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
e.inflight--
|
||||
if o.Overload {
|
||||
e.overloads++
|
||||
}
|
||||
if o.Sample && o.Latency > 0 {
|
||||
e.samples = append(e.samples, o.Latency)
|
||||
}
|
||||
if o.Overload || len(e.samples) >= e.cfg.Window {
|
||||
e.adjust()
|
||||
}
|
||||
e.wake()
|
||||
}
|
||||
|
||||
// adjust runs one control decision. Must hold e.mu.
|
||||
func (e *endpoint) adjust() {
|
||||
minL, maxL := float64(e.cfg.Min), float64(e.cfg.Max)
|
||||
defer func() {
|
||||
e.samples = e.samples[:0]
|
||||
e.overloads = 0
|
||||
e.peakUse = e.inflight
|
||||
}()
|
||||
|
||||
if e.overloads > 0 {
|
||||
e.limit = clamp(e.limit*0.5, minL, maxL)
|
||||
return
|
||||
}
|
||||
if len(e.samples) == 0 {
|
||||
return
|
||||
}
|
||||
median := medianOf(e.samples)
|
||||
switch {
|
||||
case e.baseline == 0 || median < e.baseline:
|
||||
e.baseline = median
|
||||
default:
|
||||
// Let the baseline drift up slowly, so one lucky early window does not
|
||||
// pin the limit forever.
|
||||
e.baseline += (median - e.baseline) / 50
|
||||
}
|
||||
if float64(median) > float64(e.baseline)*e.cfg.Tolerance {
|
||||
e.limit = clamp(e.limit*0.8, minL, maxL)
|
||||
return
|
||||
}
|
||||
// Grow only when the limit was the bottleneck; an idle endpoint keeps its
|
||||
// limit, so a later burst cannot arrive at an untested size.
|
||||
if float64(e.peakUse) >= e.limit-1 {
|
||||
e.limit = clamp(e.limit+2, minL, maxL)
|
||||
}
|
||||
}
|
||||
|
||||
// wake hands free slots to waiters in arrival order. Must hold e.mu.
|
||||
func (e *endpoint) wake() {
|
||||
for len(e.waiters) > 0 && e.inflight < int(e.limit) {
|
||||
ch := e.waiters[0]
|
||||
e.waiters = e.waiters[1:]
|
||||
e.inflight++
|
||||
e.notePeak()
|
||||
close(ch)
|
||||
}
|
||||
}
|
||||
|
||||
func medianOf(s []time.Duration) time.Duration {
|
||||
c := append([]time.Duration(nil), s...)
|
||||
sort.Slice(c, func(i, j int) bool { return c[i] < c[j] })
|
||||
return c[len(c)/2]
|
||||
}
|
||||
|
||||
func clamp(v, lo, hi float64) float64 {
|
||||
if v < lo {
|
||||
return lo
|
||||
}
|
||||
if v > hi {
|
||||
return hi
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// IsOverload reports whether a transport error suggests the endpoint is
|
||||
// overloaded rather than the scan stopping: a timeout, a reset, a refused
|
||||
// connection or a truncated reply. A cancelled context is not overload.
|
||||
func IsOverload(err error) bool {
|
||||
if err == nil || stderrors.Is(err, context.Canceled) {
|
||||
return false
|
||||
}
|
||||
var ne net.Error
|
||||
if stderrors.As(err, &ne) && ne.Timeout() {
|
||||
return true
|
||||
}
|
||||
if stderrors.Is(err, context.DeadlineExceeded) || stderrors.Is(err, syscall.ECONNRESET) ||
|
||||
stderrors.Is(err, syscall.ECONNREFUSED) || stderrors.Is(err, io.ErrUnexpectedEOF) {
|
||||
return true
|
||||
}
|
||||
msg := strings.ToLower(err.Error())
|
||||
return strings.Contains(msg, "timeout") || strings.Contains(msg, "connection reset") || strings.Contains(msg, "connection refused")
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
package hostlimit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// In-flight requests per endpoint never exceed the limit.
|
||||
func TestNeverExceedsLimit(t *testing.T) {
|
||||
l := New(Config{Initial: 4, Min: 4, Max: 4, Window: 1000})
|
||||
var cur, peak atomic.Int64
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 200; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
tk, err := l.Acquire(context.Background(), "h:80")
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
n := cur.Add(1)
|
||||
for {
|
||||
p := peak.Load()
|
||||
if n <= p || peak.CompareAndSwap(p, n) {
|
||||
break
|
||||
}
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
cur.Add(-1)
|
||||
tk.Release(Outcome{})
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
if peak.Load() > 4 {
|
||||
t.Fatalf("peak in flight %d > limit 4", peak.Load())
|
||||
}
|
||||
}
|
||||
|
||||
// Endpoints are independent: a saturated one does not block another.
|
||||
func TestEndpointsAreIndependent(t *testing.T) {
|
||||
l := New(Config{Initial: 2, Min: 2, Max: 2})
|
||||
a1, _ := l.Acquire(context.Background(), "a:80")
|
||||
a2, _ := l.Acquire(context.Background(), "a:80")
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
b, err := l.Acquire(context.Background(), "b:80")
|
||||
if err == nil {
|
||||
b.Release(Outcome{})
|
||||
}
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("endpoint b was blocked by endpoint a")
|
||||
}
|
||||
a1.Release(Outcome{})
|
||||
a2.Release(Outcome{})
|
||||
}
|
||||
|
||||
// A healthy endpoint that uses its whole limit grows.
|
||||
func TestGrowsWhileHealthy(t *testing.T) {
|
||||
l := New(Config{Initial: 4, Min: 2, Max: 64, Window: 4})
|
||||
for round := 0; round < 20; round++ {
|
||||
var tks []*Ticket
|
||||
for i := 0; i < l.Limit("h:80"); i++ {
|
||||
tk, _ := l.Acquire(context.Background(), "h:80")
|
||||
tks = append(tks, tk)
|
||||
}
|
||||
for _, tk := range tks {
|
||||
tk.Release(Outcome{Sample: true, Latency: 10 * time.Millisecond})
|
||||
}
|
||||
}
|
||||
if got := l.Limit("h:80"); got <= 4 {
|
||||
t.Fatalf("limit did not grow on a healthy saturated endpoint: %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// An overload error halves the limit at once.
|
||||
func TestCutsOnOverload(t *testing.T) {
|
||||
l := New(Config{Initial: 40, Min: 2, Max: 64})
|
||||
tk, _ := l.Acquire(context.Background(), "h:80")
|
||||
tk.Release(Outcome{Overload: true})
|
||||
if got := l.Limit("h:80"); got != 20 {
|
||||
t.Fatalf("limit after overload = %d want 20", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Rising latency cuts the limit even without errors.
|
||||
func TestCutsOnRisingLatency(t *testing.T) {
|
||||
l := New(Config{Initial: 32, Min: 2, Max: 64, Window: 4})
|
||||
feed := func(lat time.Duration) {
|
||||
for i := 0; i < 4; i++ {
|
||||
tk, _ := l.Acquire(context.Background(), "h:80")
|
||||
tk.Release(Outcome{Sample: true, Latency: lat})
|
||||
}
|
||||
}
|
||||
feed(10 * time.Millisecond) // baseline
|
||||
before := l.Limit("h:80")
|
||||
feed(100 * time.Millisecond) // 10x the baseline
|
||||
if got := l.Limit("h:80"); got >= before {
|
||||
t.Fatalf("limit did not drop on rising latency: before=%d after=%d", before, got)
|
||||
}
|
||||
}
|
||||
|
||||
// Slow-by-design requests do not feed the signal.
|
||||
func TestNonSampleLatencyIgnored(t *testing.T) {
|
||||
l := New(Config{Initial: 32, Min: 2, Max: 64, Window: 4})
|
||||
for i := 0; i < 4; i++ {
|
||||
tk, _ := l.Acquire(context.Background(), "h:80")
|
||||
tk.Release(Outcome{Sample: true, Latency: 10 * time.Millisecond})
|
||||
}
|
||||
before := l.Limit("h:80")
|
||||
for i := 0; i < 40; i++ {
|
||||
tk, _ := l.Acquire(context.Background(), "h:80")
|
||||
tk.Release(Outcome{Sample: false, Latency: 15 * time.Second})
|
||||
}
|
||||
if got := l.Limit("h:80"); got < before {
|
||||
t.Fatalf("slow-by-design requests cut the limit: before=%d after=%d", before, got)
|
||||
}
|
||||
}
|
||||
|
||||
// A waiter whose context ends leaves the queue and does not leak a slot.
|
||||
func TestCancelledWaiterDoesNotLeak(t *testing.T) {
|
||||
l := New(Config{Initial: 1, Min: 1, Max: 1})
|
||||
held, _ := l.Acquire(context.Background(), "h:80")
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
|
||||
defer cancel()
|
||||
if _, err := l.Acquire(ctx, "h:80"); err == nil {
|
||||
t.Fatal("expected context error")
|
||||
}
|
||||
held.Release(Outcome{})
|
||||
got, err := l.Acquire(context.Background(), "h:80")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got.Release(Outcome{})
|
||||
}
|
||||
|
||||
// Releasing twice is harmless.
|
||||
func TestDoubleReleaseIsSafe(t *testing.T) {
|
||||
l := New(Config{Initial: 1, Min: 1, Max: 1})
|
||||
tk, _ := l.Acquire(context.Background(), "h:80")
|
||||
tk.Release(Outcome{})
|
||||
tk.Release(Outcome{})
|
||||
tk2, _ := l.Acquire(context.Background(), "h:80")
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
tk3, _ := l.Acquire(context.Background(), "h:80")
|
||||
tk3.Release(Outcome{})
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
t.Fatal("a double release created an extra slot")
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
tk2.Release(Outcome{})
|
||||
<-done
|
||||
}
|
||||
|
||||
func TestNilLimiter(t *testing.T) {
|
||||
var l *Limiter
|
||||
tk, err := l.Acquire(context.Background(), "h:80")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tk.Release(Outcome{})
|
||||
}
|
||||
|
||||
type timeoutErr struct{}
|
||||
|
||||
func (timeoutErr) Error() string { return "i/o timeout" }
|
||||
func (timeoutErr) Timeout() bool { return true }
|
||||
func (timeoutErr) Temporary() bool { return true }
|
||||
|
||||
func TestIsOverload(t *testing.T) {
|
||||
cases := []struct {
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{nil, false},
|
||||
{context.Canceled, false},
|
||||
{context.DeadlineExceeded, true},
|
||||
{timeoutErr{}, true},
|
||||
{errors.New("read tcp: connection reset by peer"), true},
|
||||
{errors.New("dial tcp 127.0.0.1:445: connect: connection refused"), true},
|
||||
{io.ErrUnexpectedEOF, true},
|
||||
{errors.New("tls: handshake failure"), false},
|
||||
{errors.New("could not evaluate template expressions"), false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := IsOverload(c.err); got != c.want {
|
||||
t.Errorf("IsOverload(%v)=%v want %v", c.err, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
// Package httpservice answers, once per scan and per host:port, whether that
|
||||
// port speaks HTTP.
|
||||
//
|
||||
// It exists for one decision: a template that can only speak a dedicated
|
||||
// non-HTTP protocol (SMB, SSH, MySQL, ...) cannot produce a finding on a port
|
||||
// that answers HTTP, so running it there only costs time and sends garbage to
|
||||
// the target. The answer must come from evidence, never from the scheme the
|
||||
// operator typed: "http://host:22" still gets its SSH templates unless port 22
|
||||
// really answers HTTP.
|
||||
//
|
||||
// Every failure (policy denial, refusal, timeout, TLS error, a reply that is
|
||||
// not an HTTP status line) reports false, so a caller that skips work only on
|
||||
// true always fails open.
|
||||
package httpservice
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"net"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sync/singleflight"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
)
|
||||
|
||||
// DefaultTimeout bounds one probe: connect, request and the first bytes of the
|
||||
// reply.
|
||||
const DefaultTimeout = 3 * time.Second
|
||||
|
||||
// Dialer opens a connection. It matches fastdialer's Dial and DialTLS.
|
||||
type Dialer func(ctx context.Context, network, address string) (net.Conn, error)
|
||||
|
||||
// Cache memoizes probe results for one scan execution.
|
||||
type Cache struct {
|
||||
executionID string
|
||||
timeout time.Duration
|
||||
|
||||
// dial and dialTLS default to the scan's fastdialer, which enforces the
|
||||
// network policy. Tests replace them.
|
||||
dial Dialer
|
||||
dialTLS Dialer
|
||||
allowed func(hostPort string) bool
|
||||
|
||||
group singleflight.Group
|
||||
mu sync.RWMutex
|
||||
seen map[string]bool
|
||||
probes atomic.Uint64
|
||||
}
|
||||
|
||||
// New returns a cache bound to the dialers and network policy of executionID.
|
||||
func New(executionID string) *Cache {
|
||||
c := &Cache{
|
||||
executionID: executionID,
|
||||
timeout: DefaultTimeout,
|
||||
seen: make(map[string]bool),
|
||||
}
|
||||
c.allowed = func(hostPort string) bool {
|
||||
return protocolstate.IsHostAllowed(executionID, hostPort)
|
||||
}
|
||||
c.dial = func(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
d := protocolstate.GetDialersWithId(executionID)
|
||||
if d == nil || d.Fastdialer == nil {
|
||||
return nil, errNoDialer
|
||||
}
|
||||
return d.Fastdialer.Dial(ctx, network, address)
|
||||
}
|
||||
c.dialTLS = func(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
d := protocolstate.GetDialersWithId(executionID)
|
||||
if d == nil || d.Fastdialer == nil {
|
||||
return nil, errNoDialer
|
||||
}
|
||||
return d.Fastdialer.DialTLS(ctx, network, address)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
type probeError string
|
||||
|
||||
func (e probeError) Error() string { return string(e) }
|
||||
|
||||
const errNoDialer = probeError("httpservice: no dialer for this execution")
|
||||
|
||||
// SpeaksHTTP reports whether hostPort answered an HTTP request with an HTTP
|
||||
// status line. useTLS selects a TLS connection. The first call for a key
|
||||
// probes; concurrent and later calls share that result.
|
||||
func (c *Cache) SpeaksHTTP(ctx context.Context, hostPort string, useTLS bool) bool {
|
||||
if c == nil || hostPort == "" {
|
||||
return false
|
||||
}
|
||||
key := hostPort
|
||||
if useTLS {
|
||||
key = "tls://" + hostPort
|
||||
}
|
||||
c.mu.RLock()
|
||||
v, ok := c.seen[key]
|
||||
c.mu.RUnlock()
|
||||
if ok {
|
||||
return v
|
||||
}
|
||||
res, _, _ := c.group.Do(key, func() (any, error) {
|
||||
got := c.probe(ctx, hostPort, useTLS)
|
||||
c.mu.Lock()
|
||||
c.seen[key] = got
|
||||
c.mu.Unlock()
|
||||
return got, nil
|
||||
})
|
||||
return res.(bool)
|
||||
}
|
||||
|
||||
// Reset forgets every probe result, so the next scan probes again.
|
||||
func (c *Cache) Reset() {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
c.seen = make(map[string]bool)
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
// Probes returns how many network probes were sent (tests and stats).
|
||||
func (c *Cache) Probes() uint64 {
|
||||
if c == nil {
|
||||
return 0
|
||||
}
|
||||
return c.probes.Load()
|
||||
}
|
||||
|
||||
func (c *Cache) probe(ctx context.Context, hostPort string, useTLS bool) bool {
|
||||
host, _, err := net.SplitHostPort(hostPort)
|
||||
if err != nil || host == "" {
|
||||
return false
|
||||
}
|
||||
if c.allowed != nil && !c.allowed(hostPort) {
|
||||
return false
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, c.timeout)
|
||||
defer cancel()
|
||||
|
||||
dial := c.dial
|
||||
if useTLS {
|
||||
dial = c.dialTLS
|
||||
}
|
||||
if dial == nil {
|
||||
return false
|
||||
}
|
||||
c.probes.Add(1)
|
||||
conn, err := dial(ctx, "tcp", hostPort)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer func() { _ = conn.Close() }()
|
||||
if deadline, ok := ctx.Deadline(); ok {
|
||||
_ = conn.SetDeadline(deadline)
|
||||
}
|
||||
|
||||
req := "HEAD / HTTP/1.0\r\nHost: " + host + "\r\nConnection: close\r\n\r\n"
|
||||
if _, err := io.WriteString(conn, req); err != nil {
|
||||
return false
|
||||
}
|
||||
buf := make([]byte, 8)
|
||||
if _, err := io.ReadFull(conn, buf); err != nil {
|
||||
return false
|
||||
}
|
||||
return isHTTPStatusLine(buf)
|
||||
}
|
||||
|
||||
// isHTTPStatusLine reports whether b starts like an HTTP/1.x status line.
|
||||
func isHTTPStatusLine(b []byte) bool {
|
||||
return bytes.HasPrefix(b, []byte("HTTP/1."))
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
package httpservice
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func testCache() *Cache {
|
||||
d := &net.Dialer{}
|
||||
return &Cache{
|
||||
timeout: time.Second,
|
||||
seen: make(map[string]bool),
|
||||
allowed: func(string) bool { return true },
|
||||
dial: d.DialContext,
|
||||
dialTLS: d.DialContext, // TLS is not exercised in these tests
|
||||
}
|
||||
}
|
||||
|
||||
func TestSpeaksHTTPOnHTTPServer(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
|
||||
defer srv.Close()
|
||||
c := testCache()
|
||||
if !c.SpeaksHTTP(context.Background(), srv.Listener.Addr().String(), false) {
|
||||
t.Fatal("an HTTP server must be reported as HTTP")
|
||||
}
|
||||
}
|
||||
|
||||
// A service that sends its own banner (SSH, SMTP, FTP, ...) is not HTTP.
|
||||
func TestBannerServiceIsNotHTTP(t *testing.T) {
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = ln.Close() }()
|
||||
go func() {
|
||||
for {
|
||||
conn, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_, _ = conn.Write([]byte("SSH-2.0-OpenSSH_9.6\r\n"))
|
||||
_ = conn.Close()
|
||||
}
|
||||
}()
|
||||
c := testCache()
|
||||
if c.SpeaksHTTP(context.Background(), ln.Addr().String(), false) {
|
||||
t.Fatal("an SSH banner must not be reported as HTTP")
|
||||
}
|
||||
}
|
||||
|
||||
// A service that never answers must not be reported as HTTP (fail open), and
|
||||
// the probe must respect its timeout.
|
||||
func TestSilentServiceFailsOpen(t *testing.T) {
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = ln.Close() }()
|
||||
var conns []net.Conn
|
||||
var mu sync.Mutex
|
||||
go func() {
|
||||
for {
|
||||
conn, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
mu.Lock()
|
||||
conns = append(conns, conn) // hold it open, never answer
|
||||
mu.Unlock()
|
||||
}
|
||||
}()
|
||||
c := testCache()
|
||||
c.timeout = 200 * time.Millisecond
|
||||
start := time.Now()
|
||||
if c.SpeaksHTTP(context.Background(), ln.Addr().String(), false) {
|
||||
t.Fatal("a silent service must not be reported as HTTP")
|
||||
}
|
||||
if time.Since(start) > 2*time.Second {
|
||||
t.Fatal("the probe did not respect its timeout")
|
||||
}
|
||||
mu.Lock()
|
||||
for _, c := range conns {
|
||||
_ = c.Close()
|
||||
}
|
||||
mu.Unlock()
|
||||
}
|
||||
|
||||
func TestClosedPortFailsOpen(t *testing.T) {
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
addr := ln.Addr().String()
|
||||
_ = ln.Close()
|
||||
if testCache().SpeaksHTTP(context.Background(), addr, false) {
|
||||
t.Fatal("a closed port must not be reported as HTTP")
|
||||
}
|
||||
}
|
||||
|
||||
// The network policy is checked before any dial.
|
||||
func TestPolicyDenialFailsOpenWithoutDialing(t *testing.T) {
|
||||
c := testCache()
|
||||
c.allowed = func(string) bool { return false }
|
||||
dialed := false
|
||||
c.dial = func(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
dialed = true
|
||||
return nil, errNoDialer
|
||||
}
|
||||
if c.SpeaksHTTP(context.Background(), "127.0.0.1:1", false) {
|
||||
t.Fatal("a denied host must not be reported as HTTP")
|
||||
}
|
||||
if dialed {
|
||||
t.Fatal("a denied host must not be dialed")
|
||||
}
|
||||
}
|
||||
|
||||
// One probe per host:port, however many templates ask concurrently.
|
||||
func TestProbeIsMemoized(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
|
||||
defer srv.Close()
|
||||
c := testCache()
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 50; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if !c.SpeaksHTTP(context.Background(), srv.Listener.Addr().String(), false) {
|
||||
t.Error("expected HTTP")
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
if got := c.Probes(); got != 1 {
|
||||
t.Fatalf("probes=%d want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNilCache(t *testing.T) {
|
||||
var c *Cache
|
||||
if c.SpeaksHTTP(context.Background(), "127.0.0.1:80", false) {
|
||||
t.Fatal("a nil cache must report false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResetForgetsResults(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
|
||||
defer srv.Close()
|
||||
c := testCache()
|
||||
addr := srv.Listener.Addr().String()
|
||||
c.SpeaksHTTP(context.Background(), addr, false)
|
||||
c.Reset()
|
||||
c.SpeaksHTTP(context.Background(), addr, false)
|
||||
if got := c.Probes(); got != 2 {
|
||||
t.Fatalf("after Reset the next call must probe again: probes=%d", got)
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/useragent"
|
||||
@@ -580,8 +581,7 @@ func (r *requestGenerator) fillRequest(req *retryablehttp.Request, values map[st
|
||||
}
|
||||
|
||||
if !r.request.Unsafe {
|
||||
userAgent := useragent.PickRandom()
|
||||
httputil.SetHeader(req, "User-Agent", userAgent.Raw)
|
||||
httputil.SetHeader(req, "User-Agent", r.userAgent())
|
||||
}
|
||||
|
||||
// Only set these headers on non-raw requests
|
||||
@@ -626,3 +626,27 @@ func (r *requestGenerator) renderText(text string, values map[string]interface{}
|
||||
|
||||
return result.Text, nil
|
||||
}
|
||||
|
||||
// StableUserAgent is one Chrome user agent, chosen once per process. A random
|
||||
// user agent per request makes otherwise identical requests differ, so the
|
||||
// scan-scoped response cache can never reuse them. Whenever that cache may be
|
||||
// active we send one constant user agent instead, which also makes scans
|
||||
// reproducible. Anything that seeds the cache sends the same one, so a seeded
|
||||
// response is reachable by the templates that follow.
|
||||
var StableUserAgent = sync.OnceValue(func() string {
|
||||
if agents, err := useragent.PickWithFilters(1, useragent.Chrome); err == nil && len(agents) > 0 {
|
||||
return agents[0].Raw
|
||||
}
|
||||
return useragent.PickRandom().Raw
|
||||
})
|
||||
|
||||
// userAgent returns the user agent for one request.
|
||||
func (r *requestGenerator) userAgent() string {
|
||||
if r.request == nil || r.request.options == nil || r.request.options.Options == nil {
|
||||
return useragent.PickRandom().Raw
|
||||
}
|
||||
if r.request.options.Options.UsesResponseCache() {
|
||||
return StableUserAgent()
|
||||
}
|
||||
return useragent.PickRandom().Raw
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hostlimit"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
|
||||
)
|
||||
|
||||
// admitToEndpoint waits for room on the request's endpoint when adaptive
|
||||
// concurrency is on. Race requests are exempt: they must leave together.
|
||||
//
|
||||
// The wait uses scanCtx, not the request's own context: the request deadline
|
||||
// (the HTTP timeout) must not run while the request is still queued, or a
|
||||
// queued request could time out before it is sent and be counted as a host
|
||||
// error. Time spent waiting is added back to the request deadline. The returned
|
||||
// release function undoes that and must be called once the request is done.
|
||||
func (request *Request) admitToEndpoint(scanCtx context.Context, req *generatedRequest) (*hostlimit.Ticket, func(), error) {
|
||||
noop := func() {}
|
||||
if request.options == nil || request.options.HostLimit == nil || req == nil || req.request == nil || req.request.Request == nil {
|
||||
return nil, noop, nil
|
||||
}
|
||||
if req.original != nil && req.original.Race {
|
||||
return nil, noop, nil
|
||||
}
|
||||
start := time.Now()
|
||||
ticket, err := request.options.HostLimit.Acquire(scanCtx, endpointKey(req.request.Request.URL))
|
||||
if err != nil {
|
||||
return nil, noop, err
|
||||
}
|
||||
waited := time.Since(start)
|
||||
reqCtx := req.request.Context()
|
||||
deadline, ok := reqCtx.Deadline()
|
||||
if !ok || waited <= 0 {
|
||||
return ticket, noop, nil
|
||||
}
|
||||
// Keep the request's values (custom IP, timeout markers) but move its
|
||||
// deadline back by the time spent queued. The scan's cancellation still
|
||||
// stops the request.
|
||||
extended, cancel := context.WithDeadlineCause(context.WithoutCancel(reqCtx), deadline.Add(waited), ErrHttpEngineRequestDeadline)
|
||||
stop := context.AfterFunc(scanCtx, cancel)
|
||||
req.request.Request = req.request.Request.WithContext(extended)
|
||||
return ticket, func() { stop(); cancel() }, nil
|
||||
}
|
||||
|
||||
// releaseEndpoint returns the slot and tells the limiter how the request went.
|
||||
func (request *Request) releaseEndpoint(ticket *hostlimit.Ticket, req *generatedRequest, resp *http.Response, err error) {
|
||||
if ticket == nil {
|
||||
return
|
||||
}
|
||||
custom := request.hasCustomTimeout(req)
|
||||
o := hostlimit.Outcome{Latency: ticket.Since()}
|
||||
switch {
|
||||
case err != nil:
|
||||
// A timeout the template asked for is not the endpoint struggling.
|
||||
o.Overload = !custom && hostlimit.IsOverload(err)
|
||||
case resp != nil && (resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode == http.StatusServiceUnavailable):
|
||||
o.Overload = true
|
||||
default:
|
||||
o.Sample = !custom
|
||||
}
|
||||
ticket.Release(o)
|
||||
}
|
||||
|
||||
// hasCustomTimeout reports whether the template set its own timeout, which
|
||||
// means its latency is slow by design and says nothing about the endpoint.
|
||||
func (request *Request) hasCustomTimeout(req *generatedRequest) bool {
|
||||
if request.connConfiguration != nil && (request.connConfiguration.NoTimeout || request.connConfiguration.Connection != nil && request.connConfiguration.Connection.CustomMaxTimeout > 0) {
|
||||
return true
|
||||
}
|
||||
if req != nil && req.request != nil {
|
||||
if _, ok := req.request.Context().Value(httpclientpool.WithCustomTimeout{}).(httpclientpool.WithCustomTimeout); ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// endpointKey is host:port with the scheme's default port filled in.
|
||||
func endpointKey(u *url.URL) string {
|
||||
if u == nil {
|
||||
return ""
|
||||
}
|
||||
host, port := u.Hostname(), u.Port()
|
||||
if port == "" {
|
||||
switch strings.ToLower(u.Scheme) {
|
||||
case "https", "wss":
|
||||
port = "443"
|
||||
default:
|
||||
port = "80"
|
||||
}
|
||||
}
|
||||
return net.JoinHostPort(host, port)
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"context"
|
||||
nethttp "net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hostlimit"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
)
|
||||
|
||||
// Time spent queued for an endpoint slot is added back to the request deadline,
|
||||
// so a queued request cannot time out before it is sent.
|
||||
func TestAdmitExtendsDeadlineByQueueTime(t *testing.T) {
|
||||
limit := hostlimit.New(hostlimit.Config{Initial: 1, Min: 1, Max: 1})
|
||||
r := &Request{options: &protocols.ExecutorOptions{Options: &types.Options{}, HostLimit: limit}}
|
||||
|
||||
held, _ := limit.Acquire(context.Background(), "example.com:80")
|
||||
go func() { time.Sleep(150 * time.Millisecond); held.Release(hostlimit.Outcome{}) }()
|
||||
|
||||
reqCtx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
||||
defer cancel()
|
||||
rr, err := retryablehttp.NewRequestWithContext(reqCtx, nethttp.MethodGet, "http://example.com/", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gen := &generatedRequest{request: rr, original: &Request{}}
|
||||
before, _ := rr.Context().Deadline()
|
||||
|
||||
ticket, release, err := r.admitToEndpoint(context.Background(), gen)
|
||||
if err != nil {
|
||||
t.Fatalf("admission must not fail on the request deadline: %v", err)
|
||||
}
|
||||
defer release()
|
||||
defer ticket.Release(hostlimit.Outcome{})
|
||||
|
||||
after, ok := gen.request.Context().Deadline()
|
||||
if !ok || !after.After(before.Add(100*time.Millisecond)) {
|
||||
t.Fatalf("deadline not extended by the queue time: before=%v after=%v", before, after)
|
||||
}
|
||||
if gen.request.Context().Err() != nil {
|
||||
t.Fatal("the request context must still be live after admission")
|
||||
}
|
||||
}
|
||||
|
||||
// Without adaptive concurrency admission is a no-op.
|
||||
func TestAdmitWithoutLimiter(t *testing.T) {
|
||||
r := &Request{options: &protocols.ExecutorOptions{Options: &types.Options{}}}
|
||||
rr, _ := retryablehttp.NewRequest(nethttp.MethodGet, "http://example.com/", nil)
|
||||
ticket, release, err := r.admitToEndpoint(context.Background(), &generatedRequest{request: rr})
|
||||
if err != nil || ticket != nil {
|
||||
t.Fatalf("expected a no-op, got ticket=%v err=%v", ticket, err)
|
||||
}
|
||||
release()
|
||||
}
|
||||
@@ -0,0 +1,503 @@
|
||||
// Package httprespcache is a scan-scoped in-memory cache for safe HTTP GET/HEAD
|
||||
// responses. It lets tech fingerprinting and templates share the same RTT.
|
||||
package httprespcache
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/cookiejar"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/cespare/xxhash"
|
||||
)
|
||||
|
||||
// representationHeaderNames are folded into the cache key so two requests
|
||||
// that differ only in content-negotiation / UA identity cannot share a body.
|
||||
var representationHeaderNames = []string{
|
||||
"accept",
|
||||
"accept-encoding",
|
||||
"accept-language",
|
||||
"user-agent",
|
||||
}
|
||||
|
||||
const (
|
||||
// Entry cap sized above a typical per-host unique GET set; the byte cap below
|
||||
// is the real memory bound, so a larger entry cap cannot grow memory.
|
||||
defaultMaxEntries = 16384
|
||||
defaultMaxBytes = 64 << 20 // 64 MiB of retained responses (key, headers and body)
|
||||
// defaultMaxSeen bounds the (owner, key) record: about 16 bytes per pair.
|
||||
defaultMaxSeen = 1 << 20
|
||||
)
|
||||
|
||||
// Cache stores response snapshots keyed by METHOD + URL + representation headers.
|
||||
type Cache struct {
|
||||
mu sync.RWMutex
|
||||
entries map[string]*Entry
|
||||
disabled atomic.Bool
|
||||
hits atomic.Uint64
|
||||
misses atomic.Uint64
|
||||
stores atomic.Uint64
|
||||
skipped atomic.Uint64
|
||||
maxEntries int
|
||||
maxBytes int64
|
||||
curBytes int64
|
||||
|
||||
// seen records every (owner, key) pair that has been looked up or stored,
|
||||
// as a hash. A pair already seen is never served from the cache: a template
|
||||
// that repeats a request means it (state changes, race checks,
|
||||
// request-and-compare), so the repeat must reach the server even when another
|
||||
// template stored a response in between. When the record is full the cache
|
||||
// stops serving hits instead of forgetting pairs.
|
||||
seen map[uint64]struct{}
|
||||
maxSeen int
|
||||
|
||||
// flights holds the requests now on the wire, by key. Identical requests
|
||||
// that arrive meanwhile wait for the leader instead of sending their own.
|
||||
flights map[string]*flight
|
||||
}
|
||||
|
||||
// flight is one request on the wire that other callers may wait for.
|
||||
type flight struct {
|
||||
owner string
|
||||
done chan struct{}
|
||||
// ent is the entry the leader stored, or nil when it stored nothing
|
||||
// (error, uncacheable response, size limit). Set before done is closed.
|
||||
ent *Entry
|
||||
}
|
||||
|
||||
// Entry is a reusable response snapshot.
|
||||
type Entry struct {
|
||||
// Owner identifies who stored the entry (a template ID). It is never served
|
||||
// back to the same owner: a template that repeats its own request means it
|
||||
// (state changes, race checks, request-and-compare), so that repeat must
|
||||
// reach the server.
|
||||
Owner string
|
||||
StatusCode int
|
||||
Status string
|
||||
Proto string
|
||||
ProtoMajor int
|
||||
ProtoMinor int
|
||||
Header http.Header
|
||||
Body []byte
|
||||
ContentLength int64
|
||||
// size is what the entry is charged against maxBytes: key, headers and body.
|
||||
size int64
|
||||
}
|
||||
|
||||
// New returns an empty cache with default memory bounds.
|
||||
func New() *Cache {
|
||||
return &Cache{
|
||||
entries: make(map[string]*Entry),
|
||||
maxEntries: defaultMaxEntries,
|
||||
maxBytes: defaultMaxBytes,
|
||||
seen: make(map[uint64]struct{}),
|
||||
maxSeen: defaultMaxSeen,
|
||||
flights: make(map[string]*flight),
|
||||
}
|
||||
}
|
||||
|
||||
// Disable stops all Get/Set operations (used when -tf is set but the planner
|
||||
// skips tech filtering, so default scan semantics stay unchanged).
|
||||
func (c *Cache) Disable() {
|
||||
if c != nil {
|
||||
c.disabled.Store(true)
|
||||
}
|
||||
}
|
||||
|
||||
// Enabled reports whether the cache is accepting Get/Set.
|
||||
func (c *Cache) Enabled() bool {
|
||||
return c != nil && !c.disabled.Load()
|
||||
}
|
||||
|
||||
// Key builds a cache key for a method and absolute URL (no header identity).
|
||||
func Key(method, rawURL string) string {
|
||||
method = strings.ToUpper(strings.TrimSpace(method))
|
||||
rawURL = strings.TrimSpace(rawURL)
|
||||
return method + " " + rawURL
|
||||
}
|
||||
|
||||
// KeyFromRequest builds a key from an http.Request, including representation
|
||||
// headers (UA / Accept*) so distinct content-negotiation contexts do not collide.
|
||||
func KeyFromRequest(req *http.Request) string {
|
||||
if req == nil || req.URL == nil {
|
||||
return ""
|
||||
}
|
||||
base := Key(req.Method, req.URL.String())
|
||||
parts := make([]string, 0, len(representationHeaderNames))
|
||||
for _, name := range representationHeaderNames {
|
||||
vals := req.Header.Values(name)
|
||||
if len(vals) == 0 {
|
||||
continue
|
||||
}
|
||||
parts = append(parts, name+":"+strings.Join(vals, ","))
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return base
|
||||
}
|
||||
sort.Strings(parts)
|
||||
return base + " |" + strings.Join(parts, "|")
|
||||
}
|
||||
|
||||
// CacheableRequest reports whether the request is safe to cache/serve.
|
||||
// Requests with auth, cookies, Host override, Cache-Control/Pragma, or any
|
||||
// non-allowlisted header are excluded so they cannot reuse another context.
|
||||
func CacheableRequest(req *http.Request) bool {
|
||||
if req == nil || req.URL == nil {
|
||||
return false
|
||||
}
|
||||
method := strings.ToUpper(req.Method)
|
||||
if method != http.MethodGet && method != http.MethodHead {
|
||||
return false
|
||||
}
|
||||
// A GET/HEAD carrying a real payload is excluded: the body is part of the
|
||||
// request identity but not of the cache key. An empty body is not.
|
||||
// Every GET/HEAD built here goes through retryablehttp, which computes
|
||||
// ContentLength from the buffered body, so ContentLength == 0 proves the
|
||||
// body carries no bytes even when the reader itself is non-nil.
|
||||
if req.Body != nil && req.Body != http.NoBody && req.ContentLength != 0 {
|
||||
return false
|
||||
}
|
||||
if req.Host != "" && req.URL.Host != "" && !strings.EqualFold(req.Host, req.URL.Host) {
|
||||
return false
|
||||
}
|
||||
for k := range req.Header {
|
||||
switch strings.ToLower(k) {
|
||||
case "user-agent", "accept", "accept-language", "accept-encoding",
|
||||
"connection", "upgrade-insecure-requests":
|
||||
continue
|
||||
case "host":
|
||||
// net/http never writes Header["Host"] on the wire; it sends
|
||||
// req.Host or req.URL.Host. A Host header that repeats the target
|
||||
// host therefore changes nothing. A different one is a deliberate
|
||||
// Host override and must not share a cached response.
|
||||
if !sameHost(req.Header.Get("Host"), req.URL) {
|
||||
return false
|
||||
}
|
||||
case "cache-control", "pragma":
|
||||
// Explicit cache directives: never serve from the scan cache.
|
||||
return false
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// sameHost reports whether host names the same authority as u, ignoring the
|
||||
// default port for the URL scheme.
|
||||
func sameHost(host string, u *url.URL) bool {
|
||||
if host == "" || u == nil {
|
||||
return false
|
||||
}
|
||||
return strings.EqualFold(canonicalHost(host, u.Scheme), canonicalHost(u.Host, u.Scheme))
|
||||
}
|
||||
|
||||
func canonicalHost(host, scheme string) string {
|
||||
host = strings.TrimSpace(host)
|
||||
defaultPort := ""
|
||||
switch strings.ToLower(scheme) {
|
||||
case "http":
|
||||
defaultPort = ":80"
|
||||
case "https":
|
||||
defaultPort = ":443"
|
||||
}
|
||||
if defaultPort != "" {
|
||||
host = strings.TrimSuffix(host, defaultPort)
|
||||
}
|
||||
return host
|
||||
}
|
||||
|
||||
// JarAllows reports whether the cookie jar would attach no cookies to u.
|
||||
// A jar is created for every input, so its mere presence must not disable the
|
||||
// cache. What matters is whether cookies would change the request or response:
|
||||
// if the jar has nothing for this URL, a response captured without cookies is
|
||||
// equivalent to a fresh one.
|
||||
func JarAllows(jar *cookiejar.Jar, u *url.URL) bool {
|
||||
if jar == nil {
|
||||
return true
|
||||
}
|
||||
if u == nil {
|
||||
return false
|
||||
}
|
||||
return len(jar.Cookies(u)) == 0
|
||||
}
|
||||
|
||||
// ResponseSetsCookie reports whether resp carries Set-Cookie. Such responses are
|
||||
// never stored, so serving from cache can never hide a jar update.
|
||||
func ResponseSetsCookie(resp *http.Response) bool {
|
||||
return resp != nil && len(resp.Header.Values("Set-Cookie")) > 0
|
||||
}
|
||||
|
||||
// Get returns a fresh *http.Response (new Body reader) or nil on miss.
|
||||
// req is attached as Response.Request so downstream dump/curl code works.
|
||||
// An entry stored by owner is not served back to owner; see Entry.Owner.
|
||||
func (c *Cache) Get(key, owner string, req *http.Request) *http.Response {
|
||||
if !c.Enabled() || key == "" {
|
||||
return nil
|
||||
}
|
||||
c.mu.Lock()
|
||||
firstLook := c.markSeen(owner, key)
|
||||
ent, ok := c.entries[key]
|
||||
c.mu.Unlock()
|
||||
if !firstLook || !ok || ent == nil || (owner != "" && ent.Owner == owner) {
|
||||
c.misses.Add(1)
|
||||
return nil
|
||||
}
|
||||
c.hits.Add(1)
|
||||
return ent.toResponse(req)
|
||||
}
|
||||
|
||||
// GetOrLead is Get that also shares requests in flight. On a miss the caller
|
||||
// either leads a new flight or, when an identical request from another owner is
|
||||
// already on the wire, waits for it and gets its stored response.
|
||||
//
|
||||
// The returned done func must be called once the caller has stored its
|
||||
// response (or knows it will not). It is never nil and is safe to call more
|
||||
// than once. Waiters get their own response and body reader. When the leader
|
||||
// stores nothing (an error, an uncacheable response, the size limit), waiters
|
||||
// get nil and send their own request, so failures are never shared.
|
||||
// A canceled req context stops the wait.
|
||||
func (c *Cache) GetOrLead(key, owner string, req *http.Request) (*http.Response, func()) {
|
||||
noop := func() {}
|
||||
if !c.Enabled() || key == "" {
|
||||
return nil, noop
|
||||
}
|
||||
c.mu.Lock()
|
||||
firstLook := c.markSeen(owner, key)
|
||||
ent, ok := c.entries[key]
|
||||
if firstLook && ok && ent != nil && (owner == "" || ent.Owner != owner) {
|
||||
c.mu.Unlock()
|
||||
c.hits.Add(1)
|
||||
return ent.toResponse(req), noop
|
||||
}
|
||||
if !firstLook {
|
||||
// A repeat of the owner's own request must reach the server.
|
||||
c.mu.Unlock()
|
||||
c.misses.Add(1)
|
||||
return nil, noop
|
||||
}
|
||||
if c.flights == nil {
|
||||
c.flights = make(map[string]*flight)
|
||||
}
|
||||
f, inFlight := c.flights[key]
|
||||
if !inFlight {
|
||||
f = &flight{owner: owner, done: make(chan struct{})}
|
||||
c.flights[key] = f
|
||||
c.mu.Unlock()
|
||||
c.misses.Add(1)
|
||||
return nil, c.landFunc(key, f)
|
||||
}
|
||||
c.mu.Unlock()
|
||||
if owner != "" && f.owner == owner {
|
||||
// The owner's own response is never served back to it.
|
||||
c.misses.Add(1)
|
||||
return nil, noop
|
||||
}
|
||||
var ctxDone <-chan struct{}
|
||||
if req != nil {
|
||||
ctxDone = req.Context().Done()
|
||||
}
|
||||
select {
|
||||
case <-f.done:
|
||||
case <-ctxDone:
|
||||
c.misses.Add(1)
|
||||
return nil, noop
|
||||
}
|
||||
if f.ent == nil {
|
||||
c.misses.Add(1)
|
||||
return nil, noop
|
||||
}
|
||||
c.hits.Add(1)
|
||||
return f.ent.toResponse(req), noop
|
||||
}
|
||||
|
||||
// landFunc returns the func that ends flight f and wakes its waiters.
|
||||
func (c *Cache) landFunc(key string, f *flight) func() {
|
||||
var once sync.Once
|
||||
return func() {
|
||||
once.Do(func() {
|
||||
c.mu.Lock()
|
||||
if c.flights[key] == f {
|
||||
delete(c.flights, key)
|
||||
}
|
||||
c.mu.Unlock()
|
||||
close(f.done)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Set stores a snapshot for key. body should already be fully read.
|
||||
// Entries that would exceed the scan-wide budget are skipped.
|
||||
func (c *Cache) Set(key, owner string, resp *http.Response, body []byte) {
|
||||
if !c.Enabled() || key == "" || resp == nil {
|
||||
return
|
||||
}
|
||||
bodyLen := int64(len(body))
|
||||
size := int64(len(key)) + headerSize(resp.Header) + bodyLen
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.markSeen(owner, key)
|
||||
|
||||
if old, ok := c.entries[key]; ok {
|
||||
c.curBytes -= old.size
|
||||
delete(c.entries, key)
|
||||
} else if c.maxEntries > 0 && len(c.entries) >= c.maxEntries {
|
||||
c.skipped.Add(1)
|
||||
return
|
||||
}
|
||||
if c.maxBytes > 0 && c.curBytes+size > c.maxBytes {
|
||||
c.skipped.Add(1)
|
||||
return
|
||||
}
|
||||
|
||||
ent := &Entry{
|
||||
Owner: owner,
|
||||
StatusCode: resp.StatusCode,
|
||||
Status: resp.Status,
|
||||
Proto: resp.Proto,
|
||||
ProtoMajor: resp.ProtoMajor,
|
||||
ProtoMinor: resp.ProtoMinor,
|
||||
Header: resp.Header.Clone(),
|
||||
Body: append([]byte(nil), body...),
|
||||
ContentLength: bodyLen,
|
||||
size: size,
|
||||
}
|
||||
c.entries[key] = ent
|
||||
c.curBytes += size
|
||||
c.stores.Add(1)
|
||||
if f, ok := c.flights[key]; ok && f.owner == owner {
|
||||
// Written under c.mu before the leader closes f.done, so waiters
|
||||
// that read it after <-f.done see it.
|
||||
f.ent = ent
|
||||
}
|
||||
}
|
||||
|
||||
// SeedHTTP stores a fingerprint/probe response under the request's full key
|
||||
// (method + URL + representation headers). readLimit is the most the probe
|
||||
// read of the body; see CompleteBody.
|
||||
func (c *Cache) SeedHTTP(req *http.Request, resp *http.Response, body []byte, readLimit int64) {
|
||||
if !CacheableRequest(req) || !CompleteBody(body, readLimit) {
|
||||
return
|
||||
}
|
||||
c.Set(KeyFromRequest(req), "", resp, body)
|
||||
}
|
||||
|
||||
// CompleteBody reports whether body was read to its end under readLimit. A body
|
||||
// that fills the limit may be cut short, and a reader with a larger limit
|
||||
// (another template's max-size) would then match against the cut copy, so such
|
||||
// a body must not be cached. A readLimit of zero or less means no limit.
|
||||
func CompleteBody(body []byte, readLimit int64) bool {
|
||||
return readLimit <= 0 || int64(len(body)) < readLimit
|
||||
}
|
||||
|
||||
// Reset drops every entry and the counters, keeping the cache usable. It lets a
|
||||
// long-lived engine reuse one cache instance while scoping its content to one
|
||||
// scan.
|
||||
func (c *Cache) Reset() {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
c.entries = make(map[string]*Entry)
|
||||
c.curBytes = 0
|
||||
c.seen = make(map[uint64]struct{})
|
||||
c.mu.Unlock()
|
||||
c.hits.Store(0)
|
||||
c.misses.Store(0)
|
||||
c.stores.Store(0)
|
||||
c.skipped.Store(0)
|
||||
}
|
||||
|
||||
// Stats returns hit/miss/store counters.
|
||||
func (c *Cache) Stats() (hits, misses, stores uint64) {
|
||||
if c == nil {
|
||||
return 0, 0, 0
|
||||
}
|
||||
return c.hits.Load(), c.misses.Load(), c.stores.Load()
|
||||
}
|
||||
|
||||
// Len returns the number of cached entries (tests).
|
||||
func (c *Cache) Len() int {
|
||||
if c == nil {
|
||||
return 0
|
||||
}
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
return len(c.entries)
|
||||
}
|
||||
|
||||
func (e *Entry) toResponse(req *http.Request) *http.Response {
|
||||
return &http.Response{
|
||||
Status: e.Status,
|
||||
StatusCode: e.StatusCode,
|
||||
Proto: e.Proto,
|
||||
ProtoMajor: e.ProtoMajor,
|
||||
ProtoMinor: e.ProtoMinor,
|
||||
Header: e.Header.Clone(),
|
||||
Body: io.NopCloser(bytes.NewReader(e.Body)),
|
||||
ContentLength: e.ContentLength,
|
||||
Request: req,
|
||||
}
|
||||
}
|
||||
|
||||
// markSeen records the (owner, key) pair and reports whether this is its first
|
||||
// sighting. It reports false when the record is full, so nothing is served
|
||||
// that could not be remembered. An empty owner is not tracked. Must hold c.mu.
|
||||
func (c *Cache) markSeen(owner, key string) bool {
|
||||
if owner == "" {
|
||||
return true
|
||||
}
|
||||
if c.seen == nil {
|
||||
c.seen = make(map[uint64]struct{})
|
||||
}
|
||||
h := xxhash.Sum64String(owner + "\x00" + key)
|
||||
if _, ok := c.seen[h]; ok {
|
||||
return false
|
||||
}
|
||||
if c.maxSeen > 0 && len(c.seen) >= c.maxSeen {
|
||||
return false
|
||||
}
|
||||
c.seen[h] = struct{}{}
|
||||
return true
|
||||
}
|
||||
|
||||
func headerSize(h http.Header) int64 {
|
||||
var n int64
|
||||
for k, vs := range h {
|
||||
for _, v := range vs {
|
||||
n += int64(len(k) + len(v) + 4) // "k: v\r\n"
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// ReplayableEncoding reports whether a response can be replayed from its
|
||||
// decoded body with identical results. The response chain decodes the body by
|
||||
// its Content-Encoding header, and a cache hit keeps that header (templates may
|
||||
// match on it), so a hit is decoded again. That is harmless only when the
|
||||
// second decode fails at once and falls back to the same bytes: no encoding, or
|
||||
// gzip whose decoded body does not itself start with the gzip magic bytes.
|
||||
// Other encodings (deflate, br, zstd) can fail mid-read or decode twice, so
|
||||
// they are not cached.
|
||||
func ReplayableEncoding(h http.Header, decodedBody []byte) bool {
|
||||
switch strings.ToLower(strings.TrimSpace(h.Get("Content-Encoding"))) {
|
||||
case "", "identity":
|
||||
return true
|
||||
case "gzip":
|
||||
return len(decodedBody) < 2 || decodedBody[0] != 0x1f || decodedBody[1] != 0x8b
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// StoredStatus reports whether a response status may be cached. Transient
|
||||
// failures (408, 429, 5xx) are not: a later request may find the server
|
||||
// recovered, and serving the failure would hide that.
|
||||
func StoredStatus(code int) bool {
|
||||
return code != http.StatusRequestTimeout && code != http.StatusTooManyRequests && code < http.StatusInternalServerError
|
||||
}
|
||||
@@ -0,0 +1,363 @@
|
||||
package httprespcache
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCacheRoundTrip(t *testing.T) {
|
||||
c := New()
|
||||
req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
|
||||
resp := &http.Response{
|
||||
Status: "200 OK",
|
||||
StatusCode: 200,
|
||||
Proto: "HTTP/1.1",
|
||||
ProtoMajor: 1,
|
||||
ProtoMinor: 1,
|
||||
Header: http.Header{"Server": []string{"nginx"}},
|
||||
Body: io.NopCloser(http.NoBody),
|
||||
}
|
||||
c.Set(KeyFromRequest(req), "", resp, []byte("ok"))
|
||||
|
||||
got := c.Get(KeyFromRequest(req), "", req)
|
||||
if got == nil {
|
||||
t.Fatal("expected cache hit")
|
||||
}
|
||||
body, _ := io.ReadAll(got.Body)
|
||||
if string(body) != "ok" {
|
||||
t.Fatalf("body=%q", body)
|
||||
}
|
||||
if got.Header.Get("Server") != "nginx" {
|
||||
t.Fatalf("header=%v", got.Header)
|
||||
}
|
||||
hits, misses, stores := c.Stats()
|
||||
if hits != 1 || misses != 0 || stores != 1 {
|
||||
t.Fatalf("stats hits=%d misses=%d stores=%d", hits, misses, stores)
|
||||
}
|
||||
if c.Get(Key(http.MethodGet, "http://example.com/other"), "", req) != nil {
|
||||
t.Fatal("expected miss")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCacheableRequest(t *testing.T) {
|
||||
get := httptest.NewRequest(http.MethodGet, "http://x/", nil)
|
||||
if !CacheableRequest(get) {
|
||||
t.Fatal("GET should be cacheable")
|
||||
}
|
||||
post := httptest.NewRequest(http.MethodPost, "http://x/", nil)
|
||||
if CacheableRequest(post) {
|
||||
t.Fatal("POST should not be cacheable")
|
||||
}
|
||||
auth := httptest.NewRequest(http.MethodGet, "http://x/", nil)
|
||||
auth.Header.Set("Authorization", "Bearer t")
|
||||
if CacheableRequest(auth) {
|
||||
t.Fatal("Authorization must not be cacheable")
|
||||
}
|
||||
cookie := httptest.NewRequest(http.MethodGet, "http://x/", nil)
|
||||
cookie.Header.Set("Cookie", "a=b")
|
||||
if CacheableRequest(cookie) {
|
||||
t.Fatal("Cookie must not be cacheable")
|
||||
}
|
||||
hostOverride := httptest.NewRequest(http.MethodGet, "http://x/", nil)
|
||||
hostOverride.Host = "other.example"
|
||||
if CacheableRequest(hostOverride) {
|
||||
t.Fatal("Host override must not be cacheable")
|
||||
}
|
||||
cc := httptest.NewRequest(http.MethodGet, "http://x/", nil)
|
||||
cc.Header.Set("Cache-Control", "no-cache")
|
||||
if CacheableRequest(cc) {
|
||||
t.Fatal("Cache-Control must not be cacheable")
|
||||
}
|
||||
pragma := httptest.NewRequest(http.MethodGet, "http://x/", nil)
|
||||
pragma.Header.Set("Pragma", "no-cache")
|
||||
if CacheableRequest(pragma) {
|
||||
t.Fatal("Pragma must not be cacheable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestKeyFromRequestIncludesRepresentationHeaders(t *testing.T) {
|
||||
a := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
|
||||
a.Header.Set("User-Agent", "ua-a")
|
||||
a.Header.Set("Accept-Language", "en")
|
||||
|
||||
b := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
|
||||
b.Header.Set("User-Agent", "ua-a")
|
||||
b.Header.Set("Accept-Language", "fr")
|
||||
|
||||
if KeyFromRequest(a) == KeyFromRequest(b) {
|
||||
t.Fatal("distinct Accept-Language must produce distinct keys")
|
||||
}
|
||||
|
||||
c := New()
|
||||
c.Set(KeyFromRequest(a), "", &http.Response{StatusCode: 200, Header: http.Header{}}, []byte("en-body"))
|
||||
if c.Get(KeyFromRequest(b), "", b) != nil {
|
||||
t.Fatal("must not reuse response across Accept-Language")
|
||||
}
|
||||
if got := c.Get(KeyFromRequest(a), "", a); got == nil {
|
||||
t.Fatal("expected hit for matching headers")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCacheDisabled(t *testing.T) {
|
||||
c := New()
|
||||
req, err := http.NewRequest(http.MethodGet, "http://example.com/", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.Set(KeyFromRequest(req), "", &http.Response{StatusCode: 200, Header: http.Header{}}, []byte("x"))
|
||||
c.Disable()
|
||||
if c.Get(KeyFromRequest(req), "", req) != nil {
|
||||
t.Fatal("disabled cache must miss")
|
||||
}
|
||||
c.Set(KeyFromRequest(req), "", &http.Response{StatusCode: 200, Header: http.Header{}}, []byte("y"))
|
||||
if c.Enabled() {
|
||||
t.Fatal("expected disabled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCacheEntryBudget(t *testing.T) {
|
||||
c := New()
|
||||
c.maxEntries = 2
|
||||
c.maxBytes = 100
|
||||
mk := func(path, body string) {
|
||||
req := httptest.NewRequest(http.MethodGet, "http://example.com"+path, nil)
|
||||
c.Set(KeyFromRequest(req), "", &http.Response{StatusCode: 200, Header: http.Header{}}, []byte(body))
|
||||
}
|
||||
mk("/a", "aa")
|
||||
mk("/b", "bb")
|
||||
mk("/c", "cc") // over entry budget
|
||||
if c.Len() != 2 {
|
||||
t.Fatalf("len=%d want 2", c.Len())
|
||||
}
|
||||
if c.skipped.Load() == 0 {
|
||||
t.Fatal("expected skipped store")
|
||||
}
|
||||
|
||||
c2 := New()
|
||||
c2.maxEntries = 10
|
||||
// The budget covers key, headers and body: room for "/a" with a 5-byte body.
|
||||
c2.maxBytes = int64(len(KeyFromRequest(httptest.NewRequest(http.MethodGet, "http://example.com/a", nil)))) + 5
|
||||
mk2 := func(path, body string) {
|
||||
req := httptest.NewRequest(http.MethodGet, "http://example.com"+path, nil)
|
||||
c2.Set(KeyFromRequest(req), "", &http.Response{StatusCode: 200, Header: http.Header{}}, []byte(body))
|
||||
}
|
||||
mk2("/a", "12345")
|
||||
mk2("/b", "x") // over byte budget
|
||||
if c2.Len() != 1 {
|
||||
t.Fatalf("byte budget len=%d want 1", c2.Len())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCacheableRequestEmptyBody(t *testing.T) {
|
||||
// Raw templates hand over a non-nil but empty reader with ContentLength 0.
|
||||
// That request is bodyless on the wire, so it stays cacheable.
|
||||
empty := httptest.NewRequest(http.MethodGet, "http://x/", strings.NewReader(""))
|
||||
empty.ContentLength = 0
|
||||
if empty.Body == nil {
|
||||
t.Fatal("test needs a non-nil body")
|
||||
}
|
||||
if !CacheableRequest(empty) {
|
||||
t.Fatal("GET with an empty body should be cacheable")
|
||||
}
|
||||
|
||||
// A GET that really carries bytes is excluded: the body is part of the
|
||||
// request identity but is not part of the cache key.
|
||||
withBody := httptest.NewRequest(http.MethodGet, "http://x/", strings.NewReader("payload"))
|
||||
withBody.ContentLength = int64(len("payload"))
|
||||
if CacheableRequest(withBody) {
|
||||
t.Fatal("GET with a payload must not be cacheable")
|
||||
}
|
||||
|
||||
// Unknown length is treated as a payload.
|
||||
unknown := httptest.NewRequest(http.MethodGet, "http://x/", strings.NewReader("payload"))
|
||||
unknown.ContentLength = -1
|
||||
if CacheableRequest(unknown) {
|
||||
t.Fatal("GET with an unknown body length must not be cacheable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCacheableRequestHostHeader(t *testing.T) {
|
||||
// Raw templates repeat the target host in a Host header. net/http drops it
|
||||
// and sends the URL host, so the request is unchanged and stays cacheable.
|
||||
same := httptest.NewRequest(http.MethodGet, "http://x.example/", nil)
|
||||
same.Header.Set("Host", "x.example")
|
||||
if !CacheableRequest(same) {
|
||||
t.Fatal("redundant Host header should stay cacheable")
|
||||
}
|
||||
|
||||
withDefaultPort := httptest.NewRequest(http.MethodGet, "http://x.example/", nil)
|
||||
withDefaultPort.Header.Set("Host", "x.example:80")
|
||||
if !CacheableRequest(withDefaultPort) {
|
||||
t.Fatal("Host header with the default port should stay cacheable")
|
||||
}
|
||||
|
||||
// A different Host is a deliberate override and must not share a response.
|
||||
other := httptest.NewRequest(http.MethodGet, "http://x.example/", nil)
|
||||
other.Header.Set("Host", "evil.example")
|
||||
if CacheableRequest(other) {
|
||||
t.Fatal("Host override must not be cacheable")
|
||||
}
|
||||
|
||||
otherPort := httptest.NewRequest(http.MethodGet, "http://x.example/", nil)
|
||||
otherPort.Header.Set("Host", "x.example:8080")
|
||||
if CacheableRequest(otherPort) {
|
||||
t.Fatal("Host header with a different port must not be cacheable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCacheReset(t *testing.T) {
|
||||
c := New()
|
||||
req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
|
||||
c.Set(KeyFromRequest(req), "", &http.Response{StatusCode: 200, Header: http.Header{}}, []byte("body"))
|
||||
c.Get(KeyFromRequest(req), "", req)
|
||||
c.Reset()
|
||||
if c.Len() != 0 {
|
||||
t.Fatal("Reset must drop entries")
|
||||
}
|
||||
if h, m, s := c.Stats(); h != 0 || m != 0 || s != 0 {
|
||||
t.Fatalf("Reset must clear counters: %d %d %d", h, m, s)
|
||||
}
|
||||
c.Set(KeyFromRequest(req), "", &http.Response{StatusCode: 200, Header: http.Header{}}, []byte("body"))
|
||||
if c.Len() != 1 {
|
||||
t.Fatal("cache must stay usable after Reset")
|
||||
}
|
||||
}
|
||||
|
||||
// A template that repeats its own request means it, so the repeat must reach
|
||||
// the server; other templates still share the response.
|
||||
func TestCacheNotServedBackToOwner(t *testing.T) {
|
||||
c := New()
|
||||
req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
|
||||
key := KeyFromRequest(req)
|
||||
c.Set(key, "template-a", &http.Response{StatusCode: 200, Header: http.Header{}}, []byte("1"))
|
||||
if c.Get(key, "template-a", req) != nil {
|
||||
t.Fatal("an entry must not be served back to the template that stored it")
|
||||
}
|
||||
if c.Get(key, "template-b", req) == nil {
|
||||
t.Fatal("another template must reuse the entry")
|
||||
}
|
||||
}
|
||||
|
||||
func ok200() *http.Response { return &http.Response{StatusCode: 200, Header: http.Header{}} }
|
||||
|
||||
// Each template gets at most one cached answer per request: its repeat reaches
|
||||
// the server even when the entry was stored by someone else.
|
||||
func TestAtMostOneCachedAnswerPerTemplate(t *testing.T) {
|
||||
c := New()
|
||||
req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
|
||||
key := KeyFromRequest(req)
|
||||
c.Set(key, "a", ok200(), []byte("1"))
|
||||
if c.Get(key, "b", req) == nil {
|
||||
t.Fatal("first lookup by another template should hit")
|
||||
}
|
||||
if c.Get(key, "b", req) != nil {
|
||||
t.Fatal("the same template's repeat must reach the server")
|
||||
}
|
||||
}
|
||||
|
||||
// A template that already sent a request (miss) must not later be answered from
|
||||
// an entry another template stored in between.
|
||||
func TestRepeatAfterOthersStoreStillMisses(t *testing.T) {
|
||||
c := New()
|
||||
req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
|
||||
key := KeyFromRequest(req)
|
||||
if c.Get(key, "a", req) != nil {
|
||||
t.Fatal("empty cache must miss")
|
||||
}
|
||||
c.Set(key, "b", ok200(), []byte("1"))
|
||||
if c.Get(key, "a", req) != nil {
|
||||
t.Fatal("a template that already sent this request must not be served")
|
||||
}
|
||||
}
|
||||
|
||||
// When the record of seen pairs is full, nothing is served.
|
||||
func TestFullSeenRecordStopsHits(t *testing.T) {
|
||||
c := New()
|
||||
c.maxSeen = 1
|
||||
req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
|
||||
key := KeyFromRequest(req)
|
||||
c.Set(key, "a", ok200(), []byte("1")) // fills the record
|
||||
if c.Get(key, "b", req) != nil {
|
||||
t.Fatal("a full record must stop hits")
|
||||
}
|
||||
}
|
||||
|
||||
// Headers and key count against the memory budget, not only the body.
|
||||
func TestBudgetCountsHeaders(t *testing.T) {
|
||||
c := New()
|
||||
c.maxBytes = 512
|
||||
req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
|
||||
resp := ok200()
|
||||
resp.Header.Set("X-Big", strings.Repeat("h", 1024))
|
||||
c.Set(KeyFromRequest(req), "a", resp, []byte("tiny"))
|
||||
if c.Len() != 0 {
|
||||
t.Fatal("an entry whose headers exceed the budget must not be stored")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplayableEncoding(t *testing.T) {
|
||||
h := func(enc string) http.Header { x := http.Header{}; x.Set("Content-Encoding", enc); return x }
|
||||
cases := []struct {
|
||||
name string
|
||||
h http.Header
|
||||
body []byte
|
||||
want bool
|
||||
}{
|
||||
{"no encoding", http.Header{}, []byte("x"), true},
|
||||
{"identity", h("identity"), []byte("x"), true},
|
||||
{"gzip, plain decoded body", h("gzip"), []byte("<html>"), true},
|
||||
{"gzip, decoded body is gzip again", h("gzip"), []byte{0x1f, 0x8b, 0x08}, false},
|
||||
{"deflate", h("deflate"), []byte("x"), false},
|
||||
{"brotli", h("br"), []byte("x"), false},
|
||||
{"zstd", h("zstd"), []byte("x"), false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := ReplayableEncoding(tc.h, tc.body); got != tc.want {
|
||||
t.Errorf("%s: got %v want %v", tc.name, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoredStatus(t *testing.T) {
|
||||
for code, want := range map[int]bool{200: true, 301: true, 404: true, 400: true, 408: false, 429: false, 500: false, 503: false} {
|
||||
if got := StoredStatus(code); got != want {
|
||||
t.Errorf("StoredStatus(%d)=%v want %v", code, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompleteBody(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
body int
|
||||
limit int64
|
||||
want bool
|
||||
}{
|
||||
{"shorter than limit", 10, 50, true},
|
||||
{"fills limit, may be cut", 50, 50, false},
|
||||
{"no limit", 50, 0, true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := CompleteBody(make([]byte, tc.body), tc.limit); got != tc.want {
|
||||
t.Errorf("%s: got %v want %v", tc.name, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeedHTTPSkipsCutBody(t *testing.T) {
|
||||
c := New()
|
||||
req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
|
||||
resp := &http.Response{StatusCode: 200, Header: http.Header{}}
|
||||
|
||||
c.SeedHTTP(req, resp, []byte("12345"), 5)
|
||||
if c.Len() != 0 {
|
||||
t.Fatal("a body that fills the probe limit must not be seeded")
|
||||
}
|
||||
c.SeedHTTP(req, resp, []byte("1234"), 5)
|
||||
if c.Len() != 1 {
|
||||
t.Fatal("a complete body must be seeded")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
package httprespcache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func okResponse() *http.Response {
|
||||
return &http.Response{
|
||||
Status: "200 OK",
|
||||
StatusCode: 200,
|
||||
Proto: "HTTP/1.1",
|
||||
ProtoMajor: 1,
|
||||
ProtoMinor: 1,
|
||||
Header: http.Header{"Server": []string{"nginx"}},
|
||||
Body: io.NopCloser(http.NoBody),
|
||||
}
|
||||
}
|
||||
|
||||
// startWaiters runs n GetOrLead calls from distinct owners and returns their
|
||||
// results once all of them return.
|
||||
func startWaiters(c *Cache, key string, n int, req *http.Request) (results chan *http.Response, returned chan struct{}) {
|
||||
results = make(chan *http.Response, n)
|
||||
returned = make(chan struct{}, n)
|
||||
for i := 0; i < n; i++ {
|
||||
go func(i int) {
|
||||
resp, done := c.GetOrLead(key, "waiter-"+strconv.Itoa(i), req)
|
||||
done()
|
||||
returned <- struct{}{}
|
||||
results <- resp
|
||||
}(i)
|
||||
}
|
||||
return results, returned
|
||||
}
|
||||
|
||||
// Concurrent identical requests send one request: the others wait for the
|
||||
// leader and each gets its own response and body reader.
|
||||
func TestInFlightShared(t *testing.T) {
|
||||
c := New()
|
||||
req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
|
||||
key := KeyFromRequest(req)
|
||||
|
||||
resp, done := c.GetOrLead(key, "leader", req)
|
||||
if resp != nil {
|
||||
t.Fatal("first request must lead, not hit")
|
||||
}
|
||||
const n = 16
|
||||
results, returned := startWaiters(c, key, n, req)
|
||||
|
||||
select {
|
||||
case <-returned:
|
||||
t.Fatal("a waiter returned before the leader finished")
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
|
||||
c.Set(key, "leader", okResponse(), []byte("shared body"))
|
||||
done()
|
||||
done() // safe to call twice
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < n; i++ {
|
||||
got := <-results
|
||||
if got == nil {
|
||||
t.Fatal("waiter got no response")
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(r *http.Response) {
|
||||
defer wg.Done()
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
_ = r.Body.Close()
|
||||
if string(body) != "shared body" {
|
||||
t.Errorf("body=%q", body)
|
||||
}
|
||||
r.Header.Set("Server", "changed")
|
||||
}(got)
|
||||
}
|
||||
wg.Wait()
|
||||
hits, misses, stores := c.Stats()
|
||||
if hits != n || misses != 1 || stores != 1 {
|
||||
t.Fatalf("stats hits=%d misses=%d stores=%d", hits, misses, stores)
|
||||
}
|
||||
again, _ := c.GetOrLead(key, "late", req)
|
||||
if again == nil || again.Header.Get("Server") != "nginx" {
|
||||
t.Fatal("a waiter must not change the stored entry")
|
||||
}
|
||||
}
|
||||
|
||||
// A leader that stores nothing (error, uncacheable response) shares nothing:
|
||||
// waiters get nil and send their own request, and nothing is cached.
|
||||
func TestInFlightErrorNotShared(t *testing.T) {
|
||||
c := New()
|
||||
req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
|
||||
key := KeyFromRequest(req)
|
||||
|
||||
_, done := c.GetOrLead(key, "leader", req)
|
||||
results, _ := startWaiters(c, key, 4, req)
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
done()
|
||||
for i := 0; i < 4; i++ {
|
||||
if <-results != nil {
|
||||
t.Fatal("a failed leader must not share a response")
|
||||
}
|
||||
}
|
||||
if c.Len() != 0 {
|
||||
t.Fatal("nothing must be cached")
|
||||
}
|
||||
if resp, done := c.GetOrLead(key, "later", req); resp != nil {
|
||||
t.Fatal("a later caller must send its own request")
|
||||
} else {
|
||||
done()
|
||||
}
|
||||
}
|
||||
|
||||
// A response over the size limit is not stored, so it is not shared either.
|
||||
func TestInFlightRespectsSizeLimit(t *testing.T) {
|
||||
c := New()
|
||||
c.maxBytes = 8
|
||||
req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
|
||||
key := KeyFromRequest(req)
|
||||
|
||||
_, done := c.GetOrLead(key, "leader", req)
|
||||
results, _ := startWaiters(c, key, 2, req)
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
c.Set(key, "leader", okResponse(), []byte("a body larger than the limit"))
|
||||
done()
|
||||
for i := 0; i < 2; i++ {
|
||||
if <-results != nil {
|
||||
t.Fatal("an entry over the limit must not be shared")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The owner rule holds in flight: a template never waits for its own request.
|
||||
func TestInFlightNotSharedWithOwner(t *testing.T) {
|
||||
c := New()
|
||||
req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
|
||||
key := KeyFromRequest(req)
|
||||
|
||||
_, done := c.GetOrLead(key, "same", req)
|
||||
defer done()
|
||||
returned := make(chan *http.Response, 1)
|
||||
go func() {
|
||||
resp, d := c.GetOrLead(key, "same", req)
|
||||
d()
|
||||
returned <- resp
|
||||
}()
|
||||
select {
|
||||
case resp := <-returned:
|
||||
if resp != nil {
|
||||
t.Fatal("a template must not get its own response")
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("a template must not wait for its own request")
|
||||
}
|
||||
}
|
||||
|
||||
// A canceled request stops waiting.
|
||||
func TestInFlightWaitHonorsContext(t *testing.T) {
|
||||
c := New()
|
||||
req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
|
||||
key := KeyFromRequest(req)
|
||||
|
||||
_, done := c.GetOrLead(key, "leader", req)
|
||||
defer done()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
returned := make(chan *http.Response, 1)
|
||||
go func() {
|
||||
resp, d := c.GetOrLead(key, "waiter", req.WithContext(ctx))
|
||||
d()
|
||||
returned <- resp
|
||||
}()
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
cancel()
|
||||
select {
|
||||
case resp := <-returned:
|
||||
if resp != nil {
|
||||
t.Fatal("a canceled waiter must get nil")
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("a canceled waiter must stop waiting")
|
||||
}
|
||||
}
|
||||
@@ -41,6 +41,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httputils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signer"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signerpool"
|
||||
@@ -1086,6 +1087,8 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
|
||||
// including follow-ups when the initial response came from the cache.
|
||||
executingClient *retryablehttp.Client
|
||||
)
|
||||
// respCacheDone ends this request's response cache flight, if it leads one.
|
||||
respCacheDone := func() {}
|
||||
|
||||
// Dump request for variables checks
|
||||
// For race conditions we can't dump the request body at this point as it's already waiting the open-gate event, already handled with a similar code within the race function
|
||||
@@ -1241,6 +1244,25 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
|
||||
fromCache = false
|
||||
}
|
||||
}
|
||||
// Scan-scoped in-memory cache (tech-filter / response reuse). Prefer this
|
||||
// over a network round-trip for identical safe GET/HEAD requests. An
|
||||
// identical request already on the wire is waited for, not sent again.
|
||||
if resp == nil && request.options.HTTPResponseCache != nil && request.usesResponseCache(generatedRequest) &&
|
||||
generatedRequest.request != nil && generatedRequest.request.Request != nil &&
|
||||
httprespcache.CacheableRequest(generatedRequest.request.Request) &&
|
||||
httprespcache.JarAllows(input.CookieJar, generatedRequest.request.Request.URL) {
|
||||
if key := respCacheKey(input, generatedRequest.request.Request); key != "" {
|
||||
cached, done := request.options.HTTPResponseCache.GetOrLead(key, request.options.TemplateID, generatedRequest.request.Request)
|
||||
// Every return path ends the flight, so waiters never hang.
|
||||
defer done()
|
||||
respCacheDone = done
|
||||
if cached != nil {
|
||||
resp = cached
|
||||
fromCache = true
|
||||
err = nil
|
||||
}
|
||||
}
|
||||
}
|
||||
if resp == nil || (request.Analyzer != nil && cookieFuzzing) {
|
||||
connConfig := request.connConfiguration
|
||||
if input.CookieJar != nil && !request.DisableCookie {
|
||||
@@ -1301,6 +1323,11 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
|
||||
}
|
||||
}
|
||||
|
||||
ticket, releaseDeadline, admitErr := request.admitToEndpoint(input.Context(), generatedRequest)
|
||||
if admitErr != nil {
|
||||
return admitErr
|
||||
}
|
||||
defer releaseDeadline()
|
||||
resp, err = executingClient.Do(generatedRequest.request)
|
||||
sent = true
|
||||
|
||||
@@ -1347,6 +1374,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
|
||||
rewindRequestBody(generatedRequest.request)
|
||||
resp, err = executingClient.Do(generatedRequest.request)
|
||||
}
|
||||
request.releaseEndpoint(ticket, generatedRequest, resp, err)
|
||||
}
|
||||
}
|
||||
// A project-file hit never sets sent, so a stored response cannot move the delay.
|
||||
@@ -1490,6 +1518,21 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
|
||||
errx = errors.Wrap(err, "could not store in project file")
|
||||
}
|
||||
}
|
||||
if request.options.HTTPResponseCache != nil && !fromCache && request.usesResponseCache(generatedRequest) &&
|
||||
generatedRequest.request != nil && generatedRequest.request.Request != nil &&
|
||||
httprespcache.CacheableRequest(generatedRequest.request.Request) &&
|
||||
httprespcache.JarAllows(input.CookieJar, generatedRequest.request.Request.URL) &&
|
||||
!httprespcache.ResponseSetsCookie(resp) &&
|
||||
httprespcache.StoredStatus(resp.StatusCode) &&
|
||||
httprespcache.CompleteBody(respChain.BodyBytes(), int64(maxBodylimit)) &&
|
||||
httprespcache.ReplayableEncoding(resp.Header, respChain.BodyBytes()) &&
|
||||
!isHTTPToHTTPSMismatch(resp.StatusCode, respChain.BodyString()) {
|
||||
if key := respCacheKey(input, generatedRequest.request.Request); key != "" {
|
||||
request.options.HTTPResponseCache.Set(key, request.options.TemplateID, resp, respChain.BodyBytes())
|
||||
}
|
||||
}
|
||||
// Wake requests waiting on this one now, not after matching.
|
||||
respCacheDone()
|
||||
})
|
||||
|
||||
// evaluate responses continuously until first redirect request in reverse order
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache"
|
||||
)
|
||||
|
||||
// followsRedirects reports whether this request's client follows redirects,
|
||||
// resolving the policy the same way httpclientpool does: the template setting,
|
||||
// overridden by the global follow flags, cancelled by -disable-redirects.
|
||||
//
|
||||
// The scan response cache must stay out of such requests. A redirect chain is
|
||||
// several responses, and a cache entry is one: serving it would hide the chain
|
||||
// from matchers that walk it. The cache key also does not carry the redirect
|
||||
// policy, so a followed and an unfollowed request for the same URL would
|
||||
// otherwise share one entry and each see the other's response.
|
||||
func (request *Request) followsRedirects() bool {
|
||||
if request.options == nil || request.options.Options == nil {
|
||||
return true // unknown policy: stay out of the cache
|
||||
}
|
||||
opts := request.options.Options
|
||||
if opts.DisableRedirects {
|
||||
return false
|
||||
}
|
||||
if opts.ShouldFollowHTTPRedirects() {
|
||||
return true
|
||||
}
|
||||
return request.connConfiguration == nil ||
|
||||
request.connConfiguration.RedirectFlow != httpclientpool.DontFollowRedirect
|
||||
}
|
||||
|
||||
// usesResponseCache reports whether this request may be served from, or stored
|
||||
// in, the scan response cache. A template's own disable-http-cache opts out,
|
||||
// race requests must all reach the server, and redirect-following requests
|
||||
// carry a chain the cache cannot represent.
|
||||
func (request *Request) usesResponseCache(req *generatedRequest) bool {
|
||||
if request.DisableHTTPCache || request.followsRedirects() {
|
||||
return false
|
||||
}
|
||||
return req == nil || req.original == nil || !req.original.Race
|
||||
}
|
||||
|
||||
// respCacheKey is the response cache key for a request. Inputs that share a
|
||||
// URL but dial different addresses (-scan-all-ips, -iv) must not share
|
||||
// responses, so the selected IP is part of the key.
|
||||
func respCacheKey(input *contextargs.Context, req *http.Request) string {
|
||||
key := httprespcache.KeyFromRequest(req)
|
||||
if key == "" || input == nil || input.MetaInput == nil || input.MetaInput.CustomIP == "" {
|
||||
return key
|
||||
}
|
||||
return key + " |ip:" + input.MetaInput.CustomIP
|
||||
}
|
||||
|
||||
// isHTTPToHTTPSMismatch reports the 400 a server returns for plain HTTP sent to
|
||||
// an HTTPS port. The scan records a correction for that host:port, so this
|
||||
// response must not be cached: a later hit would skip the correction.
|
||||
func isHTTPToHTTPSMismatch(statusCode int, body string) bool {
|
||||
return statusCode == http.StatusBadRequest && strings.Contains(body, httpToHTTPSMismatchSignature)
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
)
|
||||
|
||||
// The scan response cache must never serve or store a request that follows
|
||||
// redirects: a chain is several responses, a cache entry is one, and the key
|
||||
// does not carry the redirect policy.
|
||||
func TestFollowsRedirects(t *testing.T) {
|
||||
mk := func(flow httpclientpool.RedirectFlow, opts *types.Options) *Request {
|
||||
return &Request{
|
||||
options: &protocols.ExecutorOptions{Options: opts},
|
||||
connConfiguration: &httpclientpool.Configuration{RedirectFlow: flow},
|
||||
}
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
req *Request
|
||||
want bool
|
||||
}{
|
||||
{"template does not follow", mk(httpclientpool.DontFollowRedirect, &types.Options{}), false},
|
||||
{"template follows all", mk(httpclientpool.FollowAllRedirect, &types.Options{}), true},
|
||||
{"template follows same host", mk(httpclientpool.FollowSameHostRedirect, &types.Options{}), true},
|
||||
{"template follows same scheme", mk(httpclientpool.FollowSameSchemeRedirect, &types.Options{}), true},
|
||||
{"global -fr overrides template", mk(httpclientpool.DontFollowRedirect, &types.Options{FollowRedirects: true}), true},
|
||||
{"global -fhr overrides template", mk(httpclientpool.DontFollowRedirect, &types.Options{FollowHostRedirects: true}), true},
|
||||
{"-disable-redirects wins", mk(httpclientpool.FollowAllRedirect, &types.Options{FollowRedirects: true, DisableRedirects: true}), false},
|
||||
{"unknown options stay out of cache", &Request{}, true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := c.req.followsRedirects(); got != c.want {
|
||||
t.Errorf("%s: followsRedirects()=%v want %v", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUsesResponseCache(t *testing.T) {
|
||||
plain := &Request{options: &protocols.ExecutorOptions{Options: &types.Options{}}, connConfiguration: &httpclientpool.Configuration{}}
|
||||
if !plain.usesResponseCache(&generatedRequest{original: &Request{}}) {
|
||||
t.Fatal("an ordinary request may use the cache")
|
||||
}
|
||||
optOut := &Request{DisableHTTPCache: true, options: plain.options, connConfiguration: plain.connConfiguration}
|
||||
if optOut.usesResponseCache(&generatedRequest{original: optOut}) {
|
||||
t.Fatal("disable-http-cache must keep the request out of the cache")
|
||||
}
|
||||
if plain.usesResponseCache(&generatedRequest{original: &Request{Race: true}}) {
|
||||
t.Fatal("race requests must all reach the server")
|
||||
}
|
||||
redirect := &Request{options: plain.options, connConfiguration: &httpclientpool.Configuration{RedirectFlow: httpclientpool.FollowAllRedirect}}
|
||||
if redirect.usesResponseCache(&generatedRequest{original: redirect}) {
|
||||
t.Fatal("redirect-following requests must stay out of the cache")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRespCacheKeyIncludesCustomIP(t *testing.T) {
|
||||
req, _ := http.NewRequest(http.MethodGet, "http://example.com/", nil)
|
||||
plain := contextargs.NewWithInput(context.Background(), "http://example.com/")
|
||||
a := contextargs.NewWithInput(context.Background(), "http://example.com/")
|
||||
a.MetaInput.CustomIP = "10.0.0.1"
|
||||
b := contextargs.NewWithInput(context.Background(), "http://example.com/")
|
||||
b.MetaInput.CustomIP = "10.0.0.2"
|
||||
if respCacheKey(a, req) == respCacheKey(b, req) {
|
||||
t.Fatal("inputs that dial different IPs must not share a cache key")
|
||||
}
|
||||
if respCacheKey(plain, req) == respCacheKey(a, req) {
|
||||
t.Fatal("an IP-pinned input must not share the plain input's key")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsHTTPToHTTPSMismatch(t *testing.T) {
|
||||
if !isHTTPToHTTPSMismatch(400, "<h1>The plain HTTP request was sent to HTTPS port</h1>") {
|
||||
t.Fatal("expected the mismatch to be recognised")
|
||||
}
|
||||
if isHTTPToHTTPSMismatch(400, "bad request") || isHTTPToHTTPSMismatch(200, "The plain HTTP request was sent to HTTPS port") {
|
||||
t.Fatal("only the 400 with the mismatch message counts")
|
||||
}
|
||||
}
|
||||
|
||||
// A body read up to a template's max-size may be cut short. The cache must not
|
||||
// keep it, or a later template with a larger limit matches against a cut body.
|
||||
func TestResponseCacheSkipsTruncatedBody(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
_, _ = w.Write([]byte(strings.Repeat("A", 150) + "TAIL"))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
// Other tests change the shared default options, so work on a copy.
|
||||
opts := testutils.DefaultOptions.Copy()
|
||||
// -response-cache pins one user agent, so identical requests share a key.
|
||||
opts.ResponseCache = true
|
||||
opts.FollowRedirects = false
|
||||
opts.FollowHostRedirects = false
|
||||
opts.ResponseReadSize = 0
|
||||
// A custom header makes a request uncacheable.
|
||||
opts.CustomHeaders = nil
|
||||
testutils.Init(opts)
|
||||
cache := httprespcache.New()
|
||||
|
||||
execute := func(id string, maxSize int) string {
|
||||
request := &Request{ID: id, MaxSize: maxSize, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, Path: []string{"{{BaseURL}}/big"}}
|
||||
executerOptions := testutils.NewMockExecuterOptions(opts, &testutils.TemplateInfo{
|
||||
ID: id,
|
||||
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
|
||||
})
|
||||
executerOptions.HTTPResponseCache = cache
|
||||
require.NoError(t, request.Compile(executerOptions))
|
||||
|
||||
var body string
|
||||
err := request.ExecuteWithResults(contextargs.NewWithInput(context.Background(), ts.URL), make(output.InternalEvent), make(output.InternalEvent), func(event *output.InternalWrappedEvent) {
|
||||
if event != nil && event.InternalEvent != nil {
|
||||
body, _ = event.InternalEvent["body"].(string)
|
||||
}
|
||||
})
|
||||
require.NoError(t, err)
|
||||
return body
|
||||
}
|
||||
|
||||
require.Len(t, execute("small", 50), 50)
|
||||
require.Equal(t, 0, cache.Len(), "truncated body must not be cached")
|
||||
|
||||
require.True(t, strings.HasSuffix(execute("full", 0), "TAIL"))
|
||||
require.Equal(t, int32(2), hits.Load())
|
||||
require.Equal(t, 1, cache.Len())
|
||||
|
||||
require.Len(t, execute("small-again", 50), 50, "a cached full body is still cut to the reader's limit")
|
||||
require.Equal(t, int32(2), hits.Load())
|
||||
}
|
||||
|
||||
// Identical cacheable requests from different templates that run at the same
|
||||
// time send one request on the wire. Without the cache each one is sent.
|
||||
func TestResponseCacheSharesInFlight(t *testing.T) {
|
||||
var hits atomic.Int32
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
_, _ = w.Write([]byte("shared"))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
opts := testutils.DefaultOptions.Copy()
|
||||
opts.ResponseCache = true
|
||||
opts.FollowRedirects = false
|
||||
opts.FollowHostRedirects = false
|
||||
opts.CustomHeaders = nil
|
||||
testutils.Init(opts)
|
||||
|
||||
run := func(cache *httprespcache.Cache) []string {
|
||||
hits.Store(0)
|
||||
const n = 4
|
||||
bodies := make([]string, n)
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < n; i++ {
|
||||
id := "tpl-" + strconv.Itoa(i)
|
||||
request := &Request{ID: id, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}, Path: []string{"{{BaseURL}}/slow"}}
|
||||
executerOptions := testutils.NewMockExecuterOptions(opts, &testutils.TemplateInfo{
|
||||
ID: id,
|
||||
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
|
||||
})
|
||||
executerOptions.HTTPResponseCache = cache
|
||||
require.NoError(t, request.Compile(executerOptions))
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
err := request.ExecuteWithResults(contextargs.NewWithInput(context.Background(), ts.URL), make(output.InternalEvent), make(output.InternalEvent), func(event *output.InternalWrappedEvent) {
|
||||
if event != nil && event.InternalEvent != nil {
|
||||
bodies[i], _ = event.InternalEvent["body"].(string)
|
||||
}
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
return bodies
|
||||
}
|
||||
|
||||
for _, body := range run(httprespcache.New()) {
|
||||
require.Equal(t, "shared", body)
|
||||
}
|
||||
require.Equal(t, int32(1), hits.Load(), "concurrent identical requests must share one request")
|
||||
|
||||
for _, body := range run(nil) {
|
||||
require.Equal(t, "shared", body)
|
||||
}
|
||||
require.Equal(t, int32(4), hits.Load(), "without the cache every request is sent")
|
||||
}
|
||||
@@ -27,6 +27,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hostlimit"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
|
||||
@@ -98,11 +99,15 @@ type Request struct {
|
||||
preConditionCompiled *goja.Program `yaml:"-" json:"-"`
|
||||
|
||||
scriptCompiled *goja.Program `yaml:"-" json:"-"`
|
||||
// nonHTTPOnly is set when the scripts can only speak dedicated non-HTTP
|
||||
// protocols; see speaksOnlyNonHTTP.
|
||||
nonHTTPOnly bool `yaml:"-" json:"-"`
|
||||
}
|
||||
|
||||
// Compile compiles the request generators preparing any requests possible.
|
||||
func (request *Request) Compile(options *protocols.ExecutorOptions) error {
|
||||
request.options = options
|
||||
request.nonHTTPOnly = speaksOnlyNonHTTP(request.Code, request.PreCondition)
|
||||
|
||||
var err error
|
||||
if len(request.Payloads) > 0 {
|
||||
@@ -335,6 +340,13 @@ func (request *Request) executeWithResults(port string, target *contextargs.Cont
|
||||
request.options.Progress.IncrementFailedRequestsBy(1)
|
||||
return err
|
||||
}
|
||||
if request.skipOnHTTPPort(target.Context(), port, input, hostPort) {
|
||||
// Requests() already counted the requests this port now skips
|
||||
request.options.Progress.SetRequests(uint64(request.requestsPerPort()))
|
||||
request.options.Output.Request(request.options.TemplateID, hostPort, request.Type().String(), errkit.New(skipReason(port)))
|
||||
return nil
|
||||
}
|
||||
|
||||
hostname, port, _ := net.SplitHostPort(hostPort)
|
||||
if hostname == "" {
|
||||
hostname = hostPort
|
||||
@@ -675,6 +687,13 @@ func (request *Request) executeRequestWithPayloads(
|
||||
return err
|
||||
}
|
||||
|
||||
// The script holds one stream slot while it runs. Its errors cannot be
|
||||
// classified, so it only counts against the endpoint and never cuts the
|
||||
// limit: this can lower concurrency, never raise it.
|
||||
ticket, err := request.options.StreamLimit.Acquire(input.Context(), hostPort)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx := input.Context()
|
||||
if request.options.HostBackoff != nil {
|
||||
// the script's own dials are the only signal: report them as they happen
|
||||
@@ -689,6 +708,7 @@ func (request *Request) executeRequestWithPayloads(
|
||||
Source: &request.Code,
|
||||
},
|
||||
)
|
||||
ticket.Release(hostlimit.Outcome{})
|
||||
if err != nil {
|
||||
// shouldn't fail even if it returned error instead create a failure event
|
||||
results = compiler.ExecuteResult{"success": false, "error": err.Error()}
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
package javascript
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
)
|
||||
|
||||
// nonHTTPModules are nuclei JS libraries that each speak one dedicated wire
|
||||
// protocol. None of them can produce a meaningful exchange with an HTTP server.
|
||||
var nonHTTPModules = map[string]struct{}{
|
||||
"smb": {}, "smbsession": {}, "ssh": {}, "mysql": {}, "mssql": {},
|
||||
"postgres": {}, "oracle": {}, "redis": {}, "ldap": {}, "vnc": {},
|
||||
"rdp": {}, "rsync": {}, "smtp": {}, "pop3": {}, "telnet": {},
|
||||
"ikev2": {}, "kerberos": {}, "krbforge": {}, "krbroast": {},
|
||||
"dcerpc": {}, "dcom": {}, "scmr": {}, "secretsdump": {}, "tsch": {},
|
||||
"wmi": {}, "goexec": {},
|
||||
}
|
||||
|
||||
// neutralModules open no connection of their own, so they do not change what a
|
||||
// script can speak.
|
||||
var neutralModules = map[string]struct{}{
|
||||
"bytes": {}, "structs": {}, "fs": {}, "goconsole": {},
|
||||
}
|
||||
|
||||
var (
|
||||
requireCall = regexp.MustCompile(`require\s*\(`)
|
||||
requireLiteral = regexp.MustCompile(`require\s*\(\s*["']nuclei/([a-z0-9_]+)["']\s*\)`)
|
||||
)
|
||||
|
||||
// speaksOnlyNonHTTP reports whether the scripts can only talk dedicated
|
||||
// non-HTTP protocols. Anything uncertain returns false, so the template runs:
|
||||
// raw sockets (net), HTTP-capable libraries (http, websocket, grpc), unknown
|
||||
// libraries, no library at all, or a require() whose module is not a literal.
|
||||
func speaksOnlyNonHTTP(scripts ...string) bool {
|
||||
calls, protocol := 0, 0
|
||||
for _, script := range scripts {
|
||||
calls += len(requireCall.FindAllStringIndex(script, -1))
|
||||
for _, m := range requireLiteral.FindAllStringSubmatch(script, -1) {
|
||||
calls--
|
||||
name := m[1]
|
||||
if _, ok := neutralModules[name]; ok {
|
||||
continue
|
||||
}
|
||||
if _, ok := nonHTTPModules[name]; !ok {
|
||||
return false
|
||||
}
|
||||
protocol++
|
||||
}
|
||||
}
|
||||
return calls == 0 && protocol > 0
|
||||
}
|
||||
|
||||
// skipOnHTTPPort reports whether this execution should not run: the template
|
||||
// only speaks non-HTTP protocols, its declared port was replaced by the input's
|
||||
// explicit port, and that host:port answered HTTP. In every other case the
|
||||
// template runs.
|
||||
func (request *Request) skipOnHTTPPort(ctx context.Context, declaredPort string, input *contextargs.Context, hostPort string) bool {
|
||||
if !request.nonHTTPOnly || declaredPort == "" || request.options == nil || request.options.HTTPService == nil {
|
||||
return false
|
||||
}
|
||||
_, effectivePort, err := net.SplitHostPort(hostPort)
|
||||
if err != nil || effectivePort == "" || effectivePort == declaredPort {
|
||||
return false
|
||||
}
|
||||
useTLS := strings.HasPrefix(strings.ToLower(input.MetaInput.Input), "https://")
|
||||
if !request.options.HTTPService.SpeaksHTTP(ctx, hostPort, useTLS) {
|
||||
return false
|
||||
}
|
||||
gologger.Verbose().Msgf("[%s] Skipped on %s: the port answers HTTP and the template only speaks non-HTTP protocols (declared port %s)", request.options.TemplateID, hostPort, declaredPort)
|
||||
return true
|
||||
}
|
||||
|
||||
// requestsPerPort is the number of requests one port of this template counts
|
||||
// in the progress total, so a skip can account for all of them.
|
||||
func (request *Request) requestsPerPort() int {
|
||||
n := 1
|
||||
if request.generator != nil {
|
||||
n = request.generator.NewIterator().Total()
|
||||
}
|
||||
if request.PreCondition != "" {
|
||||
n++
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func skipReason(declaredPort string) string {
|
||||
return fmt.Sprintf("skipped: port answers HTTP, template speaks a non-HTTP protocol on port %s", declaredPort)
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package javascript
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestSpeaksOnlyNonHTTP(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
scripts []string
|
||||
want bool
|
||||
}{
|
||||
{"smb only", []string{`var smb = require("nuclei/smb"); smb.SMBClient()`}, true},
|
||||
{"single quotes", []string{`let c = require('nuclei/ssh');`}, true},
|
||||
{"protocol plus neutral helper", []string{`require("nuclei/bytes"); require("nuclei/mysql")`}, true},
|
||||
{"pre-condition counts too", []string{`require("nuclei/redis")`, `isPortOpen(Host, Port)`}, true},
|
||||
{"raw socket can speak HTTP", []string{`require("nuclei/net")`}, false},
|
||||
{"protocol plus raw socket", []string{`require("nuclei/smb"); require("nuclei/net")`}, false},
|
||||
{"http library", []string{`require("nuclei/http")`}, false},
|
||||
{"websocket is an HTTP upgrade", []string{`require("nuclei/websocket")`}, false},
|
||||
{"grpc runs over HTTP/2", []string{`require("nuclei/grpc")`}, false},
|
||||
{"unknown library", []string{`require("nuclei/somethingnew")`}, false},
|
||||
{"only neutral helpers", []string{`require("nuclei/bytes")`}, false},
|
||||
{"no require at all", []string{`isPortOpen(Host, Port)`}, false},
|
||||
{"non-literal module name", []string{`require("nuclei/smb"); var m = "nuclei/net"; require(m)`}, false},
|
||||
{"empty", []string{"", ""}, false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := speaksOnlyNonHTTP(c.scripts...); got != c.want {
|
||||
t.Errorf("%s: got %v want %v", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -26,6 +26,7 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hostlimit"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/render"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/replacer"
|
||||
@@ -347,11 +348,22 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
|
||||
return errors.Wrap(err, "could not take rate limit")
|
||||
}
|
||||
|
||||
// One stream slot per connection, held until the connection closes. Only
|
||||
// a failed dial counts as strain: a banner read that times out on a port
|
||||
// running another service is expected, not overload.
|
||||
ticket, admitErr := request.options.StreamLimit.Acquire(input.Context(), actualAddress)
|
||||
if admitErr != nil {
|
||||
return errors.Wrap(admitErr, "could not admit request")
|
||||
}
|
||||
var dialErr error
|
||||
defer func() { ticket.Release(hostlimit.Outcome{Overload: hostlimit.IsOverload(dialErr)}) }()
|
||||
|
||||
if shouldUseTLS {
|
||||
conn, err = request.dialer.DialTLS(input.Context(), "tcp", actualAddress)
|
||||
} else {
|
||||
conn, err = request.dialer.Dial(input.Context(), "tcp", actualAddress)
|
||||
}
|
||||
dialErr = err
|
||||
// DialTLS fails the handshake after TCP has connected. Backoff follows the
|
||||
// TCP result; the request still returns the handshake error.
|
||||
request.options.HostBackoffObserveDial(actualAddress, tcpDialError(err))
|
||||
|
||||
@@ -34,11 +34,14 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/globalmatchers"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hostbackoff"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hostlimit"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hostratelimit"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/httpservice"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/excludematchers"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/variables"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/scan"
|
||||
templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
|
||||
@@ -118,9 +121,20 @@ type ExecutorOptions struct {
|
||||
InteractshScope *interactsh.RequestScope
|
||||
// HostErrorsCache is an optional cache for handling host errors
|
||||
HostErrorsCache hosterrorscache.CacheInterface
|
||||
// HTTPResponseCache reuses identical safe GET/HEAD responses inside one
|
||||
// scan. Nil disables it.
|
||||
HTTPResponseCache *httprespcache.Cache
|
||||
// HostBackoff paces a host that signals it is being overloaded, so a scan
|
||||
// slows down for it instead of being blocked outright. Nil disables it.
|
||||
HostBackoff *hostbackoff.Governor
|
||||
// HTTPService answers whether an input port speaks HTTP, for service placement.
|
||||
HTTPService *httpservice.Cache
|
||||
// HostLimit adapts in-flight requests per endpoint to what it can take.
|
||||
HostLimit *hostlimit.Limiter
|
||||
// StreamLimit bounds concurrent raw connections (network, JS) per endpoint.
|
||||
// It is separate from HostLimit so idle sockets that wait for a banner
|
||||
// never hold the slots HTTP requests need.
|
||||
StreamLimit *hostlimit.Limiter
|
||||
// Stop execution once first match is found (Assigned while parsing templates)
|
||||
// Note: this is different from Options.StopAtFirstMatch (Assigned from CLI option)
|
||||
StopAtFirstMatch bool
|
||||
@@ -397,8 +411,12 @@ func (e *ExecutorOptions) Copy() *ExecutorOptions {
|
||||
Browser: e.Browser,
|
||||
Interactsh: e.Interactsh,
|
||||
HostErrorsCache: e.HostErrorsCache,
|
||||
HTTPResponseCache: e.HTTPResponseCache,
|
||||
InteractshScope: e.InteractshScope,
|
||||
HostBackoff: e.HostBackoff,
|
||||
HTTPService: e.HTTPService,
|
||||
HostLimit: e.HostLimit,
|
||||
StreamLimit: e.StreamLimit,
|
||||
StopAtFirstMatch: e.StopAtFirstMatch,
|
||||
Variables: e.Variables,
|
||||
Constants: e.Constants,
|
||||
@@ -687,6 +705,41 @@ func (e *ExecutorOptions) EncodeTemplate() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// AttachScanFeatures gives the executor fresh per-scan state for service
|
||||
// placement, adaptive concurrency and the response cache, honouring the
|
||||
// Disable options and the opt-in response cache. Call it once per scan: the response cache in particular must
|
||||
// not outlive the scan, or a later scan could be served an earlier response.
|
||||
func (e *ExecutorOptions) AttachScanFeatures() {
|
||||
if e == nil || e.Options == nil {
|
||||
return
|
||||
}
|
||||
o := e.Options
|
||||
e.HTTPService, e.HostLimit, e.StreamLimit, e.HTTPResponseCache = nil, nil, nil, nil
|
||||
if !o.DisableServicePlacement {
|
||||
e.HTTPService = httpservice.New(o.ExecutionId)
|
||||
}
|
||||
if !o.DisableAdaptiveConcurrency {
|
||||
e.HostLimit = hostlimit.New(hostlimit.Config{})
|
||||
// Raw connections carry no latency signal, so this limit never grows:
|
||||
// a fixed bound per endpoint, halved when dials start failing.
|
||||
e.StreamLimit = hostlimit.New(hostlimit.Config{Initial: 64, Min: 8, Max: 64})
|
||||
}
|
||||
if o.UsesResponseCache() {
|
||||
e.HTTPResponseCache = httprespcache.New()
|
||||
}
|
||||
}
|
||||
|
||||
// ResetScanState clears the per-scan content of the attached features in place.
|
||||
// Compiled templates may hold their own copy of these options, so a long-lived
|
||||
// engine must reset the shared instances rather than replace them.
|
||||
func (e *ExecutorOptions) ResetScanState() {
|
||||
if e == nil {
|
||||
return
|
||||
}
|
||||
e.HTTPResponseCache.Reset()
|
||||
e.HTTPService.Reset()
|
||||
}
|
||||
|
||||
// ApplyNewEngineOptions updates an existing ExecutorOptions with options from a new engine. This
|
||||
// handles things like the ExecutionID that need to be updated.
|
||||
func (e *ExecutorOptions) ApplyNewEngineOptions(n *ExecutorOptions) {
|
||||
@@ -705,8 +758,12 @@ func (e *ExecutorOptions) ApplyNewEngineOptions(n *ExecutorOptions) {
|
||||
e.Browser = n.Browser
|
||||
e.Interactsh = n.Interactsh
|
||||
e.InteractshScope = n.InteractshScope
|
||||
e.HTTPResponseCache = n.HTTPResponseCache
|
||||
e.HostErrorsCache = n.HostErrorsCache
|
||||
e.HostBackoff = n.HostBackoff
|
||||
e.HTTPService = n.HTTPService
|
||||
e.HostLimit = n.HostLimit
|
||||
e.StreamLimit = n.StreamLimit
|
||||
e.InputHelper = n.InputHelper
|
||||
e.FuzzParamsFrequency = n.FuzzParamsFrequency
|
||||
e.FuzzStatsDB = n.FuzzStatsDB
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
package protocols
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httprespcache"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
)
|
||||
|
||||
func TestAttachScanFeaturesDefaults(t *testing.T) {
|
||||
e := &ExecutorOptions{Options: &types.Options{}}
|
||||
e.AttachScanFeatures()
|
||||
if e.HTTPService == nil || e.HostLimit == nil || e.StreamLimit == nil {
|
||||
t.Fatalf("defaults must attach placement and adaptive concurrency: %+v", e)
|
||||
}
|
||||
if e.HTTPResponseCache != nil {
|
||||
t.Fatal("the response cache is opt-in")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAttachScanFeaturesResponseCacheOptIn(t *testing.T) {
|
||||
e := &ExecutorOptions{Options: &types.Options{ResponseCache: true}}
|
||||
e.AttachScanFeatures()
|
||||
if e.HTTPResponseCache == nil {
|
||||
t.Fatal("-response-cache must attach the cache")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAttachScanFeaturesHonoursDisable(t *testing.T) {
|
||||
e := &ExecutorOptions{Options: &types.Options{
|
||||
DisableServicePlacement: true,
|
||||
DisableAdaptiveConcurrency: true,
|
||||
}}
|
||||
e.AttachScanFeatures()
|
||||
if e.HTTPService != nil || e.HostLimit != nil || e.StreamLimit != nil || e.HTTPResponseCache != nil {
|
||||
t.Fatal("disabled features must stay off")
|
||||
}
|
||||
}
|
||||
|
||||
// A long-lived engine keeps the instances (templates may hold a copy of the
|
||||
// options) and only empties their per-scan content.
|
||||
func TestResetScanStateEmptiesSharedCache(t *testing.T) {
|
||||
e := &ExecutorOptions{Options: &types.Options{ResponseCache: true}}
|
||||
e.AttachScanFeatures()
|
||||
cache := e.HTTPResponseCache
|
||||
req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil)
|
||||
cache.Set(httprespcache.KeyFromRequest(req), "", &http.Response{StatusCode: 200, Header: http.Header{}}, []byte("x"))
|
||||
if cache.Len() != 1 {
|
||||
t.Fatal("setup: entry not stored")
|
||||
}
|
||||
e.ResetScanState()
|
||||
if e.HTTPResponseCache != cache {
|
||||
t.Fatal("reset must keep the same instance")
|
||||
}
|
||||
if cache.Len() != 0 {
|
||||
t.Fatal("reset must drop cached responses")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanFeatureHelpersOnNil(t *testing.T) {
|
||||
var e *ExecutorOptions
|
||||
e.AttachScanFeatures()
|
||||
e.ResetScanState()
|
||||
(&ExecutorOptions{}).AttachScanFeatures()
|
||||
(&ExecutorOptions{}).ResetScanState()
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
package types
|
||||
|
||||
import (
|
||||
"io"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog"
|
||||
)
|
||||
|
||||
// Copy must carry every exported option. Fields added later without a matching
|
||||
// line in Copy are silently dropped wherever options are copied (for example in
|
||||
// every execution of the SDK's thread-safe engine). This test sets each
|
||||
// exported field to a non-zero value and checks it survives Copy.
|
||||
func TestCopyKeepsEveryExportedField(t *testing.T) {
|
||||
src := &Options{}
|
||||
v := reflect.ValueOf(src).Elem()
|
||||
typ := v.Type()
|
||||
var skipped []string
|
||||
for i := 0; i < v.NumField(); i++ {
|
||||
f := typ.Field(i)
|
||||
if !f.IsExported() {
|
||||
continue
|
||||
}
|
||||
if !fillNonZero(v.Field(i)) {
|
||||
skipped = append(skipped, f.Name)
|
||||
}
|
||||
}
|
||||
got := reflect.ValueOf(src.Copy()).Elem()
|
||||
for i := 0; i < v.NumField(); i++ {
|
||||
f := typ.Field(i)
|
||||
if !f.IsExported() || contains(skipped, f.Name) {
|
||||
continue
|
||||
}
|
||||
if !reflect.DeepEqual(v.Field(i).Interface(), got.Field(i).Interface()) {
|
||||
t.Errorf("Copy drops %s", f.Name)
|
||||
}
|
||||
}
|
||||
if len(skipped) > 0 {
|
||||
t.Logf("not checked (kind cannot be filled generically): %v", skipped)
|
||||
}
|
||||
}
|
||||
|
||||
// fillNonZero cannot fill an interface or a func, so check those two directly.
|
||||
func TestCopyKeepsParserAndLoadHelperFileFunction(t *testing.T) {
|
||||
parser := &struct{}{}
|
||||
loader := LoadHelperFileFunction(func(string, string, catalog.Catalog) (io.ReadCloser, error) {
|
||||
return nil, nil
|
||||
})
|
||||
src := &Options{
|
||||
Parser: parser,
|
||||
LoadHelperFileFunction: loader,
|
||||
}
|
||||
|
||||
got := src.Copy()
|
||||
if got.Parser != parser {
|
||||
t.Fatal("Copy drops Parser")
|
||||
}
|
||||
if got.LoadHelperFileFunction == nil {
|
||||
t.Fatal("Copy drops LoadHelperFileFunction")
|
||||
}
|
||||
}
|
||||
|
||||
func contains(list []string, s string) bool {
|
||||
for _, v := range list {
|
||||
if v == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// fillNonZero sets v to a non-zero value and reports whether it could.
|
||||
func fillNonZero(v reflect.Value) bool {
|
||||
switch v.Kind() {
|
||||
case reflect.Bool:
|
||||
v.SetBool(true)
|
||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
||||
v.SetInt(7)
|
||||
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
|
||||
v.SetUint(7)
|
||||
case reflect.Float32, reflect.Float64:
|
||||
v.SetFloat(7)
|
||||
case reflect.String:
|
||||
v.SetString("x")
|
||||
case reflect.Slice:
|
||||
s := reflect.MakeSlice(v.Type(), 1, 1)
|
||||
fillNonZero(s.Index(0))
|
||||
v.Set(s)
|
||||
case reflect.Map:
|
||||
v.Set(reflect.MakeMap(v.Type()))
|
||||
v.SetMapIndex(reflect.New(v.Type().Key()).Elem(), reflect.New(v.Type().Elem()).Elem())
|
||||
case reflect.Pointer:
|
||||
v.Set(reflect.New(v.Type().Elem()))
|
||||
case reflect.Struct:
|
||||
filled := false
|
||||
for i := 0; i < v.NumField(); i++ {
|
||||
if v.Type().Field(i).IsExported() && fillNonZero(v.Field(i)) {
|
||||
filled = true
|
||||
}
|
||||
}
|
||||
return filled
|
||||
default:
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package types
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/projectdiscovery/gologger/levels"
|
||||
)
|
||||
|
||||
func TestResolveTemplateThreads(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
set int
|
||||
disabled bool
|
||||
want int
|
||||
}{
|
||||
{"not set, adaptive on", 0, false, AdaptiveTemplateThreads},
|
||||
{"not set, adaptive off keeps the historical default", 0, true, DefaultTemplateThreads},
|
||||
{"explicit value wins with adaptive on", 25, false, 25},
|
||||
{"explicit value wins with adaptive off", 50, true, 50},
|
||||
{"explicit high value is kept", 300, false, 300},
|
||||
}
|
||||
for _, c := range cases {
|
||||
o := &Options{TemplateThreads: c.set, DisableAdaptiveConcurrency: c.disabled}
|
||||
o.ResolveTemplateThreads()
|
||||
if o.TemplateThreads != c.want {
|
||||
t.Errorf("%s: got %d want %d", c.name, o.TemplateThreads, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveTemplateThreadsRecordsAuto(t *testing.T) {
|
||||
auto := &Options{}
|
||||
auto.ResolveTemplateThreads()
|
||||
if !auto.TemplateThreadsAuto {
|
||||
t.Fatal("an unset -c must be recorded as automatic")
|
||||
}
|
||||
explicit := &Options{TemplateThreads: 25}
|
||||
explicit.ResolveTemplateThreads()
|
||||
if explicit.TemplateThreadsAuto {
|
||||
t.Fatal("an explicit -c must not be recorded as automatic")
|
||||
}
|
||||
if !auto.Copy().TemplateThreadsAuto {
|
||||
t.Fatal("Copy must keep TemplateThreadsAuto")
|
||||
}
|
||||
}
|
||||
|
||||
// Raising the default logger's level must log, not panic on a nil formatter.
|
||||
func TestDefaultLoggerCanBeRaised(t *testing.T) {
|
||||
o := DefaultOptions()
|
||||
o.Logger.SetMaxLevel(levels.LevelVerbose)
|
||||
o.Logger.Verbose().Msgf("verbose %d", 1)
|
||||
o.Logger.Info().Msg("info")
|
||||
}
|
||||
+82
-1
@@ -11,6 +11,9 @@ import (
|
||||
|
||||
"github.com/projectdiscovery/goflags"
|
||||
"github.com/projectdiscovery/gologger"
|
||||
"github.com/projectdiscovery/gologger/formatter"
|
||||
"github.com/projectdiscovery/gologger/levels"
|
||||
"github.com/projectdiscovery/gologger/writer"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
|
||||
@@ -141,6 +144,9 @@ type Options struct {
|
||||
BulkSize int
|
||||
// TemplateThreads is the number of templates executed in parallel
|
||||
TemplateThreads int
|
||||
// TemplateThreadsAuto is set when -c was not given and TemplateThreads was
|
||||
// filled in by ResolveTemplateThreads.
|
||||
TemplateThreadsAuto bool
|
||||
// HeadlessBulkSize is the of targets analyzed in parallel for each headless template
|
||||
HeadlessBulkSize int
|
||||
// HeadlessTemplateThreads is the number of headless templates executed in parallel
|
||||
@@ -211,6 +217,13 @@ type Options struct {
|
||||
// PreflightPortScan enables a preflight resolve + TCP portscan and filters targets
|
||||
// before running templates. Disabled by default.
|
||||
PreflightPortScan bool
|
||||
// DisableServicePlacement turns off skipping JS templates whose only
|
||||
// protocol libraries cannot speak HTTP (SMB, SSH, MySQL, ...) on an input
|
||||
// port that answered HTTP. Placement is on by default.
|
||||
DisableServicePlacement bool
|
||||
// DisableAdaptiveConcurrency turns off the per-endpoint limit that adapts
|
||||
// in-flight requests to each endpoint's latency and errors. On by default.
|
||||
DisableAdaptiveConcurrency bool
|
||||
// HostBackoff slows a host down when it signals it is being overloaded
|
||||
// (429, a run of 403s, or transport failures) instead of scanning it at full
|
||||
// rate until it blocks the scanner. Opt-in, since it changes scan pacing.
|
||||
@@ -239,6 +252,11 @@ type Options struct {
|
||||
// NOTE(dwisiswant0): this is experimental and might be enabled by default
|
||||
// in the future releases.
|
||||
EnableHTTPCache bool
|
||||
// ResponseCache reuses identical safe GET/HEAD responses inside one scan.
|
||||
// Only requests without auth, cookies, Host override, body, cache directives
|
||||
// or non-allowlisted headers are eligible, so the reuse is lossless. Opt-in:
|
||||
// a template that depends on state or timing could see a different result.
|
||||
ResponseCache bool
|
||||
// LeaveDefaultPorts skips normalization of default ports
|
||||
LeaveDefaultPorts bool
|
||||
// AutomaticScan enables automatic tech based template execution
|
||||
@@ -659,6 +677,23 @@ func (options *Options) Copy() *Options {
|
||||
NoHostErrors: options.NoHostErrors,
|
||||
BulkSize: options.BulkSize,
|
||||
TemplateThreads: options.TemplateThreads,
|
||||
TemplateThreadsAuto: options.TemplateThreadsAuto,
|
||||
CDPEndpoint: options.CDPEndpoint,
|
||||
HostBackoff: options.HostBackoff,
|
||||
HostBackoffMax: options.HostBackoffMax,
|
||||
MaxURLsPerPattern: options.MaxURLsPerPattern,
|
||||
DisableURLNormalization: options.DisableURLNormalization,
|
||||
EnableLLM: options.EnableLLM,
|
||||
LLMProvider: options.LLMProvider,
|
||||
LLMBaseURL: options.LLMBaseURL,
|
||||
LLMModel: options.LLMModel,
|
||||
LLMTimeout: options.LLMTimeout,
|
||||
LLMMaxCalls: options.LLMMaxCalls,
|
||||
LLMConcurrency: options.LLMConcurrency,
|
||||
LLMCache: options.LLMCache,
|
||||
VarsTextTemplating: options.VarsTextTemplating,
|
||||
VarsFilePaths: options.VarsFilePaths,
|
||||
TemplateLoadingConcurrency: options.TemplateLoadingConcurrency,
|
||||
templateThreadsProvider: options.templateThreadsProvider,
|
||||
templateThreadAcquire: options.templateThreadAcquire,
|
||||
templateThreadRelease: options.templateThreadRelease,
|
||||
@@ -696,6 +731,7 @@ func (options *Options) Copy() *Options {
|
||||
PerHostRateLimit: options.PerHostRateLimit,
|
||||
PerHostRateLimitPoolSize: options.PerHostRateLimitPoolSize,
|
||||
EnableHTTPCache: options.EnableHTTPCache,
|
||||
ResponseCache: options.ResponseCache,
|
||||
LeaveDefaultPorts: options.LeaveDefaultPorts,
|
||||
AutomaticScan: options.AutomaticScan,
|
||||
Silent: options.Silent,
|
||||
@@ -723,6 +759,8 @@ func (options *Options) Copy() *Options {
|
||||
TagList: options.TagList,
|
||||
HangMonitor: options.HangMonitor,
|
||||
Stdin: options.Stdin,
|
||||
DisableServicePlacement: options.DisableServicePlacement,
|
||||
DisableAdaptiveConcurrency: options.DisableAdaptiveConcurrency,
|
||||
StopAtFirstMatch: options.StopAtFirstMatch,
|
||||
HoneypotDetection: options.HoneypotDetection,
|
||||
HoneypotThreshold: options.HoneypotThreshold,
|
||||
@@ -924,12 +962,43 @@ func (options *Options) ShouldFollowHTTPRedirects() bool {
|
||||
return options.FollowRedirects || options.FollowHostRedirects
|
||||
}
|
||||
|
||||
// UsesResponseCache reports whether the scan-scoped response cache may be active:
|
||||
// -response-cache.
|
||||
func (options *Options) UsesResponseCache() bool {
|
||||
return options.ResponseCache
|
||||
}
|
||||
|
||||
// HasClientCertificates determines if any client certificate was specified
|
||||
func (options *Options) HasClientCertificates() bool {
|
||||
return options.ClientCertFile != "" || options.ClientCAFile != "" || options.ClientKeyFile != ""
|
||||
}
|
||||
|
||||
// DefaultOptions returns default options for nuclei
|
||||
// AdaptiveTemplateThreads is the template concurrency used when adaptive
|
||||
// concurrency is on and -c was not set. Each endpoint's own limit then decides
|
||||
// what the target receives, so more ready templates only keep endpoints busy.
|
||||
// Without the endpoint limit this value overloads slow targets and loses
|
||||
// findings, so it applies only while adaptive concurrency is on.
|
||||
const AdaptiveTemplateThreads = 100
|
||||
|
||||
// DefaultTemplateThreads is the template concurrency without adaptive
|
||||
// concurrency.
|
||||
const DefaultTemplateThreads = 25
|
||||
|
||||
// ResolveTemplateThreads fills in -c when it was not set (0). Any explicit
|
||||
// value is kept. Without adaptive concurrency the historical default applies,
|
||||
// because a higher value overloads slow targets and loses findings.
|
||||
func (options *Options) ResolveTemplateThreads() {
|
||||
if options.TemplateThreads > 0 {
|
||||
return
|
||||
}
|
||||
options.TemplateThreadsAuto = true
|
||||
options.TemplateThreads = DefaultTemplateThreads
|
||||
if !options.DisableAdaptiveConcurrency {
|
||||
options.TemplateThreads = AdaptiveTemplateThreads
|
||||
}
|
||||
}
|
||||
|
||||
func DefaultOptions() *Options {
|
||||
return &Options{
|
||||
RateLimit: 150,
|
||||
@@ -948,10 +1017,22 @@ func DefaultOptions() *Options {
|
||||
ResponseSaveSize: unitutils.Mega,
|
||||
RequestSaveSize: 32 * unitutils.Kilo,
|
||||
ExecutionId: xid.New().String(),
|
||||
Logger: &gologger.Logger{},
|
||||
Logger: newQuietLogger(),
|
||||
}
|
||||
}
|
||||
|
||||
// newQuietLogger returns the default options logger: as quiet as the zero value
|
||||
// (fatal only), but with a formatter and a writer, so raising its level (for
|
||||
// example with the SDK's WithVerbosity) logs instead of dereferencing a nil
|
||||
// formatter.
|
||||
func newQuietLogger() *gologger.Logger {
|
||||
l := &gologger.Logger{}
|
||||
l.SetMaxLevel(levels.LevelFatal)
|
||||
l.SetFormatter(formatter.NewCLI(false))
|
||||
l.SetWriter(writer.NewCLI())
|
||||
return l
|
||||
}
|
||||
|
||||
func (options *Options) ShouldUseHostError() bool {
|
||||
return options.MaxHostError > 0 && !options.NoHostErrors
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user