mirror of
https://github.com/projectdiscovery/nuclei.git
synced 2026-11-08 17:39:09 -10:00
Compare commits
17
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Mzack9999
|
4434e544b3 | ||
|
Mzack9999
|
941783e4ae | ||
|
Mzack9999
|
31d22f1113 | ||
|
Mzack9999
|
55e5b99efd | ||
|
Mzack9999
|
28787c019c | ||
|
Mzack9999
|
92e91e842c | ||
|
Mzack9999
|
0f8567a023 | ||
|
Mzack9999
|
4e496bb606 | ||
|
Mzack9999
|
c402526699 | ||
|
Mzack9999
|
0549f25301 | ||
|
Mzack9999
|
ec63c55bf7 | ||
|
Mzack9999
|
a116664aad | ||
|
Mzack9999
|
99e1309586 | ||
|
Mzack9999
|
531387f5c7 | ||
|
Mzack9999
|
7a26399fe9 | ||
|
Mzack9999
|
09b61208b6 | ||
|
Mzack9999
|
22255ede65 |
No files matched your search
@@ -295,19 +295,21 @@ UNCOVER:
|
||||
-ur, -uncover-ratelimit int override ratelimit of engines with unknown ratelimit (default 60 req/min) (default 60)
|
||||
|
||||
RATE-LIMIT:
|
||||
-rl, -rate-limit int maximum number of requests to send per second (default 150)
|
||||
-rld, -rate-limit-duration value maximum number of requests to send per second (default 1s)
|
||||
-host-backoff slow down a host that starts blocking (429, repeated 403s, connection failures)
|
||||
-host-backoff-max value maximum delay applied to a host that is blocking (default 30s)
|
||||
-per-host-rate-limit enable per-host rate limiting (global rate limit becomes unlimited when enabled)
|
||||
-rlm, -rate-limit-minute int maximum number of requests to send per minute (DEPRECATED)
|
||||
-bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25)
|
||||
-c, -concurrency int maximum number of templates to be executed in parallel (default 25)
|
||||
-hbs, -headless-bulk-size int maximum number of headless hosts to be analyzed in parallel per template (default 10)
|
||||
-headc, -headless-concurrency int maximum number of headless templates to be executed in parallel (default 10)
|
||||
-jsc, -js-concurrency int maximum number of javascript runtimes to be executed in parallel (default 120)
|
||||
-pc, -payload-concurrency int max payload concurrency for each template (default 25)
|
||||
-prc, -probe-concurrency int http probe concurrency with httpx (default 50)
|
||||
-rl, -rate-limit int maximum number of requests to send per second (ignored when -rate-limit-host is set) (default 150)
|
||||
-rld, -rate-limit-duration value rate-limit interval/window for -rate-limit (default 1s)
|
||||
-host-backoff slow down a host that starts blocking (429, repeated 403s, connection failures)
|
||||
-host-backoff-max value maximum delay applied to a host that is blocking (default 30s)
|
||||
-per-host-rate-limit enable per-host rate limiting (global rate limit becomes unlimited when enabled)
|
||||
-rlm, -rate-limit-minute int maximum number of requests to send per minute (DEPRECATED)
|
||||
-rlh, -rate-limit-host int maximum number of requests to send per host per rate-limit-host-duration (0 = disabled, takes priority over -rate-limit)
|
||||
-rlhd, -rate-limit-host-duration value refill interval for the per-host rate limit bucket (default 1s)
|
||||
-bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25)
|
||||
-c, -concurrency int maximum number of templates to be executed in parallel (default 25)
|
||||
-hbs, -headless-bulk-size int maximum number of headless hosts to be analyzed in parallel per template (default 10)
|
||||
-headc, -headless-concurrency int maximum number of headless templates to be executed in parallel (default 10)
|
||||
-jsc, -js-concurrency int maximum number of javascript runtimes to be executed in parallel (default 120)
|
||||
-pc, -payload-concurrency int max payload concurrency for each template (default 25)
|
||||
-prc, -probe-concurrency int http probe concurrency with httpx (default 50)
|
||||
-tlc, -template-loading-concurrency int maximum number of concurrent template loading operations (default 50)
|
||||
|
||||
OPTIMIZATIONS:
|
||||
|
||||
+4
-2
@@ -292,10 +292,12 @@ UNCOVER:
|
||||
-ur, -uncover-ratelimit int override ratelimit of engines with unknown ratelimit (default 60 req/min) (default 60)
|
||||
|
||||
RATE-LIMIT:
|
||||
-rl, -rate-limit int maximum number of requests to send per second (default 150)
|
||||
-rld, -rate-limit-duration value maximum number of requests to send per second (default 1s)
|
||||
-rl, -rate-limit int maximum number of requests to send per second (ignored when -rate-limit-host is set) (default 150)
|
||||
-rld, -rate-limit-duration value rate-limit interval/window for -rate-limit (default 1s)
|
||||
-per-host-rate-limit enable per-host rate limiting (global rate limit becomes unlimited when enabled)
|
||||
-rlm, -rate-limit-minute int maximum number of requests to send per minute (DEPRECATED)
|
||||
-rlh, -rate-limit-host int maximum number of requests to send per host per rate-limit-host-duration (0 = disabled, takes priority over -rate-limit)
|
||||
-rlhd, -rate-limit-host-duration value refill interval for the per-host rate limit bucket (default 1s)
|
||||
-bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25)
|
||||
-c, -concurrency int maximum number of templates to be executed in parallel (default 25)
|
||||
-hbs, -headless-bulk-size int maximum number of headless hosts to be analyzed in parallel per template (default 10)
|
||||
|
||||
+13
-11
@@ -292,18 +292,20 @@ UNCOVER:
|
||||
-ur, -uncover-ratelimit int override ratelimit of engines with unknown ratelimit (default 60 req/min) (default 60)
|
||||
|
||||
RATE-LIMIT:
|
||||
-rl, -rate-limit int maximum number of requests to send per second (default 150)
|
||||
-rld, -rate-limit-duration value maximum number of requests to send per second (default 1s)
|
||||
-rl, -rate-limit int número máximo de peticiones a enviar por segundo (se ignora cuando -rate-limit-host está activo) (por defecto 150)
|
||||
-rld, -rate-limit-duration value intervalo/ventana de recarga para -rate-limit (por defecto 1s)
|
||||
-per-host-rate-limit enable per-host rate limiting (global rate limit becomes unlimited when enabled)
|
||||
-rlm, -rate-limit-minute int maximum number of requests to send per minute (DEPRECATED)
|
||||
-bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25)
|
||||
-c, -concurrency int maximum number of templates to be executed in parallel (default 25)
|
||||
-hbs, -headless-bulk-size int maximum number of headless hosts to be analyzed in parallel per template (default 10)
|
||||
-headc, -headless-concurrency int maximum number of headless templates to be executed in parallel (default 10)
|
||||
-jsc, -js-concurrency int maximum number of javascript runtimes to be executed in parallel (default 120)
|
||||
-pc, -payload-concurrency int max payload concurrency for each template (default 25)
|
||||
-prc, -probe-concurrency int http probe concurrency with httpx (default 50)
|
||||
-tlc, -template-loading-concurrency int maximum number of concurrent template loading operations (default 50)
|
||||
-rlm, -rate-limit-minute int número máximo de peticiones a enviar por minuto (OBSOLETO)
|
||||
-rlh, -rate-limit-host int número máximo de peticiones por host por rate-limit-host-duration (0 = desactivado, tiene prioridad sobre -rate-limit)
|
||||
-rlhd, -rate-limit-host-duration value intervalo de recarga del bucket de rate-limit por host (por defecto 1s)
|
||||
-bs, -bulk-size int número máximo de hosts a ser analizados en paralelo por plantilla (por defecto 25)
|
||||
-c, -concurrency int número máximo de plantillas a ejecutar en paralelo (por defecto 25)
|
||||
-hbs, -headless-bulk-size int número máximo de hosts headless a ser analizados en paralelo por plantilla (por defecto 10)
|
||||
-headc, -headless-concurrency int número máximo de plantillas headless a ejecutar en paralelo (por defecto 10)
|
||||
-jsc, -js-concurrency int número máximo de entornos de ejecución de JavaScript a ejecutar en paralelo (por defecto 120)
|
||||
-pc, -payload-concurrency int concurrencia máxima de carga útil para cada plantilla (por defecto 25)
|
||||
-prc, -probe-concurrency int concurrencia de sondas http con httpx (por defecto 50)
|
||||
-tlc, -template-loading-concurrency int número máximo de operaciones de carga de plantillas concurrentes (por defecto 50)
|
||||
|
||||
OPTIMIZATIONS:
|
||||
-timeout int time to wait in seconds before timeout (default 10)
|
||||
|
||||
+4
-2
@@ -310,12 +310,14 @@ UNCOVER:
|
||||
-ur, -uncover-ratelimit int override ratelimit of engines with unknown ratelimit (default 60 req/min) (default 60)
|
||||
|
||||
RATE-LIMIT:
|
||||
-rl, -rate-limit int maximum number of requests to send per second (default 150)
|
||||
-rld, -rate-limit-duration value maximum number of requests to send per second (default 1s)
|
||||
-rl, -rate-limit int maximum number of requests to send per second (ignored when -rate-limit-host is set) (default 150)
|
||||
-rld, -rate-limit-duration value rate-limit interval/window for -rate-limit (default 1s)
|
||||
-host-backoff slow down a host that starts blocking (429, repeated 403s, connection failures)
|
||||
-host-backoff-max value maximum delay applied to a host that is blocking (default 30s)
|
||||
-per-host-rate-limit enable per-host rate limiting (global rate limit becomes unlimited when enabled)
|
||||
-rlm, -rate-limit-minute int maximum number of requests to send per minute (DEPRECATED)
|
||||
-rlh, -rate-limit-host int maximum number of requests to send per host per rate-limit-host-duration (0 = disabled, takes priority over -rate-limit)
|
||||
-rlhd, -rate-limit-host-duration value refill interval for the per-host rate limit bucket (default 1s)
|
||||
-bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25)
|
||||
-c, -concurrency int maximum number of templates to be executed in parallel (default 25)
|
||||
-hbs, -headless-bulk-size int maximum number of headless hosts to be analyzed in parallel per template (default 10)
|
||||
|
||||
+4
-2
@@ -292,10 +292,12 @@ UNCOVER:
|
||||
-ur, -uncover-ratelimit int override ratelimit of engines with unknown ratelimit (default 60 req/min) (default 60)
|
||||
|
||||
RATE-LIMIT:
|
||||
-rl, -rate-limit int maximum number of requests to send per second (default 150)
|
||||
-rld, -rate-limit-duration value maximum number of requests to send per second (default 1s)
|
||||
-rl, -rate-limit int maximum number of requests to send per second (ignored when -rate-limit-host is set) (default 150)
|
||||
-rld, -rate-limit-duration value rate-limit interval/window for -rate-limit (default 1s)
|
||||
-per-host-rate-limit enable per-host rate limiting (global rate limit becomes unlimited when enabled)
|
||||
-rlm, -rate-limit-minute int maximum number of requests to send per minute (DEPRECATED)
|
||||
-rlh, -rate-limit-host int maximum number of requests to send per host per rate-limit-host-duration (0 = disabled, takes priority over -rate-limit)
|
||||
-rlhd, -rate-limit-host-duration value refill interval for the per-host rate limit bucket (default 1s)
|
||||
-bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25)
|
||||
-c, -concurrency int maximum number of templates to be executed in parallel (default 25)
|
||||
-hbs, -headless-bulk-size int maximum number of headless hosts to be analyzed in parallel per template (default 10)
|
||||
|
||||
+4
-2
@@ -292,10 +292,12 @@ UNCOVER:
|
||||
-ur, -uncover-ratelimit int override ratelimit of engines with unknown ratelimit (default 60 req/min) (default 60)
|
||||
|
||||
RATE-LIMIT:
|
||||
-rl, -rate-limit int maximum number of requests to send per second (default 150)
|
||||
-rld, -rate-limit-duration value maximum number of requests to send per second (default 1s)
|
||||
-rl, -rate-limit int maximum number of requests to send per second (ignored when -rate-limit-host is set) (default 150)
|
||||
-rld, -rate-limit-duration value rate-limit interval/window for -rate-limit (default 1s)
|
||||
-per-host-rate-limit enable per-host rate limiting (global rate limit becomes unlimited when enabled)
|
||||
-rlm, -rate-limit-minute int maximum number of requests to send per minute (DEPRECATED)
|
||||
-rlh, -rate-limit-host int maximum number of requests to send per host per rate-limit-host-duration (0 = disabled, takes priority over -rate-limit)
|
||||
-rlhd, -rate-limit-host-duration value refill interval for the per-host rate limit bucket (default 1s)
|
||||
-bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25)
|
||||
-c, -concurrency int maximum number of templates to be executed in parallel (default 25)
|
||||
-hbs, -headless-bulk-size int maximum number of headless hosts to be analyzed in parallel per template (default 10)
|
||||
|
||||
+13
-11
@@ -292,18 +292,20 @@ UNCOVER:
|
||||
-ur, -uncover-ratelimit int override ratelimit of engines with unknown ratelimit (default 60 req/min) (default 60)
|
||||
|
||||
RATE-LIMIT:
|
||||
-rl, -rate-limit int maximum number of requests to send per second (default 150)
|
||||
-rld, -rate-limit-duration value maximum number of requests to send per second (default 1s)
|
||||
-rl, -rate-limit int 초당 보낼 최대 요청 수 (-rate-limit-host 설정 시 무시됨) (기본값 150)
|
||||
-rld, -rate-limit-duration value -rate-limit의 리필 간격/윈도우 (기본값 1s)
|
||||
-per-host-rate-limit enable per-host rate limiting (global rate limit becomes unlimited when enabled)
|
||||
-rlm, -rate-limit-minute int maximum number of requests to send per minute (DEPRECATED)
|
||||
-bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25)
|
||||
-c, -concurrency int maximum number of templates to be executed in parallel (default 25)
|
||||
-hbs, -headless-bulk-size int maximum number of headless hosts to be analyzed in parallel per template (default 10)
|
||||
-headc, -headless-concurrency int maximum number of headless templates to be executed in parallel (default 10)
|
||||
-jsc, -js-concurrency int maximum number of javascript runtimes to be executed in parallel (default 120)
|
||||
-pc, -payload-concurrency int max payload concurrency for each template (default 25)
|
||||
-prc, -probe-concurrency int http probe concurrency with httpx (default 50)
|
||||
-tlc, -template-loading-concurrency int maximum number of concurrent template loading operations (default 50)
|
||||
-rlm, -rate-limit-minute int 분당 보낼 최대 요청 수 (사용 중단됨)
|
||||
-rlh, -rate-limit-host int 호스트당 rate-limit-host-duration 동안 보낼 최대 요청 수 (0 = 비활성, -rate-limit보다 우선합니다)
|
||||
-rlhd, -rate-limit-host-duration value 호스트별 rate-limit 버킷의 리필 간격 (기본값 1s)
|
||||
-bs, -bulk-size int 템플릿당 병렬로 분석할 최대 호스트 수 (기본값 25)
|
||||
-c, -concurrency int 병렬로 실행할 최대 템플릿 수 (기본값 25)
|
||||
-hbs, -headless-bulk-size int 템플릿당 병렬로 분석할 최대 headless 호스트 수 (기본값 10)
|
||||
-headc, -headless-concurrency int 병렬로 실행할 최대 headless 템플릿 수 (기본값 10)
|
||||
-jsc, -js-concurrency int 병렬로 실행할 최대 JavaScript 런타임 수 (기본값 120)
|
||||
-pc, -payload-concurrency int 템플릿당 최대 페이로드 동시성 (기본값 25)
|
||||
-prc, -probe-concurrency int httpx를 사용한 http 프로브 동시성 (기본값 50)
|
||||
-tlc, -template-loading-concurrency int 최대 동시 템플릿 로딩 작업 수 (기본값 50)
|
||||
|
||||
OPTIMIZATIONS:
|
||||
-timeout int time to wait in seconds before timeout (default 10)
|
||||
|
||||
+13
-11
@@ -292,18 +292,20 @@ UNCOVER:
|
||||
-ur, -uncover-ratelimit int override ratelimit of engines with unknown ratelimit (default 60 req/min) (default 60)
|
||||
|
||||
RATE-LIMIT:
|
||||
-rl, -rate-limit int maximum number of requests to send per second (default 150)
|
||||
-rld, -rate-limit-duration value maximum number of requests to send per second (default 1s)
|
||||
-rl, -rate-limit int número máximo de solicitações a serem enviadas por segundo (ignorado quando -rate-limit-host está definido) (padrão 150)
|
||||
-rld, -rate-limit-duration value intervalo/janela de recarga para -rate-limit (padrão 1s)
|
||||
-per-host-rate-limit enable per-host rate limiting (global rate limit becomes unlimited when enabled)
|
||||
-rlm, -rate-limit-minute int maximum number of requests to send per minute (DEPRECATED)
|
||||
-bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25)
|
||||
-c, -concurrency int maximum number of templates to be executed in parallel (default 25)
|
||||
-hbs, -headless-bulk-size int maximum number of headless hosts to be analyzed in parallel per template (default 10)
|
||||
-headc, -headless-concurrency int maximum number of headless templates to be executed in parallel (default 10)
|
||||
-jsc, -js-concurrency int maximum number of javascript runtimes to be executed in parallel (default 120)
|
||||
-pc, -payload-concurrency int max payload concurrency for each template (default 25)
|
||||
-prc, -probe-concurrency int http probe concurrency with httpx (default 50)
|
||||
-tlc, -template-loading-concurrency int maximum number of concurrent template loading operations (default 50)
|
||||
-rlm, -rate-limit-minute int número máximo de solicitações a serem enviadas por minuto (OBSOLETO)
|
||||
-rlh, -rate-limit-host int número máximo de solicitações por host por rate-limit-host-duration (0 = desativado, tem prioridade sobre -rate-limit)
|
||||
-rlhd, -rate-limit-host-duration value intervalo de recarga do bucket de rate-limit por host (padrão 1s)
|
||||
-bs, -bulk-size int número máximo de hosts a serem analisados em paralelo por template (padrão 25)
|
||||
-c, -concurrency int número máximo de templates a serem executados em paralelo (padrão 25)
|
||||
-hbs, -headless-bulk-size int número máximo de hosts headless a serem analisados em paralelo por template (padrão 10)
|
||||
-headc, -headless-concurrency int número máximo de templates headless a serem executados em paralelo (padrão 10)
|
||||
-jsc, -js-concurrency int número máximo de ambientes de execução de JavaScript a serem executados em paralelo (padrão 120)
|
||||
-pc, -payload-concurrency int concorrência máxima de payload para cada template (padrão 25)
|
||||
-prc, -probe-concurrency int concorrência de sondas http com httpx (padrão 50)
|
||||
-tlc, -template-loading-concurrency int número máximo de operações de carregamento de templates concorrentes (padrão 50)
|
||||
|
||||
OPTIMIZATIONS:
|
||||
-timeout int time to wait in seconds before timeout (default 10)
|
||||
|
||||
+4
-2
@@ -292,10 +292,12 @@ UNCOVER:
|
||||
-ur, -uncover-ratelimit int override ratelimit of engines with unknown ratelimit (default 60 req/min) (default 60)
|
||||
|
||||
RATE-LIMIT:
|
||||
-rl, -rate-limit int maximum number of requests to send per second (default 150)
|
||||
-rld, -rate-limit-duration value maximum number of requests to send per second (default 1s)
|
||||
-rl, -rate-limit int maximum number of requests to send per second (ignored when -rate-limit-host is set) (default 150)
|
||||
-rld, -rate-limit-duration value rate-limit interval/window for -rate-limit (default 1s)
|
||||
-per-host-rate-limit enable per-host rate limiting (global rate limit becomes unlimited when enabled)
|
||||
-rlm, -rate-limit-minute int maximum number of requests to send per minute (DEPRECATED)
|
||||
-rlh, -rate-limit-host int maximum number of requests to send per host per rate-limit-host-duration (0 = disabled, takes priority over -rate-limit)
|
||||
-rlhd, -rate-limit-host-duration value refill interval for the per-host rate limit bucket (default 1s)
|
||||
-bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25)
|
||||
-c, -concurrency int maximum number of templates to be executed in parallel (default 25)
|
||||
-hbs, -headless-bulk-size int maximum number of headless hosts to be analyzed in parallel per template (default 10)
|
||||
|
||||
+4
-2
@@ -434,12 +434,14 @@ on extensive configurability, massive extensibility and ease of use.`)
|
||||
)
|
||||
|
||||
flagSet.CreateGroup("rate-limit", "Rate-Limit",
|
||||
flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "maximum number of requests to send per second"),
|
||||
flagSet.DurationVarP(&options.RateLimitDuration, "rate-limit-duration", "rld", time.Second, "maximum number of requests to send per second"),
|
||||
flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "maximum number of requests to send per second (ignored when -rate-limit-host is set)"),
|
||||
flagSet.DurationVarP(&options.RateLimitDuration, "rate-limit-duration", "rld", time.Second, "rate-limit interval/window for -rate-limit"),
|
||||
flagSet.BoolVar(&options.HostBackoff, "host-backoff", false, "slow down a host that starts blocking (429, repeated 403s, connection failures)"),
|
||||
flagSet.DurationVar(&options.HostBackoffMax, "host-backoff-max", 30*time.Second, "maximum delay applied to a host that is blocking"),
|
||||
flagSet.BoolVar(&options.PerHostRateLimit, "per-host-rate-limit", false, "enable per-host rate limiting (global rate limit becomes unlimited when enabled)"),
|
||||
flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute (DEPRECATED)"),
|
||||
flagSet.IntVarP(&options.RateLimitHost, "rate-limit-host", "rlh", 0, "maximum number of requests to send per host per rate-limit-host-duration (0 = disabled, takes priority over -rate-limit)"),
|
||||
flagSet.DurationVarP(&options.RateLimitHostDuration, "rate-limit-host-duration", "rlhd", time.Second, "refill interval for the per-host rate limit bucket"),
|
||||
flagSet.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "maximum number of hosts to be analyzed in parallel per template"),
|
||||
flagSet.IntVarP(&options.TemplateThreads, "concurrency", "c", 25, "maximum number of templates to be executed in parallel"),
|
||||
flagSet.IntVarP(&options.HeadlessBulkSize, "headless-bulk-size", "hbs", 10, "maximum number of headless hosts to be analyzed in parallel per template"),
|
||||
|
||||
@@ -189,6 +189,29 @@ func WithGlobalRateLimitCtx(ctx context.Context, maxTokens int, duration time.Du
|
||||
}
|
||||
}
|
||||
|
||||
// WithHostRateLimit configures a per-host rate limit. Each unique target
|
||||
// host is capped at maxTokens requests per duration. Pass maxTokens=0 to
|
||||
// disable per-host limiting (the default).
|
||||
//
|
||||
// The per-host limiter takes priority over the global rate limit
|
||||
// (WithGlobalRateLimitCtx): when this option is in effect, requests carrying
|
||||
// a host scope consult only the per-host bucket, and the global limiter is
|
||||
// bypassed. This is intentional — the global rate limit defaults to a
|
||||
// non-zero value, so layering both would silently throttle aggregate
|
||||
// throughput and defeat the point of opting into a per-host budget.
|
||||
// Aggregate scan throughput is naturally bounded by num_hosts * maxTokens.
|
||||
//
|
||||
// Requests without a host scope (rare, e.g. self-contained templates) fall
|
||||
// back to the global limiter so they remain paced.
|
||||
func WithHostRateLimit(ctx context.Context, maxTokens int, duration time.Duration) NucleiSDKOptions {
|
||||
return func(e *NucleiEngine) error {
|
||||
e.opts.RateLimitHost = maxTokens
|
||||
e.opts.RateLimitHostDuration = duration
|
||||
// the per-host limiters are created on first use for each host
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// HeadlessOpts contains options for headless templates
|
||||
type HeadlessOpts struct {
|
||||
PageTimeout int // timeout for page load
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
// GetPerHostRateLimiter gets or creates a rate limiter for a specific host
|
||||
// Returns nil if per-host rate limiting is not enabled
|
||||
func GetPerHostRateLimiter(options *types.Options, hostname string) (*PerHostRateLimiter, error) {
|
||||
if !options.PerHostRateLimit {
|
||||
if !options.PerHostRateLimitEnabled() {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
@@ -48,7 +48,7 @@ func GetPerHostRateLimiter(options *types.Options, hostname string) (*PerHostRat
|
||||
|
||||
// RecordPerHostRateLimitRequest records a request for pps stats calculation
|
||||
func RecordPerHostRateLimitRequest(options *types.Options, hostname string) {
|
||||
if !options.PerHostRateLimit || hostname == "" {
|
||||
if !options.PerHostRateLimitEnabled() || hostname == "" {
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -319,28 +319,44 @@ func newPerHostRateLimiter(options *types.Options) (*PerHostRateLimiter, error)
|
||||
if options == nil {
|
||||
return &PerHostRateLimiter{ctx: ctx, cancel: cancel, unlimited: true, limit: math.MaxUint32, duration: time.Millisecond}, nil
|
||||
}
|
||||
if options.RateLimit < 0 {
|
||||
limit, duration := hostRateLimit(options)
|
||||
if limit < 0 {
|
||||
cancel()
|
||||
return nil, fmt.Errorf("rate limit must not be negative: %d", options.RateLimit)
|
||||
return nil, fmt.Errorf("rate limit must not be negative: %d", limit)
|
||||
}
|
||||
if options.RateLimitDuration < 0 {
|
||||
if duration < 0 {
|
||||
cancel()
|
||||
return nil, fmt.Errorf("rate limit duration must not be negative: %s", options.RateLimitDuration)
|
||||
return nil, fmt.Errorf("rate limit duration must not be negative: %s", duration)
|
||||
}
|
||||
if options.RateLimit == 0 || options.RateLimitDuration == 0 {
|
||||
if limit == 0 || duration == 0 {
|
||||
return &PerHostRateLimiter{ctx: ctx, cancel: cancel, unlimited: true, limit: math.MaxUint32, duration: time.Millisecond}, nil
|
||||
}
|
||||
requestsPerSecond := rate.Limit(float64(options.RateLimit) / options.RateLimitDuration.Seconds())
|
||||
requestsPerSecond := rate.Limit(float64(limit) / duration.Seconds())
|
||||
return &PerHostRateLimiter{
|
||||
limiter: rate.NewLimiter(requestsPerSecond, options.RateLimit),
|
||||
limiter: rate.NewLimiter(requestsPerSecond, limit),
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
admission: make(chan struct{}, 1),
|
||||
limit: uint(options.RateLimit),
|
||||
duration: options.RateLimitDuration,
|
||||
limit: uint(limit),
|
||||
duration: duration,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// hostRateLimit returns the budget of one host: -rate-limit-host per
|
||||
// -rate-limit-host-duration when it is set, so the per-host budget is separate
|
||||
// from the global rate limit, and -rate-limit per -rate-limit-duration
|
||||
// otherwise (-per-host-rate-limit).
|
||||
func hostRateLimit(options *types.Options) (int, time.Duration) {
|
||||
if options.RateLimitHost > 0 {
|
||||
duration := options.RateLimitHostDuration
|
||||
if duration <= 0 {
|
||||
duration = time.Second
|
||||
}
|
||||
return options.RateLimitHost, duration
|
||||
}
|
||||
return options.RateLimit, options.RateLimitDuration
|
||||
}
|
||||
|
||||
func (p *PerHostRateLimitPool) EvictHost(host string) bool {
|
||||
normalizedHost := NormalizeHostPort(host)
|
||||
|
||||
|
||||
@@ -230,7 +230,11 @@ func (p *Page) WaitVisible(act *Action, out ActionData) error {
|
||||
}
|
||||
|
||||
func (p *Page) Sleeper(pollTimeout, timeout time.Duration) *Page {
|
||||
// copying the whole struct reads shared fields (History, rules, ...) that the
|
||||
// hijack goroutine mutates under p.mutex, so guard the copy with a read lock
|
||||
p.mutex.RLock()
|
||||
page := *p
|
||||
p.mutex.RUnlock()
|
||||
page.page = page.Page().Sleeper(func() utils.Sleeper {
|
||||
return createBackOffSleeper(pollTimeout, timeout)
|
||||
})
|
||||
@@ -238,7 +242,9 @@ func (p *Page) Sleeper(pollTimeout, timeout time.Duration) *Page {
|
||||
}
|
||||
|
||||
func (p *Page) Timeout(timeout time.Duration) *Page {
|
||||
p.mutex.RLock()
|
||||
page := *p
|
||||
p.mutex.RUnlock()
|
||||
page.page = page.Page().Timeout(timeout)
|
||||
return &page
|
||||
}
|
||||
|
||||
@@ -666,11 +666,19 @@ func shouldDisableKeepAliveForHTTPProxy(request *Request, options *protocols.Exe
|
||||
// user enables it, since the pooled transport sets custom dial hooks). It only
|
||||
// affects the time_delay analyzer decision below.
|
||||
func (r *Request) AnalyzeConnectionReuse(forceHTTP2 bool) ConnectionReusePolicy {
|
||||
// Priority 0: race and pipeline requests need dedicated connections. Race
|
||||
// uses a one-shot synced body gate that breaks if a connection is reused and
|
||||
// the body is re-read, and reusing a single keep-alive connection would also
|
||||
// serialize the requests and defeat the race.
|
||||
if r.Race || r.Pipeline {
|
||||
// Priority 0: race requests need dedicated connections. Race uses a one-shot
|
||||
// synced body gate that breaks if a connection is reused and the body is
|
||||
// re-read, and reusing a single keep-alive connection would also serialize
|
||||
// the requests and defeat the race.
|
||||
//
|
||||
// Pipelined requests are deliberately not listed here: they never go through
|
||||
// the pooled client, they are sent over rawhttp's pipeline client, which owns
|
||||
// its connections. Marking them unsafe adds "Connection: close" to every
|
||||
// pipelined request, so the server hangs up after the first response, the
|
||||
// remaining pipelined requests fail and the pipeline client is left with
|
||||
// goroutines that never retire. Templates that really want a close are still
|
||||
// caught by the explicit header checks below.
|
||||
if r.Race {
|
||||
return ReuseUnsafe
|
||||
}
|
||||
|
||||
|
||||
@@ -43,7 +43,7 @@ Accept-Encoding: gzip`},
|
||||
}
|
||||
|
||||
// TestAnalyzeConnectionReuse guards the connection-reuse policy: requests that
|
||||
// must not reuse pooled keep-alive connections (race, pipeline, explicit
|
||||
// must not reuse pooled keep-alive connections (race, explicit
|
||||
// "Connection: close", time-based analyzers) must be flagged ReuseUnsafe, while
|
||||
// everything else stays ReuseSafe so dev's per-host pooling keeps connections alive.
|
||||
func TestAnalyzeConnectionReuse(t *testing.T) {
|
||||
@@ -69,8 +69,16 @@ func TestAnalyzeConnectionReuse(t *testing.T) {
|
||||
want: ReuseUnsafe,
|
||||
},
|
||||
{
|
||||
name: "pipeline is unsafe",
|
||||
// Pipelined requests are sent over rawhttp's pipeline client, so a
|
||||
// forced "Connection: close" would only make the server hang up after
|
||||
// the first response of the pipeline.
|
||||
name: "pipeline is safe",
|
||||
request: &Request{Pipeline: true},
|
||||
want: ReuseSafe,
|
||||
},
|
||||
{
|
||||
name: "pipeline with explicit close is unsafe",
|
||||
request: &Request{Pipeline: true, Raw: []string{"GET / HTTP/1.1\r\nHost: {{Hostname}}\r\nConnection: close\r\n\r\n"}},
|
||||
want: ReuseUnsafe,
|
||||
},
|
||||
{
|
||||
|
||||
@@ -678,8 +678,17 @@ const defaultMaxRedirects = 10
|
||||
|
||||
type checkRedirectFunc func(req *http.Request, via []*http.Request) error
|
||||
|
||||
type redirectCallbackContextKey struct{}
|
||||
|
||||
// WithRedirectCallback returns a context that invokes callback with the
|
||||
// destination URL immediately before each allowed redirect is followed.
|
||||
func WithRedirectCallback(ctx context.Context, callback func(context.Context, string) error) context.Context {
|
||||
return context.WithValue(ctx, redirectCallbackContextKey{}, callback)
|
||||
}
|
||||
|
||||
func makeCheckRedirectFunc(redirectType RedirectFlow, maxRedirects int) checkRedirectFunc {
|
||||
return func(req *http.Request, via []*http.Request) error {
|
||||
var err error
|
||||
switch redirectType {
|
||||
case DontFollowRedirect:
|
||||
return http.ErrUseLastResponse
|
||||
@@ -694,15 +703,23 @@ func makeCheckRedirectFunc(redirectType RedirectFlow, maxRedirects int) checkRed
|
||||
if newHost != oldHost {
|
||||
return http.ErrUseLastResponse
|
||||
}
|
||||
return checkMaxRedirects(req, via, maxRedirects)
|
||||
err = checkMaxRedirects(req, via, maxRedirects)
|
||||
case FollowAllRedirect:
|
||||
return checkMaxRedirects(req, via, maxRedirects)
|
||||
err = checkMaxRedirects(req, via, maxRedirects)
|
||||
case FollowSameSchemeRedirect:
|
||||
previousScheme := via[len(via)-1].URL.Scheme
|
||||
if req.URL.Scheme != previousScheme {
|
||||
return http.ErrUseLastResponse
|
||||
}
|
||||
return checkMaxRedirects(req, via, maxRedirects)
|
||||
err = checkMaxRedirects(req, via, maxRedirects)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if callback, ok := req.Context().Value(redirectCallbackContextKey{}).(func(context.Context, string) error); ok && callback != nil {
|
||||
if err := callback(req.Context(), req.URL.String()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
package httpclientpool
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestNormalizeHost(t *testing.T) {
|
||||
@@ -170,6 +173,90 @@ func TestFollowAllRedirect(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedirectCallback(t *testing.T) {
|
||||
destination := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
defer destination.Close()
|
||||
|
||||
source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, destination.URL+"/final", http.StatusFound)
|
||||
}))
|
||||
defer source.Close()
|
||||
|
||||
var gotHost string
|
||||
ctx := WithRedirectCallback(context.Background(), func(_ context.Context, host string) error {
|
||||
gotHost = host
|
||||
return nil
|
||||
})
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, source.URL, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := &http.Client{CheckRedirect: makeCheckRedirectFunc(FollowAllRedirect, 10)}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
|
||||
if want := destination.URL + "/final"; gotHost != want {
|
||||
t.Fatalf("redirect callback url = %q, want %q", gotHost, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedirectCallbackPreservesSchemeAndPort(t *testing.T) {
|
||||
oldReq, err := http.NewRequest(http.MethodGet, "http://source.example/", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var gotHost string
|
||||
ctx := WithRedirectCallback(context.Background(), func(_ context.Context, host string) error {
|
||||
gotHost = host
|
||||
return nil
|
||||
})
|
||||
newReq, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://EXAMPLE.COM:8080/final", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := makeCheckRedirectFunc(FollowAllRedirect, 10)(newReq, []*http.Request{oldReq}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if gotHost != "http://EXAMPLE.COM:8080/final" {
|
||||
t.Fatalf("redirect callback url = %q, want %q", gotHost, "http://EXAMPLE.COM:8080/final")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedirectCallbackObservesRequestCancellation(t *testing.T) {
|
||||
destination := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
defer destination.Close()
|
||||
source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, destination.URL, http.StatusFound)
|
||||
}))
|
||||
defer source.Close()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
|
||||
defer cancel()
|
||||
ctx = WithRedirectCallback(ctx, func(ctx context.Context, _ string) error {
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
})
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, source.URL, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
start := time.Now()
|
||||
_, err = (&http.Client{CheckRedirect: makeCheckRedirectFunc(FollowAllRedirect, 10)}).Do(req)
|
||||
if err == nil {
|
||||
t.Fatal("expected canceled redirect")
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed > time.Second {
|
||||
t.Fatalf("redirect cancellation took %s", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFollowSameSchemeRedirect(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
@@ -255,6 +255,11 @@ func (request *Request) executeRaceRequest(input *contextargs.Context, dynamicVa
|
||||
}
|
||||
waitCtx, cancelWait := stopWithHandler(requestInput.Context(), spmHandler)
|
||||
err := request.waitHostBackoff(waitCtx, hostBackoffKey(httpRequest, requestInput))
|
||||
if err == nil && request.options.HostRateLimitEnabled() {
|
||||
// only -rate-limit-host applies here: the global limiter is not
|
||||
// consulted on the race and pipeline paths
|
||||
err = request.options.RateLimitTakeContext(waitCtx, hostBackoffKey(httpRequest, requestInput))
|
||||
}
|
||||
cancelWait()
|
||||
if err != nil || spmHandler.FoundFirstMatch() || spmHandler.Cancelled() {
|
||||
if err != nil {
|
||||
@@ -607,6 +612,11 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
|
||||
}
|
||||
waitCtx, cancelWait := stopWithHandler(requestInput.Context(), spmHandler)
|
||||
err := request.waitHostBackoff(waitCtx, hostBackoffKey(httpRequest, requestInput))
|
||||
if err == nil && request.options.HostRateLimitEnabled() {
|
||||
// only -rate-limit-host applies here: the global limiter is not
|
||||
// consulted on the race and pipeline paths
|
||||
err = request.options.RateLimitTakeContext(waitCtx, hostBackoffKey(httpRequest, requestInput))
|
||||
}
|
||||
cancelWait()
|
||||
if err != nil || spmHandler.FoundFirstMatch() || spmHandler.Cancelled() {
|
||||
if err != nil {
|
||||
@@ -692,11 +702,6 @@ executionLoop:
|
||||
// but this should be replaced once templateCtx is refactored properly
|
||||
updatedInput := contextargs.GetCopyIfHostOutdated(input, generatedHttpRequest.URL())
|
||||
|
||||
// Full URL, so https and an explicit port stay on the same key the observation uses.
|
||||
if err := request.rateLimitTake(ctx, hostBackoffKey(generatedHttpRequest, input)); err != nil {
|
||||
return false, true, nil, err
|
||||
}
|
||||
|
||||
if generatedHttpRequest.customCancelFunction != nil {
|
||||
defer generatedHttpRequest.customCancelFunction()
|
||||
}
|
||||
@@ -709,6 +714,15 @@ executionLoop:
|
||||
if request.isUnresponsiveAddress(updatedInput) {
|
||||
return false, true, nil, nil
|
||||
}
|
||||
|
||||
// The rate limit token is taken once the final request URL is known, so
|
||||
// host/port rewrites bill the right bucket and skipped (unresponsive)
|
||||
// hosts do not consume tokens. Full URL, so https and an explicit port
|
||||
// stay on the same key the observation uses.
|
||||
if err := request.rateLimitTake(ctx, hostBackoffKey(generatedHttpRequest, input)); err != nil {
|
||||
return false, true, nil, err
|
||||
}
|
||||
|
||||
var gotMatches bool
|
||||
var extractedDynamicValues map[string][]string
|
||||
execReqErr := request.executeRequest(input, generatedHttpRequest, previousEvent, hasInteractMatchers, func(event *output.InternalWrappedEvent) {
|
||||
@@ -1271,6 +1285,13 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
|
||||
}
|
||||
}
|
||||
executingClient = httpclient
|
||||
if request.options.HostRateLimitEnabled() {
|
||||
generatedRequest.request = generatedRequest.request.WithContext(
|
||||
httpclientpool.WithRedirectCallback(generatedRequest.request.Context(), func(ctx context.Context, host string) error {
|
||||
return request.options.RateLimitTakeContext(ctx, host)
|
||||
}),
|
||||
)
|
||||
}
|
||||
}
|
||||
if resp == nil {
|
||||
if errSignature := request.handleSignature(generatedRequest); errSignature != nil {
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -25,11 +26,115 @@ import (
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/output"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hostratelimit"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/ratelimit"
|
||||
"github.com/projectdiscovery/rawhttp/clientpipeline"
|
||||
"github.com/projectdiscovery/retryablehttp-go"
|
||||
)
|
||||
|
||||
func TestHostRateLimiterSpecialExecutionPaths(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
request *Request
|
||||
}{
|
||||
{
|
||||
name: "race",
|
||||
request: &Request{
|
||||
ID: "race-host-rate-limit",
|
||||
Path: []string{"{{BaseURL}}"},
|
||||
Race: true,
|
||||
RaceNumberRequests: 2,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "pipeline",
|
||||
request: &Request{
|
||||
ID: "pipeline-host-rate-limit",
|
||||
Path: []string{"{{BaseURL}}"},
|
||||
Pipeline: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
options := testutils.DefaultOptions
|
||||
options.RateLimitHost = 100
|
||||
options.RateLimitHostDuration = time.Second
|
||||
defer func() {
|
||||
options.RateLimitHost = 0
|
||||
options.RateLimitHostDuration = 0
|
||||
}()
|
||||
testutils.Init(options)
|
||||
defer testutils.Cleanup(options)
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte("ok"))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
|
||||
ID: tt.request.ID,
|
||||
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
|
||||
})
|
||||
|
||||
require.NoError(t, tt.request.Compile(executerOpts))
|
||||
ctxArgs := contextargs.NewWithInput(context.Background(), server.URL)
|
||||
require.NoError(t, tt.request.ExecuteWithResults(ctxArgs, output.InternalEvent{}, output.InternalEvent{}, func(*output.InternalWrappedEvent) {}))
|
||||
|
||||
dialers := protocolstate.GetDialersWithId(options.ExecutionId)
|
||||
require.NotNil(t, dialers)
|
||||
dialers.Lock()
|
||||
pool, ok := dialers.PerHostRateLimitPool.(*hostratelimit.PerHostRateLimitPool)
|
||||
dialers.Unlock()
|
||||
require.True(t, ok, "the race and pipeline paths must take from the per-host limiter")
|
||||
require.Equal(t, 1, pool.Size())
|
||||
|
||||
if tt.request.Pipeline {
|
||||
awaitPipelineClientShutdown(t)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// awaitPipelineClientShutdown blocks until the goroutines of every rawhttp
|
||||
// pipeline client have exited.
|
||||
//
|
||||
// rawhttp exposes no way to release a pipeline client: the worker, reader and
|
||||
// writer goroutines it starts per connection own the connection and only retire
|
||||
// once clientpipeline's idle timer fires, which is far beyond goleak's retry
|
||||
// window. Any test that drives the pipeline execution path must therefore wait
|
||||
// here, otherwise those goroutines outlive the test and are reported as leaks by
|
||||
// whichever later test runs a goleak check.
|
||||
func awaitPipelineClientShutdown(t *testing.T) {
|
||||
t.Helper()
|
||||
|
||||
deadline := time.Now().Add(clientpipeline.DefaultMaxIdleConnDuration + 30*time.Second)
|
||||
for {
|
||||
stacks := goroutineStacks()
|
||||
if !strings.Contains(stacks, "rawhttp/clientpipeline.(*pipelineConnClient)") {
|
||||
return
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("rawhttp pipeline goroutines are still running after the idle timeout:\n%s", stacks)
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func goroutineStacks() string {
|
||||
buf := make([]byte, 64*1024)
|
||||
for {
|
||||
n := runtime.Stack(buf, true)
|
||||
if n < len(buf) {
|
||||
return string(buf[:n])
|
||||
}
|
||||
buf = make([]byte, 2*len(buf))
|
||||
}
|
||||
}
|
||||
|
||||
func executeDynamicHTTPTestRequest(t *testing.T, request *Request, target string) {
|
||||
t.Helper()
|
||||
options := testutils.DefaultOptions
|
||||
@@ -1394,9 +1499,11 @@ func TestExecuteParallelHTTP_SkipOnUnresponsiveFromCache(t *testing.T) {
|
||||
require.Equal(t, int32(0), atomic.LoadInt32(&matches), "expected no matches when host is marked unresponsive")
|
||||
}
|
||||
|
||||
// TestExecuteParallelHTTP_GoroutineLeaks uses goleak to detect goroutine leaks in all HTTP parallel execution scenarios
|
||||
func TestExecuteParallelHTTP_GoroutineLeaks(t *testing.T) {
|
||||
defer goleak.VerifyNone(t,
|
||||
// goroutineLeakOptions are the goleak exclusions shared by the leak tests of
|
||||
// this package, all of them background workers owned by dependencies rather
|
||||
// than by the code under test.
|
||||
func goroutineLeakOptions() []goleak.Option {
|
||||
return []goleak.Option{
|
||||
goleak.IgnoreAnyContainingPkg("go.opencensus.io/stats/view"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/syndtr/goleveldb"),
|
||||
goleak.IgnoreAnyContainingPkg("github.com/go-rod/rod"),
|
||||
@@ -1417,7 +1524,12 @@ func TestExecuteParallelHTTP_GoroutineLeaks(t *testing.T) {
|
||||
// (e.g. windows) that teardown can outlast goleak's retry window, so ignore them.
|
||||
goleak.IgnoreAnyFunction("net/http.(*persistConn).writeLoop"),
|
||||
goleak.IgnoreAnyFunction("net/http.(*persistConn).readLoop"),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// TestExecuteParallelHTTP_GoroutineLeaks uses goleak to detect goroutine leaks in all HTTP parallel execution scenarios
|
||||
func TestExecuteParallelHTTP_GoroutineLeaks(t *testing.T) {
|
||||
defer goleak.VerifyNone(t, goroutineLeakOptions()...)
|
||||
|
||||
options := testutils.DefaultOptions
|
||||
testutils.Init(options)
|
||||
@@ -1556,6 +1668,79 @@ func TestExecuteParallelHTTP_GoroutineLeaks(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// TestExecuteTurboHTTP_GoroutineLeaks locks the pipeline execution path: every
|
||||
// pipelined request must reach the server over a keep-alive connection, and the
|
||||
// pipeline client created for the execution must not leave goroutines behind
|
||||
// once it goes idle. A request that asks the server to close the connection
|
||||
// breaks both: the rest of the pipeline fails and the client's worker is left
|
||||
// waiting on work that never arrives.
|
||||
func TestExecuteTurboHTTP_GoroutineLeaks(t *testing.T) {
|
||||
defer goleak.VerifyNone(t, goroutineLeakOptions()...)
|
||||
|
||||
options := testutils.DefaultOptions
|
||||
testutils.Init(options)
|
||||
defer testutils.Cleanup(options)
|
||||
|
||||
var mu sync.Mutex
|
||||
var served int
|
||||
var closeRequested bool
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
mu.Lock()
|
||||
served++
|
||||
closeRequested = closeRequested || r.Close
|
||||
mu.Unlock()
|
||||
_, _ = fmt.Fprint(w, "pipelined response")
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
req := &Request{
|
||||
ID: "pipeline-goroutine-leaks",
|
||||
Raw: []string{
|
||||
`GET /test?param={{payload}} HTTP/1.1
|
||||
Host: {{Hostname}}
|
||||
Accept: */*
|
||||
`,
|
||||
},
|
||||
Pipeline: true,
|
||||
PipelineConcurrentConnections: 2,
|
||||
PipelineRequestsPerConnection: 10,
|
||||
Payloads: map[string]interface{}{
|
||||
"payload": []string{"1", "2", "3", "4"},
|
||||
},
|
||||
Operators: operators.Operators{
|
||||
Matchers: []*matchers.Matcher{{
|
||||
Part: "body",
|
||||
Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
|
||||
Words: []string{"pipelined response"},
|
||||
}},
|
||||
},
|
||||
}
|
||||
|
||||
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
|
||||
ID: req.ID,
|
||||
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
|
||||
})
|
||||
require.NoError(t, req.Compile(executerOpts))
|
||||
|
||||
var matches int32
|
||||
ctxArgs := contextargs.NewWithInput(context.Background(), ts.URL)
|
||||
err := req.ExecuteWithResults(ctxArgs, output.InternalEvent{}, output.InternalEvent{}, func(event *output.InternalWrappedEvent) {
|
||||
if event.OperatorsResult != nil && event.OperatorsResult.Matched {
|
||||
atomic.AddInt32(&matches, 1)
|
||||
}
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, int32(4), atomic.LoadInt32(&matches), "expected every pipelined request to match")
|
||||
|
||||
mu.Lock()
|
||||
servedRequests, sawClose := served, closeRequested
|
||||
mu.Unlock()
|
||||
require.Equal(t, 4, servedRequests, "expected every pipelined request to reach the server")
|
||||
require.False(t, sawClose, "pipelined requests must not ask the server to close the connection")
|
||||
|
||||
awaitPipelineClientShutdown(t)
|
||||
}
|
||||
|
||||
func TestOutputIP(t *testing.T) {
|
||||
history := func(entries map[string]string) func(string) string {
|
||||
return func(hostname string) string { return entries[hostname] }
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/projectdiscovery/fastdialer/fastdialer"
|
||||
@@ -214,18 +215,28 @@ func (e *ExecutorOptions) RateLimitTake(host string) error {
|
||||
}
|
||||
|
||||
// RateLimitTakeContext waits for admission or returns when ctx is canceled.
|
||||
//
|
||||
// With -per-host-rate-limit or -rate-limit-host, the limiter of the host is the
|
||||
// only limiter consulted for a known host. -rate-limit-host sets a per-host
|
||||
// budget separate from -rate-limit: -rate-limit defaults to 150 rps, so
|
||||
// applying both would silently cap the aggregate throughput at the global
|
||||
// default and defeat the per-host budget. The aggregate is bounded by
|
||||
// num_hosts * rate-limit-host. An empty host falls back to the global limiter
|
||||
// so self-contained requests are still paced.
|
||||
func (e *ExecutorOptions) RateLimitTakeContext(ctx context.Context, host string) error {
|
||||
if e.Options != nil && e.Options.PerHostRateLimit && host != "" {
|
||||
limiter, err := hostratelimit.GetPerHostRateLimiter(e.Options, host)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if limiter != nil {
|
||||
if err := limiter.Wait(ctx); err != nil {
|
||||
if e.Options != nil && e.Options.PerHostRateLimitEnabled() && host != "" {
|
||||
if key := HostRateLimitKey(host); key != "" {
|
||||
limiter, err := hostratelimit.GetPerHostRateLimiter(e.Options, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hostratelimit.RecordPerHostRateLimitRequest(e.Options, host)
|
||||
return nil
|
||||
if limiter != nil {
|
||||
if err := limiter.Wait(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
hostratelimit.RecordPerHostRateLimitRequest(e.Options, key)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
// The code below can race and there isn't a great way to fix this without adding an idempotent
|
||||
@@ -242,6 +253,20 @@ func (e *ExecutorOptions) RateLimitTakeContext(ctx context.Context, host string)
|
||||
return nil
|
||||
}
|
||||
|
||||
// HostRateLimitEnabled reports whether -rate-limit-host is set. The race and
|
||||
// pipeline paths and redirect hops are paced only by this limiter.
|
||||
func (e *ExecutorOptions) HostRateLimitEnabled() bool {
|
||||
return e.Options != nil && e.Options.RateLimitHost > 0
|
||||
}
|
||||
|
||||
// HostRateLimitKey returns the per-host bucket for a URL or host[:port]:
|
||||
// the lowercase host with an explicit or scheme default port, so different
|
||||
// ports on one hostname stay separate buckets and a redirect to the same
|
||||
// origin shares the bucket of the original request.
|
||||
func HostRateLimitKey(host string) string {
|
||||
return strings.ToLower(hostratelimit.NormalizeHostPort(host))
|
||||
}
|
||||
|
||||
// HostBackoffWait pauses for the delay address has earned with HostBackoff.
|
||||
func (e *ExecutorOptions) HostBackoffWait(ctx context.Context, address string) error {
|
||||
return e.HostBackoff.Wait(ctx, hostratelimit.NormalizeHostPort(address))
|
||||
|
||||
@@ -2,10 +2,13 @@ package protocols_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"runtime"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hostratelimit"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
||||
"github.com/projectdiscovery/nuclei/v3/pkg/utils"
|
||||
@@ -81,3 +84,103 @@ func TestRateLimitTakeContextCancelsGlobalWait(t *testing.T) {
|
||||
cancel()
|
||||
require.ErrorIs(t, <-result, context.Canceled)
|
||||
}
|
||||
|
||||
func TestHostRateLimitKey(t *testing.T) {
|
||||
require.Equal(t, "example.com:8080", protocols.HostRateLimitKey("http://EXAMPLE.COM:8080/path"))
|
||||
require.Equal(t, "example.com:8080", protocols.HostRateLimitKey("example.com:8080"))
|
||||
require.Equal(t, "example.com:443", protocols.HostRateLimitKey("https://Example.com/a"))
|
||||
require.Equal(t, "example.com:80", protocols.HostRateLimitKey("http://example.com"))
|
||||
}
|
||||
|
||||
// hostRateLimitExecutor returns an executor with -rate-limit-host set to limit
|
||||
// requests per duration, and a global limiter that would block for an hour.
|
||||
func hostRateLimitExecutor(t *testing.T, limit int, duration time.Duration) *protocols.ExecutorOptions {
|
||||
t.Helper()
|
||||
options := types.DefaultOptions()
|
||||
options.SetExecutionID(t.Name())
|
||||
options.RateLimit = 1
|
||||
options.RateLimitDuration = time.Hour
|
||||
options.RateLimitHost = limit
|
||||
options.RateLimitHostDuration = duration
|
||||
require.NoError(t, protocolstate.Init(options))
|
||||
t.Cleanup(func() { protocolstate.Close(options.ExecutionId) })
|
||||
|
||||
global := utils.GetRateLimiter(context.Background(), options.RateLimit, options.RateLimitDuration)
|
||||
t.Cleanup(global.Stop)
|
||||
// spend the only global token, so any use of the global limiter blocks
|
||||
require.NoError(t, global.TakeContext(context.Background()))
|
||||
return &protocols.ExecutorOptions{Options: options, RateLimiter: global}
|
||||
}
|
||||
|
||||
func perHostPoolSize(t *testing.T, options *types.Options) int {
|
||||
t.Helper()
|
||||
dialers := protocolstate.GetDialersWithId(options.ExecutionId)
|
||||
require.NotNil(t, dialers)
|
||||
dialers.Lock()
|
||||
defer dialers.Unlock()
|
||||
pool, ok := dialers.PerHostRateLimitPool.(*hostratelimit.PerHostRateLimitPool)
|
||||
require.True(t, ok)
|
||||
return pool.Size()
|
||||
}
|
||||
|
||||
func TestRateLimitTakeHostRateLimitTakesPriority(t *testing.T) {
|
||||
executor := hostRateLimitExecutor(t, 1, 300*time.Millisecond)
|
||||
|
||||
start := time.Now()
|
||||
for range 3 {
|
||||
require.NoError(t, executor.RateLimitTake("https://example.com/path"))
|
||||
}
|
||||
elapsed := time.Since(start)
|
||||
require.GreaterOrEqual(t, elapsed, 500*time.Millisecond, "one host must be limited")
|
||||
require.Less(t, elapsed, 5*time.Second, "the global limiter must not be consulted")
|
||||
require.Equal(t, 1, perHostPoolSize(t, executor.Options))
|
||||
|
||||
// a redirect to the same origin shares the bucket of the original request
|
||||
require.NoError(t, executor.RateLimitTake("EXAMPLE.com:443"))
|
||||
require.Equal(t, 1, perHostPoolSize(t, executor.Options))
|
||||
|
||||
// other hosts have their own budget
|
||||
start = time.Now()
|
||||
for _, host := range []string{"127.0.0.1:8001", "127.0.0.1:8002", "http://example.org/path"} {
|
||||
require.NoError(t, executor.RateLimitTake(host))
|
||||
}
|
||||
require.Less(t, time.Since(start), 250*time.Millisecond, "different hosts must not share a limiter")
|
||||
}
|
||||
|
||||
func TestRateLimitTakeHostRateLimitCancelKeepsToken(t *testing.T) {
|
||||
executor := hostRateLimitExecutor(t, 1, time.Second)
|
||||
host := "https://cancel.example"
|
||||
|
||||
start := time.Now()
|
||||
require.NoError(t, executor.RateLimitTakeContext(context.Background(), host))
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
result := make(chan error, 1)
|
||||
go func() { result <- executor.RateLimitTakeContext(ctx, host) }()
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
cancel()
|
||||
select {
|
||||
case err := <-result:
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
t.Fatal("a canceled wait must return at once")
|
||||
}
|
||||
|
||||
// the canceled wait gave its token back: the next request is admitted one
|
||||
// interval after the first, not two
|
||||
require.NoError(t, executor.RateLimitTakeContext(context.Background(), host))
|
||||
require.Less(t, time.Since(start), 1700*time.Millisecond)
|
||||
}
|
||||
|
||||
func TestRateLimitTakeHostRateLimitStartsNoGoroutines(t *testing.T) {
|
||||
executor := hostRateLimitExecutor(t, 5, time.Second)
|
||||
// the first call creates the pool
|
||||
require.NoError(t, executor.RateLimitTake("https://warmup.example"))
|
||||
|
||||
before := runtime.NumGoroutine()
|
||||
for i := range 200 {
|
||||
require.NoError(t, executor.RateLimitTake(fmt.Sprintf("https://host-%d.example", i)))
|
||||
}
|
||||
require.Equal(t, 201, perHostPoolSize(t, executor.Options))
|
||||
require.Less(t, runtime.NumGoroutine()-before, 10, "per-host limiters must not start goroutines")
|
||||
}
|
||||
@@ -156,6 +156,11 @@ type Options struct {
|
||||
// Rate-Limit is the maximum number of requests per minute for specified target
|
||||
// Deprecated: Use RateLimitDuration - automatically set Rate Limit Duration to 60 seconds
|
||||
RateLimitMinute int
|
||||
// RateLimitHost is the maximum number of requests per RateLimitHostDuration
|
||||
// per host. 0 disables the per-host limiter (only the global RateLimit applies).
|
||||
RateLimitHost int
|
||||
// RateLimitHostDuration is the refill interval for the per-host bucket.
|
||||
RateLimitHostDuration time.Duration
|
||||
// PageTimeout is the maximum time to wait for a page in seconds
|
||||
PageTimeout int
|
||||
// InteractionsCacheSize is the number of interaction-url->req to keep in cache at a time.
|
||||
@@ -557,6 +562,13 @@ type Options struct {
|
||||
templateThreadRelease func()
|
||||
}
|
||||
|
||||
// PerHostRateLimitEnabled reports whether requests are paced by a limiter per
|
||||
// host: with -per-host-rate-limit, or with -rate-limit-host, which also sets
|
||||
// a per-host budget separate from the global rate limit.
|
||||
func (options *Options) PerHostRateLimitEnabled() bool {
|
||||
return options.PerHostRateLimit || options.RateLimitHost > 0
|
||||
}
|
||||
|
||||
// SetTemplateThreadsProvider configures a dynamic template concurrency source.
|
||||
// The provider is consulted while an execution schedules templates, allowing an
|
||||
// embedding application to redistribute a fixed concurrency budget as peer
|
||||
@@ -669,6 +681,8 @@ func (options *Options) Copy() *Options {
|
||||
RateLimit: options.RateLimit,
|
||||
RateLimitDuration: options.RateLimitDuration,
|
||||
RateLimitMinute: options.RateLimitMinute,
|
||||
RateLimitHost: options.RateLimitHost,
|
||||
RateLimitHostDuration: options.RateLimitHostDuration,
|
||||
PageTimeout: options.PageTimeout,
|
||||
InteractionsCacheSize: options.InteractionsCacheSize,
|
||||
InteractionsPollDuration: options.InteractionsPollDuration,
|
||||
@@ -934,6 +948,7 @@ func DefaultOptions() *Options {
|
||||
return &Options{
|
||||
RateLimit: 150,
|
||||
RateLimitDuration: time.Second,
|
||||
RateLimitHostDuration: time.Second,
|
||||
BulkSize: 25,
|
||||
TemplateThreads: 25,
|
||||
HeadlessBulkSize: 10,
|
||||
|
||||
Reference in new issue
Block a user