Compare commits

..
Author SHA1 Message Date
Nakul Bharti d4800f81f8 fix(flow): bound runtime execution 2026-10-13 22:14:20 +05:30
54 changed files with 409 additions and 3663 deletions

No files matched your search

-3
View File
@@ -294,8 +294,6 @@ UNCOVER:
RATE-LIMIT:
-rl, -rate-limit int maximum number of requests to send per second (default 150)
-rld, -rate-limit-duration value maximum number of requests to send per second (default 1s)
-host-backoff slow down a host that starts blocking (429, repeated 403s, connection failures)
-host-backoff-max value maximum delay applied to a host that is blocking (default 30s)
-per-host-rate-limit enable per-host rate limiting (global rate limit becomes unlimited when enabled)
-rlm, -rate-limit-minute int maximum number of requests to send per minute (DEPRECATED)
-bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25)
@@ -339,7 +337,6 @@ DEBUG:
-dresp, -debug-resp show all received responses
-p, -proxy string[] list of http/socks5 proxy to use (comma separated or file input)
-pi, -proxy-internal proxy all internal requests
-rp, -replay-proxy string http/socks5 proxy to re-send requests that produced a finding through (e.g. burp)
-ldf, -list-dsl-function list all supported DSL function signatures
-tlog, -trace-log string file to write sent requests trace log
-elog, -error-log string file to write sent requests error log
-3
View File
@@ -432,8 +432,6 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.CreateGroup("rate-limit", "Rate-Limit",
flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "maximum number of requests to send per second"),
flagSet.DurationVarP(&options.RateLimitDuration, "rate-limit-duration", "rld", time.Second, "maximum number of requests to send per second"),
flagSet.BoolVar(&options.HostBackoff, "host-backoff", false, "slow down a host that starts blocking (429, repeated 403s, connection failures)"),
flagSet.DurationVar(&options.HostBackoffMax, "host-backoff-max", 30*time.Second, "maximum delay applied to a host that is blocking"),
flagSet.BoolVar(&options.PerHostRateLimit, "per-host-rate-limit", false, "enable per-host rate limiting (global rate limit becomes unlimited when enabled)"),
flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute (DEPRECATED)"),
flagSet.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "maximum number of hosts to be analyzed in parallel per template"),
@@ -484,7 +482,6 @@ on extensive configurability, massive extensibility and ease of use.`)
flagSet.BoolVarP(&options.DebugResponse, "debug-resp", "dresp", false, "show all received responses"),
flagSet.StringSliceVarP(&options.Proxy, "proxy", "p", nil, "list of http/socks5 proxy to use (comma separated or file input)", goflags.FileCommaSeparatedStringSliceOptions),
flagSet.BoolVarP(&options.ProxyInternal, "proxy-internal", "pi", false, "proxy all internal requests"),
flagSet.StringVarP(&options.ReplayProxy, "replay-proxy", "rp", "", "http/socks5 proxy to re-send requests that produced a finding through (e.g. burp)"),
flagSet.BoolVarP(&options.ListDslSignatures, "list-dsl-function", "ldf", false, "list all supported DSL function signatures"),
flagSet.StringVarP(&options.TraceLogFile, "trace-log", "tlog", "", "file to write sent requests trace log"),
flagSet.StringVarP(&options.ErrorLogFile, "error-log", "elog", "", "file to write sent requests error log"),
+20 -19
View File
@@ -76,7 +76,7 @@ require (
github.com/praetorian-inc/fingerprintx v1.1.15
github.com/projectdiscovery/clistats v0.1.7
github.com/projectdiscovery/dsl v0.8.24
github.com/projectdiscovery/fastdialer v0.5.26
github.com/projectdiscovery/fastdialer v0.5.25
github.com/projectdiscovery/fasttemplate v0.0.2
github.com/projectdiscovery/gcache v0.0.0-20241015120333-12546c6e3f4c
github.com/projectdiscovery/goflags v0.2.1
@@ -92,18 +92,18 @@ require (
github.com/projectdiscovery/interactsh v1.4.1
github.com/projectdiscovery/mapcidr v1.1.97
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5
github.com/projectdiscovery/networkpolicy v0.1.55
github.com/projectdiscovery/ratelimit v0.0.94
github.com/projectdiscovery/rawhttp v0.2.0
github.com/projectdiscovery/networkpolicy v0.1.54
github.com/projectdiscovery/ratelimit v0.0.93
github.com/projectdiscovery/rawhttp v0.1.92
github.com/projectdiscovery/rdap v0.9.0
github.com/projectdiscovery/retryabledns v1.0.116
github.com/projectdiscovery/retryablehttp-go v1.3.31
github.com/projectdiscovery/retryablehttp-go v1.3.30
github.com/projectdiscovery/sarif v0.1.0
github.com/projectdiscovery/tlsx v1.4.3-0.20261020052910-acb80c2c7c75
github.com/projectdiscovery/tlsx v1.4.1
github.com/projectdiscovery/uncover v1.2.1
github.com/projectdiscovery/useragent v0.0.109
github.com/projectdiscovery/utils v0.11.11-0.20261020211236-cde16c1381ec
github.com/projectdiscovery/wappalyzergo v0.3.5
github.com/projectdiscovery/utils v0.11.9
github.com/projectdiscovery/wappalyzergo v0.3.4
github.com/projectdiscovery/yamldoc-go v1.0.7
github.com/redis/go-redis/v9 v9.19.0
github.com/remeh/sizedwaitgroup v1.0.0
@@ -124,12 +124,12 @@ require (
go.mongodb.org/mongo-driver v1.17.9
go.uber.org/multierr v1.11.0
golang.org/x/exp v0.0.0-20260527015227-08cc5374adb3
golang.org/x/net v0.59.0
golang.org/x/net v0.58.0
golang.org/x/oauth2 v0.36.0
golang.org/x/sync v0.23.0
golang.org/x/sys v0.48.0
golang.org/x/term v0.46.0
golang.org/x/text v0.42.0
golang.org/x/sync v0.22.0
golang.org/x/sys v0.47.0
golang.org/x/term v0.45.0
golang.org/x/text v0.41.0
golang.org/x/time v0.15.0
golang.org/x/tools v0.49.0
google.golang.org/grpc v1.83.2
@@ -375,7 +375,7 @@ require (
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
github.com/projectdiscovery/asnmap v1.1.1 // indirect
github.com/projectdiscovery/blackrock v0.0.2 // indirect
github.com/projectdiscovery/cdncheck v1.3.2 // indirect
github.com/projectdiscovery/cdncheck v1.3.1 // indirect
github.com/projectdiscovery/freeport v0.0.7 // indirect
github.com/projectdiscovery/jarm-go v0.0.0-20260910160638-430e2bae86f6 // indirect
github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb // indirect
@@ -389,7 +389,7 @@ require (
github.com/sergi/go-diff v1.4.0 // indirect
github.com/shirou/gopsutil v3.21.11+incompatible // indirect
github.com/shirou/gopsutil/v4 v4.26.3 // indirect
github.com/sirupsen/logrus v1.10.2 // indirect
github.com/sirupsen/logrus v1.9.4 // indirect
github.com/skeema/knownhosts v1.3.1 // indirect
github.com/sorairolake/lzip-go v0.3.8 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
@@ -418,7 +418,7 @@ require (
github.com/vmihailenco/tagparser v0.1.2 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/vulncheck-oss/go-exploit v1.51.0 // indirect
github.com/weppos/publicsuffix-go v0.50.4-0.20260918082853-3c3d7dc4a0cb // indirect
github.com/weppos/publicsuffix-go v0.50.3 // indirect
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
@@ -439,7 +439,8 @@ require (
github.com/yusufpapurcu/wmi v1.2.4 // indirect
github.com/zcalusic/sysinfo v1.1.3 // indirect
github.com/zeebo/blake3 v0.2.4 // indirect
github.com/zmap/zcrypto v0.0.0-20261015050653-aa1de3126e87 // indirect
github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect
github.com/zmap/zcrypto v0.0.0-20240803002437-3a861682ac77 // indirect
go.etcd.io/bbolt v1.4.3 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect
@@ -457,8 +458,8 @@ require (
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
goftp.io/server/v2 v2.0.1 // indirect
golang.org/x/arch v0.3.0 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/mod v0.41.0 // indirect
golang.org/x/crypto v0.55.0 // indirect
golang.org/x/mod v0.39.0 // indirect
google.golang.org/api v0.264.0 // indirect
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect
+53 -38
View File
@@ -130,6 +130,7 @@ github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEV
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk=
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g=
github.com/ProtonMail/go-crypto v1.1.6 h1:ZcV+Ropw6Qn0AX9brlQLAUXfqLBc7Bl+f/DmNxpLfdw=
github.com/ProtonMail/go-crypto v1.1.6/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE=
github.com/PuerkitoBio/goquery v1.13.0 h1:mqHbjD7Jmnul4DTR24LKTjo1uUmHUh072kteGV+xpFM=
@@ -266,6 +267,7 @@ github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/
github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU5DoEw9xY/FUi1c=
github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk=
github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M=
github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM=
github.com/bytedance/sonic v1.15.3 h1:P3akjLPBtV/i6bHC6LbcLjY3KuoOvfiqF8wFHeP5IhY=
@@ -323,6 +325,7 @@ github.com/clbanning/mxj/v2 v2.7.0/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cloudflare/cfssl v1.6.5 h1:46zpNkm6dlNkMZH/wMW22ejih6gIaJbzL2du6vD7ZeI=
github.com/cloudflare/cfssl v1.6.5/go.mod h1:Bk1si7sq8h2+yVEDrFJiz3d7Aw+pfjjJSZVaD+Taky4=
github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I=
github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8=
github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4=
github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M=
@@ -589,6 +592,7 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
github.com/google/go-github/v50 v50.2.0/go.mod h1:VBY8FB6yPIjrtKhozXv4FQupxKLS6H4m6xFZlT43q8Q=
github.com/google/go-github/v92 v92.0.0 h1:4vW4RVffwIvoEfIA4RX09mRSB47qQX557/2+HgeWRRg=
github.com/google/go-github/v92 v92.0.0/go.mod h1:w3CH62ZcmRfvW1cdXpyTztSOVMtdjxtKpgo0GouLmjY=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
@@ -931,14 +935,14 @@ github.com/projectdiscovery/asnmap v1.1.1 h1:ImJiKIaACOT7HPx4Pabb5dksolzaFYsD1kI
github.com/projectdiscovery/asnmap v1.1.1/go.mod h1:QT7jt9nQanj+Ucjr9BqGr1Q2veCCKSAVyUzLXfEcQ60=
github.com/projectdiscovery/blackrock v0.0.2 h1:mxXdu0uM8P2L2Qi210COlU8QiICPFW/Rxk5QUhlPO2k=
github.com/projectdiscovery/blackrock v0.0.2/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss=
github.com/projectdiscovery/cdncheck v1.3.2 h1:I453ZWNBeG0WWIy3wi9412l4PTzorLgebEnqmbcpSfo=
github.com/projectdiscovery/cdncheck v1.3.2/go.mod h1:5IkUMMlcGaeg1DWW5oJVd3r9X3EpJ88j5HxBeF4Me9M=
github.com/projectdiscovery/cdncheck v1.3.1 h1:+qPKwqdf3mFtrw3ZoVti7hembmVaapnJBjRosYTNdRo=
github.com/projectdiscovery/cdncheck v1.3.1/go.mod h1:5IkUMMlcGaeg1DWW5oJVd3r9X3EpJ88j5HxBeF4Me9M=
github.com/projectdiscovery/clistats v0.1.7 h1:NXf4lUzx1QcbJd6QycNwwYHTv2FsW9MchQ2LsgSL6C0=
github.com/projectdiscovery/clistats v0.1.7/go.mod h1:RxRGxhMzkT344VTTvZWIrojurUXdQQRN1yfwgg/88I8=
github.com/projectdiscovery/dsl v0.8.24 h1:rN2w6LSIjA16qKGRjyj70oxdwXGaspe6sAjEkP7F2fk=
github.com/projectdiscovery/dsl v0.8.24/go.mod h1:V+EludvS5PS19UNMlM3CuBwZ+sFnMaQhCFGxysDUanQ=
github.com/projectdiscovery/fastdialer v0.5.26 h1:wCCsopLsO1EF1V1+9Wqkxj1bMm6sRo8NdlzwE2Ib2Ss=
github.com/projectdiscovery/fastdialer v0.5.26/go.mod h1:dZzEPnpsxqCiD3IkM5CN3RglW3l1gXJNyTkxqjPZIbs=
github.com/projectdiscovery/fastdialer v0.5.25 h1:xGFpc5B7cmogqnBrcVYV5yXh6UFlqRA6u5KqautKgD0=
github.com/projectdiscovery/fastdialer v0.5.25/go.mod h1:CZZ0PwdTw2KCI/PrZTW9wyBY88MIjpT8WhsKWSw3bkw=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
github.com/projectdiscovery/freeport v0.0.7 h1:Q6uXo/j8SaV/GlAHkEYQi8WQoPXyJWxyspx+aFmz9Qk=
@@ -977,32 +981,32 @@ github.com/projectdiscovery/mapcidr v1.1.97 h1:7FkxNNVXp+m1rIu5Nv/2SrF9k4+LwP8Qu
github.com/projectdiscovery/mapcidr v1.1.97/go.mod h1:9dgTJh1SP02gYZdpzMjm6vtYFkEHQHoTyaVNvaeJ7lA=
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw1pfT6bg35NiN7yd1XKtJap5Nk6lMwQ0RNi8=
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc=
github.com/projectdiscovery/networkpolicy v0.1.55 h1:D30pipxOdWnSq3r7sX0kQvGyb18sUgopuda1j9NAmnQ=
github.com/projectdiscovery/networkpolicy v0.1.55/go.mod h1:N5bXm8L3UsDw51CeA1nhTsGEbQMbgL5lXTiIU013xd0=
github.com/projectdiscovery/ratelimit v0.0.94 h1:2qciSuUxarn73YGICZeprZtjrwyAH+0iVpdKzKCXoXE=
github.com/projectdiscovery/ratelimit v0.0.94/go.mod h1:GAerX6s+eWRuACedHRv2/xmMtlW+VahZmbRstQh0sS0=
github.com/projectdiscovery/rawhttp v0.2.0 h1:TGiHC5hydL97/zoi0WjEuNrfO6Gjtg9LeXndUf1Cq3M=
github.com/projectdiscovery/rawhttp v0.2.0/go.mod h1:Waz+K5y6r44ElpqAoOfYtY03zrGGZ12nx0yDKq7Ot1E=
github.com/projectdiscovery/networkpolicy v0.1.54 h1:+v7yyuRBu2S7dzeIJToDENy1i+HjRhNzqnQrDlm00z8=
github.com/projectdiscovery/networkpolicy v0.1.54/go.mod h1:tN+YAFVlJjHv/znPEhkcSAjmBMkMhslC9XpWKTrKEHw=
github.com/projectdiscovery/ratelimit v0.0.93 h1:k2rRpCR31DqeqrORSgGzJ+z0T7+QGCt0Z0+Ov/TVrgk=
github.com/projectdiscovery/ratelimit v0.0.93/go.mod h1:Tqpx1QKdFI63L01T2gpSvV5Uro9CnZkZWr80ciSXSBY=
github.com/projectdiscovery/rawhttp v0.1.92 h1:38GRsVLNllES/dZcEcA6s6P+rYBp34H9mDphMJMDvvg=
github.com/projectdiscovery/rawhttp v0.1.92/go.mod h1:spfXTPjxCj+bfWLynMVB7uOb+LS3PDNAI9HexMEV1Zg=
github.com/projectdiscovery/rdap v0.9.0 h1:wPhHx5pQ2QI+WGhyNb2PjhTl0NtB39Nk7YFZ9cp8ZGA=
github.com/projectdiscovery/rdap v0.9.0/go.mod h1:zk4yrJFQ2Hy36Aqk+DvotYQxYAeALaCJ5ORySkff36Q=
github.com/projectdiscovery/retryabledns v1.0.116 h1:tK28PJ4tLO7bgM89RpIFjkfTPMSbRegwvg9mN/q6z78=
github.com/projectdiscovery/retryabledns v1.0.116/go.mod h1:r3vH0AKYtJ9fculcqRFrdVYe2ABcHwZeR4C6j3rqCbY=
github.com/projectdiscovery/retryablehttp-go v1.3.31 h1:0hSEZ5+jj6Oo1IL8qrytD36TK/qCxfe543jEaRqSTjs=
github.com/projectdiscovery/retryablehttp-go v1.3.31/go.mod h1:/WEa7YejxTCXEMLCGfGYaC3JsgXgzuqw8rv8cb4buEc=
github.com/projectdiscovery/retryablehttp-go v1.3.30 h1:YnDCZP04/AKTiIBT5vA1nVhy5cRW5dPy1Du5nCjxHy4=
github.com/projectdiscovery/retryablehttp-go v1.3.30/go.mod h1:MXn1MzThNmHc05KldUbfks1MvwAnUKR4GGJcFhVIc08=
github.com/projectdiscovery/sarif v0.1.0 h1:O541T+a448nSJsmIMnXXSOeDQEzpnCAYvRfe0eG5h74=
github.com/projectdiscovery/sarif v0.1.0/go.mod h1:LBC+reM3bkI3qIIhE0rZaINaYX6VG+En6u2hHa5mA7E=
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0=
github.com/projectdiscovery/tlsx v1.4.3-0.20261020052910-acb80c2c7c75 h1:3qJgfWzzwtFXc/vdUIurSzPzyjBH9NQe0wATkZMoQGI=
github.com/projectdiscovery/tlsx v1.4.3-0.20261020052910-acb80c2c7c75/go.mod h1:8jqB9aHT6U8NkUsENvaSjD/iBEGB4Ta+CbJBMcvzP9c=
github.com/projectdiscovery/tlsx v1.4.1 h1:QuECHVJERBirNrmpjgIOAFQScjy/DxVjzXeL+L81mH0=
github.com/projectdiscovery/tlsx v1.4.1/go.mod h1:WCrUXAHjdKbDNjsu5EbVBS7qvzap3fxi/E85v/Flyx8=
github.com/projectdiscovery/uncover v1.2.1 h1:8U46T/96CLT7BPoXBgkTvWqB06lOyeTSLvh5+UjzATE=
github.com/projectdiscovery/uncover v1.2.1/go.mod h1:0p8onrWxfpXQEYs90ZDzTSpu1107gWmodX1NWqu/+z4=
github.com/projectdiscovery/useragent v0.0.109 h1:b86BPdZvmgbJousa04jAugDRY4Y+rAvEamE0TFCPmeE=
github.com/projectdiscovery/useragent v0.0.109/go.mod h1:sRyBqDAcD31RKYrO8fyXGowLd36fNwY1g0MNW9mp7zk=
github.com/projectdiscovery/utils v0.11.11-0.20261020211236-cde16c1381ec h1:VhJiivJR8ljv5Qk3QZQC9og50wspEBB5qkNfdyqNhws=
github.com/projectdiscovery/utils v0.11.11-0.20261020211236-cde16c1381ec/go.mod h1:T7KzyElivWJiPP+B4sXqkp1CcFe6O/P0qBn/MjDNa74=
github.com/projectdiscovery/wappalyzergo v0.3.5 h1:UZradYF1GIcruSIV5wwo/Coph9/VnM17oajLD9eXR2U=
github.com/projectdiscovery/wappalyzergo v0.3.5/go.mod h1:4E6+fh/ckaY3wo5tlM7lNYSkhuTYRHzu2YmGIRraXP0=
github.com/projectdiscovery/utils v0.11.9 h1:aKLje3zPcZuOGmopN51PasWxPX/CypPwzP2ke/Qa2bA=
github.com/projectdiscovery/utils v0.11.9/go.mod h1:d3BcbsHhuxW9T3jpvrgzwshvqhaL9qPSkoi+3h/DY8E=
github.com/projectdiscovery/wappalyzergo v0.3.4 h1:yO+xel+C0d/vOFWcUKVZRpIDAfcEmsjpVDjQ0jifAUw=
github.com/projectdiscovery/wappalyzergo v0.3.4/go.mod h1:6xTRCUQhX8m+L1eqJDQIALJlRv2Bnhrmku80YU9bBXE=
github.com/projectdiscovery/yamldoc-go v1.0.7 h1:Odmjd1o4EkMff2F6QHi1dIYPIQPZUFy24YqmQCprmTk=
github.com/projectdiscovery/yamldoc-go v1.0.7/go.mod h1:nqD1iPMcofpRA1DbXavPByJhWURrshT+l8WN9XjyQag=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
@@ -1075,8 +1079,9 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/sirupsen/logrus v1.10.2 h1:G2SED73/qrAu6YwbdxOD6peLkCBI3z7L+ykJFTXJBBo=
github.com/sirupsen/logrus v1.10.2/go.mod h1:SLEg8TqYulVKKfIGHldVp2K2aYz2DKSVBq4g/H5bR7Q=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8=
github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
@@ -1182,8 +1187,9 @@ github.com/vulncheck-oss/go-exploit v1.51.0/go.mod h1:J28w0dLnA6DnCrnBm9Sbt6smX8
github.com/weppos/publicsuffix-go v0.12.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k=
github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k=
github.com/weppos/publicsuffix-go v0.30.0/go.mod h1:kBi8zwYnR0zrbm8RcuN1o9Fzgpnnn+btVN8uWPMyXAY=
github.com/weppos/publicsuffix-go v0.50.4-0.20260918082853-3c3d7dc4a0cb h1:QIW0GHVTSubm3VxkKpn1FnFAgins6N8FiGTvUQwpXzw=
github.com/weppos/publicsuffix-go v0.50.4-0.20260918082853-3c3d7dc4a0cb/go.mod h1:/rOa781xBykZhHK/I3QeHo92qdDKVmKZKF7s8qAEM/4=
github.com/weppos/publicsuffix-go v0.40.2/go.mod h1:XsLZnULC3EJ1Gvk9GVjuCTZ8QUu9ufE4TZpOizDShko=
github.com/weppos/publicsuffix-go v0.50.3 h1:eT5dcjHQcVDNc0igpFEsGHKIip30feuB2zuuI9eJxiE=
github.com/weppos/publicsuffix-go v0.50.3/go.mod h1:/rOa781xBykZhHK/I3QeHo92qdDKVmKZKF7s8qAEM/4=
github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220927085643-dc0d00c92642/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE=
github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
@@ -1253,14 +1259,15 @@ github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE=
github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 h1:Nzukz5fNOBIHOsnP+6I79kPx3QhLv8nBy2mfFhBRq30=
github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE=
github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is=
github.com/zmap/zcertificate v0.0.1/go.mod h1:q0dlN54Jm4NVSSuzisusQY0hqDWvu92C+TWveAxiVWk=
github.com/zmap/zcrypto v0.0.0-20201128221613-3719af1573cf/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ=
github.com/zmap/zcrypto v0.0.0-20201211161100-e54a5822fb7e/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ=
github.com/zmap/zcrypto v0.0.0-20230310154051-c8b263fd8300/go.mod h1:mOd4yUMgn2fe2nV9KXsa9AyQBFZGzygVPovsZR+Rl5w=
github.com/zmap/zcrypto v0.0.0-20261015050653-aa1de3126e87 h1:B9sBGvtHjiGOOemAh0cCGTzub76DXlxmOh3Qtx3TkRs=
github.com/zmap/zcrypto v0.0.0-20261015050653-aa1de3126e87/go.mod h1:9O2aSz1l9fUHTFxJMYCoFsZBy6yGlz3nrOp4dHue5qw=
github.com/zmap/zcrypto v0.0.0-20240803002437-3a861682ac77 h1:DCz0McWRVJNICkHdu2XpETqeLvPtZXs315OZyUs1BDk=
github.com/zmap/zcrypto v0.0.0-20240803002437-3a861682ac77/go.mod h1:aSvf+uTU222mUYq/KQj3oiEU7ajhCZe8RRSLHIoM4EM=
github.com/zmap/zflags v1.4.0-beta.1.0.20200204220219-9d95409821b6/go.mod h1:HXDUD+uue8yeLHr0eXx1lvY6CvMiHbTKw5nGmA9OUoo=
github.com/zmap/zgrab2 v0.1.8 h1:PFnXrIBcGjYFec1JNbxMKQuSXXzS+SbqE89luuF4ORY=
github.com/zmap/zgrab2 v0.1.8/go.mod h1:5d8HSmUwvllx4q1qG50v/KXphkg45ZzWdaQtgTFnegE=
@@ -1338,9 +1345,10 @@ golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1378,8 +1386,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c=
golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o=
golang.org/x/mod v0.39.0 h1:UF5zwQdCRRUpHfyPwr7d4UrGiVeldIsogtzWVnczL74=
golang.org/x/mod v0.39.0/go.mod h1:bvIbwjQ0HUFFf5AKukeeYQG4ZBUG9yxQbR9aEweIwYY=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1427,9 +1435,10 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1438,6 +1447,7 @@ golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4Iltr
golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -1456,8 +1466,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1504,6 +1514,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -1521,9 +1532,10 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -1535,9 +1547,10 @@ golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4=
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE=
golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1553,9 +1566,10 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1704,6 +1718,7 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
+2 -13
View File
@@ -56,18 +56,10 @@ func ConfigureOptions() error {
return nil
}
// readsTargetsFromStdin reports whether targets are read from a stdin pipe.
// Targets given with -u, -l, or uncover take its place: under cron, CI or an
// agent stdin is often a pipe nothing writes to or closes, and reading it
// would hold the scan until it does.
func readsTargetsFromStdin(options *types.Options, stdinIsPipe bool) bool {
hasOtherTargets := len(options.Targets) > 0 || options.TargetsFilePath != "" || options.Uncover || len(options.UncoverQuery) > 0
return stdinIsPipe && !options.DisableStdin && !hasOtherTargets
}
// ParseOptions parses the command line flags provided by a user
func ParseOptions(options *types.Options) {
options.Stdin = readsTargetsFromStdin(options, fileutil.HasStdin())
// Check if stdin pipe was given
options.Stdin = !options.DisableStdin && fileutil.HasStdin()
// Read the inputs from env variables that not passed by flag.
readEnvInputVars(options)
@@ -179,9 +171,6 @@ func ValidateOptions(options *types.Options) error {
if err := loadProxyServers(options); err != nil {
return err
}
if err := validateReplayProxy(options.ReplayProxy); err != nil {
return err
}
if options.Validate {
validateTemplatePaths(options.Logger, config.DefaultConfig.TemplatesDirectory, options.Templates, options.Workflows)
}
-21
View File
@@ -59,24 +59,3 @@ func TestParseHeadlessOptionalArguments(t *testing.T) {
})
}
}
func TestReadsTargetsFromStdin(t *testing.T) {
for _, tc := range []struct {
name string
options *types.Options
stdinIsPipe bool
want bool
}{
{"piped stdin alone", &types.Options{}, true, true},
{"no pipe", &types.Options{}, false, false},
{"disabled", &types.Options{DisableStdin: true}, true, false},
{"targets given with -u", &types.Options{Targets: goflags.StringSlice{"example.com"}}, true, false},
{"targets given with -l", &types.Options{TargetsFilePath: "targets.txt"}, true, false},
{"uncover query", &types.Options{UncoverQuery: goflags.StringSlice{"example"}}, true, false},
{"uncover enabled", &types.Options{Uncover: true}, true, false},
} {
t.Run(tc.name, func(t *testing.T) {
require.Equal(t, tc.want, readsTargetsFromStdin(tc.options, tc.stdinIsPipe))
})
}
}
-18
View File
@@ -65,21 +65,3 @@ func loadProxyServers(options *types.Options) error {
}
return nil
}
// validateReplayProxy rejects a -replay-proxy value the replay client cannot dial
func validateReplayProxy(raw string) error {
if raw == "" {
return nil
}
proxyURL, err := url.Parse(raw)
if err != nil {
return errkit.Wrapf(err, "invalid replay proxy %q", raw)
}
switch proxyURL.Scheme {
case proxyutils.HTTP, proxyutils.HTTPS, proxyutils.SOCKS5:
if proxyURL.Host != "" {
return nil
}
}
return fmt.Errorf("invalid replay proxy %q: want an http, https or socks5 URL", raw)
}
-16
View File
@@ -1,16 +0,0 @@
package runner
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestValidateReplayProxy(t *testing.T) {
for _, valid := range []string{"", "http://127.0.0.1:8080", "https://proxy.example:8443", "socks5://127.0.0.1:1080"} {
require.NoError(t, validateReplayProxy(valid), valid)
}
for _, invalid := range []string{"127.0.0.1:8080", "ftp://127.0.0.1:21", "http://", "://bad"} {
require.Error(t, validateReplayProxy(invalid), invalid)
}
}
-5
View File
@@ -56,7 +56,6 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/globalmatchers"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/honeypotdetector"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hostbackoff"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit"
@@ -728,10 +727,6 @@ func (r *Runner) RunEnumeration() error {
executorOpts.HostErrorsCache = cache
}
if r.options.HostBackoff {
executorOpts.HostBackoff = hostbackoff.New(hostbackoff.Config{Max: r.options.HostBackoffMax})
}
executorEngine := core.New(r.options)
executorEngine.SetExecuterOptions(executorOpts)
-5
View File
@@ -28,7 +28,6 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/progress"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/globalmatchers"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hostbackoff"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/excludematchers"
@@ -107,10 +106,6 @@ func newNucleiExecutor(opts *NucleiExecutorOptions) (*nucleiExecutor, error) {
executorOpts.HostErrorsCache = cache
}
if opts.Options.HostBackoff {
executorOpts.HostBackoff = hostbackoff.New(hostbackoff.Config{Max: opts.Options.HostBackoffMax})
}
executorEngine := core.New(opts.Options)
executorEngine.SetExecuterOptions(executorOpts)
+3 -5
View File
@@ -18,11 +18,9 @@ import (
)
const (
dbName = "test"
dbRepository = "mongo"
// 8.x refuses to start on kernels 6.19 through 7.0.13 (SERVER-125742).
// The GitHub Ubuntu image is in that range. 7.0 stays up.
dbTag = "7.0"
dbName = "test"
dbRepository = "mongo"
dbTag = "8"
dbPort = "27017/tcp"
mongoDatabaseReadyTimeout = 3 * time.Minute
mongoServerSelectionDelay = time.Second
+1 -1
View File
@@ -17,7 +17,7 @@ type multiProtoDynamicExtractor struct{}
// Execute executes a test case and returns an error if occurred
func (h *multiProtoDynamicExtractor) Execute(templatePath string) error {
results, err := testutils.RunNucleiTemplateAndGetResults(templatePath, "https://docs.projectdiscovery.io/", debug)
results, err := testutils.RunNucleiTemplateAndGetResults(templatePath, "docs.projectdiscovery.io", debug)
if err != nil {
return err
}
@@ -24,6 +24,6 @@ http:
matchers:
- type: dsl
dsl:
- location == '/home' # the docs host redirects here
- contains(body,'home') # check for http string
- blogid == 'cname' # check for cname (extracted information from dns response)
condition: and
@@ -24,7 +24,7 @@ http:
matchers:
- type: dsl
dsl:
- http_location == '/home' # the docs host redirects here
- contains(http_body,'home') # check for http string
- cname_filtered == 'cname' # check for cname (extracted information from dns response)
- ssl_subject_cn == 'docs.projectdiscovery.io'
condition: and
@@ -20,7 +20,7 @@ http:
matchers:
- type: dsl
dsl:
- http_location == '/home' # the docs host redirects here
- contains(http_body,'home') # check for http string
- trim_suffix(dns_cname,'.vercel-dns.com') == 'cname' # check for cname (extracted information from dns response)
- ssl_subject_cn == 'docs.projectdiscovery.io'
condition: and
+1 -1
View File
@@ -13,7 +13,7 @@ type StatsOutputWriter struct {
var _ Writer = &StatsOutputWriter{}
// NewTrackerWriter returns a new StatsOutputWriter instance.
// NewStatsOutputWriter returns a new StatsOutputWriter instance.
func NewTrackerWriter(t *stats.Tracker) *StatsOutputWriter {
return &StatsOutputWriter{
colorizer: aurora.New(aurora.WithColors(true)),
@@ -256,12 +256,6 @@ func (ctx *Context) Clone() *Context {
return newCtx
}
func (ctx *Context) CloneWithContext(parent context.Context) *Context {
cloned := ctx.Clone()
cloned.ctx = parent
return cloned
}
// resolvePortList converts a list of port strings (numeric or service names) to numeric port strings.
func resolvePortList(ports []string) []string {
resolved := make([]string, 0, len(ports))
@@ -7,25 +7,6 @@ import (
"github.com/stretchr/testify/require"
)
func TestCloneWithContextPreservesInputWithoutSharingMutableState(t *testing.T) {
parent := NewWithInput(context.Background(), "https://example.com")
parent.Set("shared", "original")
childCtx, cancel := context.WithCancel(context.Background())
child := parent.CloneWithContext(childCtx)
require.Equal(t, parent.MetaInput.Input, child.MetaInput.Input)
childValue, ok := child.Get("shared")
require.True(t, ok)
require.Equal(t, "original", childValue)
child.Set("shared", "changed")
parentValue, ok := parent.Get("shared")
require.True(t, ok)
require.Equal(t, "original", parentValue)
cancel()
require.ErrorIs(t, child.Context().Err(), context.Canceled)
require.NoError(t, parent.Context().Err())
}
// TestUseNetworkPort is the behavior matrix for UseNetworkPort.
//
// The contract (see issue #7323):
@@ -89,7 +89,7 @@ func generateCommonsCollections40Payload(cmd string) []byte {
return buffer.Bytes()
}
// generateCommonsCollections31Payload generates commons-collections 3.1
// generateCommonsCollections440PPayload generates commons-collections 3.1
// deserialization payload for a command.
func generateCommonsCollections31Payload(cmd string) []byte {
buffer := &bytes.Buffer{}
@@ -1,320 +0,0 @@
// Package hostbackoff paces requests to a host that signals it is being
// overloaded, so a scan slows down instead of being blocked outright.
//
// nuclei already skips a host after repeated transport errors
// (pkg/protocols/common/hosterrorscache), which is all or nothing. A target
// that answers 429, or starts answering 403 to everything, is asking for less
// traffic rather than none, and the useful response is to keep scanning it more
// slowly.
//
// The control rule follows shipped practice rather than anything novel: a
// per-host delay that grows multiplicatively while the host complains and
// decays while it is healthy, with the host's own Retry-After honoured when it
// asks for longer. Scrapy's AutoThrottle, Heritrix and Googlebot's crawl
// capacity limit all work this way, and the asymmetry (a bad response may only
// slow the scan, never speed it up) is what keeps one lucky success from
// undoing a backoff.
package hostbackoff
import (
"context"
"math"
"net/http"
"strconv"
"strings"
"sync"
"time"
"github.com/projectdiscovery/gcache"
)
const (
// DefaultStep is the delay applied the first time a host signals overload.
DefaultStep = 250 * time.Millisecond
// DefaultMax caps the per-host delay. Past this the host is better handled
// by the host error cache, which skips it entirely.
DefaultMax = 30 * time.Second
// DefaultFactor grows the delay on each further complaint.
DefaultFactor = 2.0
// DefaultDecay shrinks the delay after a healthy response. It is far gentler
// than the growth so that a scan recovers gradually rather than immediately
// re-flooding a host that just recovered.
DefaultDecay = 0.8
// DefaultForbiddenStreak is how many consecutive 403s count as blocking. A
// single 403 is ordinary (an endpoint the scan is not authorised for), but a
// run of them usually means the host started refusing the scanner.
DefaultForbiddenStreak = 5
// DefaultMaxHosts bounds how many hosts are tracked at once, matching the
// host error cache. A scan can carry millions of targets, so the state has
// to be evictable; losing a host's entry only resets its delay, and the next
// complaint earns it again.
DefaultMaxHosts = 10000
)
// Config tunes the governor. The zero value is replaced by the defaults above.
type Config struct {
Step time.Duration
Max time.Duration
Factor float64
Decay float64
ForbiddenStreak int
MaxHosts int
}
func (c *Config) applyDefaults() {
if c.Step <= 0 {
c.Step = DefaultStep
}
if c.Max <= 0 {
c.Max = DefaultMax
}
if c.Factor <= 1 {
c.Factor = DefaultFactor
}
if c.Decay <= 0 || c.Decay >= 1 {
c.Decay = DefaultDecay
}
if c.ForbiddenStreak <= 0 {
c.ForbiddenStreak = DefaultForbiddenStreak
}
if c.MaxHosts <= 0 {
c.MaxHosts = DefaultMaxHosts
}
}
type hostState struct {
delay time.Duration
forbidden int
// reachable is set once the host has answered. A dial that fails before
// then is a closed or filtered port, which is no reason to slow down.
reachable bool
// next is the earliest start time still free. Concurrent waits reserve
// successive slots so a delay paces the host instead of holding a burst
// and releasing it together.
next time.Time
}
// Governor holds the per-host delay for a scan.
type Governor struct {
cfg Config
mu sync.Mutex
hosts gcache.Cache[string, *hostState]
}
// New returns a governor using cfg, with unset fields defaulted.
func New(cfg Config) *Governor {
cfg.applyDefaults()
return &Governor{cfg: cfg, hosts: gcache.New[string, *hostState](cfg.MaxHosts).ARC().Build()}
}
// Delay reports the current pause for a host, zero when it looks healthy.
func (g *Governor) Delay(host string) time.Duration {
if g == nil || host == "" {
return 0
}
g.mu.Lock()
defer g.mu.Unlock()
if state, err := g.hosts.GetIFPresent(host); err == nil && state != nil {
return state.delay
}
return 0
}
// Wait pauses until this caller’s slot on the host. Callers that arrive
// together take successive slots of the current delay, so they do not all
// wake at once. It returns the context error if the scan is cancelled while
// waiting, so a stop is not held up by a long backoff.
func (g *Governor) Wait(ctx context.Context, host string) error {
if g == nil || host == "" {
return nil
}
if ctx.Err() != nil {
return ctx.Err()
}
g.mu.Lock()
state, err := g.hosts.GetIFPresent(host)
if err != nil || state == nil || state.delay <= 0 {
g.mu.Unlock()
return nil
}
now := time.Now()
start := state.next
if start.Before(now) {
start = now.Add(state.delay)
}
state.next = start.Add(state.delay)
_ = g.hosts.Set(host, state)
wait := start.Sub(now)
g.mu.Unlock()
if wait <= 0 {
return nil
}
timer := time.NewTimer(wait)
defer timer.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
return nil
}
}
// Observe feeds one response back to the governor. retryAfter is the parsed
// Retry-After header, zero when absent. err is the transport error, if any.
func (g *Governor) Observe(host string, statusCode int, retryAfter time.Duration, err error) {
if g == nil || host == "" {
return
}
g.mu.Lock()
defer g.mu.Unlock()
state := g.state(host)
defer func() { _ = g.hosts.Set(host, state) }()
if statusCode > 0 {
state.reachable = true
}
if statusCode == http.StatusForbidden {
state.forbidden++
} else if healthyStatus(statusCode) {
state.forbidden = 0
}
// A 403 that names Retry-After is the host asking for a wait. The streak
// is only for bare 403s, which are often just an unauthorised endpoint.
if statusCode == http.StatusForbidden && retryAfter > 0 && state.forbidden < g.cfg.ForbiddenStreak {
state.forbidden = g.cfg.ForbiddenStreak
}
if !g.isBlocking(statusCode, err, state) {
// Only a response that actually served (2xx/3xx) is evidence the host
// recovered. A 404 or 500 is what most templates get back, and treating
// it as healthy would wipe the delay on the next probe.
if healthyStatus(statusCode) {
g.decay(state)
}
return
}
g.grow(state, retryAfter)
}
// ObserveDial feeds one connection attempt back to the governor, for
// protocols that have no response status to judge. A failed dial slows a host
// that accepted connections before, and a successful one counts as recovery.
// Errors on an established connection are the template's business, not the
// host asking for less.
func (g *Governor) ObserveDial(host string, err error) {
if g == nil || host == "" {
return
}
g.mu.Lock()
defer g.mu.Unlock()
state := g.state(host)
switch {
case err == nil:
state.reachable = true
g.decay(state)
case state.reachable:
g.grow(state, 0)
default:
// A port that never accepted is closed or filtered. Storing it would
// fill the cache and evict hosts that actually have a delay.
return
}
_ = g.hosts.Set(host, state)
}
func (g *Governor) state(host string) *hostState {
if state, err := g.hosts.GetIFPresent(host); err == nil && state != nil {
return state
}
return &hostState{}
}
func (g *Governor) grow(state *hostState, retryAfter time.Duration) {
next := state.delay
if next <= 0 {
next = g.cfg.Step
} else {
next = time.Duration(math.Min(float64(next)*g.cfg.Factor, float64(g.cfg.Max)))
}
// The host named a wait of its own; respect it when it is the longer of the
// two, since it is the only figure that is not a guess.
if retryAfter > next {
next = retryAfter
}
if next > g.cfg.Max {
next = g.cfg.Max
}
state.delay = next
}
func (g *Governor) decay(state *hostState) {
state.delay = time.Duration(float64(state.delay) * g.cfg.Decay)
if state.delay < g.cfg.Step {
state.delay = 0
state.next = time.Time{}
}
}
// isBlocking reports whether a response means the host wants less traffic.
func (g *Governor) isBlocking(statusCode int, err error, state *hostState) bool {
if err != nil {
return true
}
switch statusCode {
case http.StatusTooManyRequests, http.StatusServiceUnavailable:
return true
case http.StatusForbidden:
return state.forbidden >= g.cfg.ForbiddenStreak
}
return false
}
// healthyStatus reports whether the host served a normal response. Template
// misses and server errors are not recovery.
func healthyStatus(statusCode int) bool {
return statusCode >= http.StatusOK && statusCode < http.StatusBadRequest
}
// RetryAfter parses a Retry-After header, in either its delay-seconds or
// HTTP-date form. Delay-seconds is a non-negative decimal integer (RFC 9110);
// fractional and compound duration strings are rejected. A digit string that
// does not fit in a Duration saturates instead of being ignored. The result
// is zero when the header is absent or unparsable, and never a negative wait
// for a date already in the past.
func RetryAfter(header string) time.Duration {
header = strings.TrimSpace(header)
if header == "" {
return 0
}
if seconds, ok := delaySeconds(header); ok {
return seconds
}
if when, err := http.ParseTime(header); err == nil {
if wait := time.Until(when); wait > 0 {
return wait
}
}
return 0
}
// delaySeconds parses a Retry-After delay-seconds value. The boolean is false
// when header is not a decimal integer.
func delaySeconds(header string) (time.Duration, bool) {
if header == "" {
return 0, false
}
for i := 0; i < len(header); i++ {
if header[i] < '0' || header[i] > '9' {
return 0, false
}
}
const maxDuration = time.Duration(1<<63 - 1)
const maxSeconds = uint64(maxDuration / time.Second)
seconds, err := strconv.ParseUint(header, 10, 64)
if err != nil || seconds > maxSeconds {
return maxDuration, true
}
return time.Duration(seconds) * time.Second, true
}
@@ -1,248 +0,0 @@
package hostbackoff
import (
"context"
"errors"
"net/http"
"sync"
"testing"
"time"
"github.com/stretchr/testify/require"
)
func TestHealthyHostIsNotDelayed(t *testing.T) {
g := New(Config{})
for i := 0; i < 10; i++ {
g.Observe("acme.test", http.StatusOK, 0, nil)
}
require.Zero(t, g.Delay("acme.test"), "a host that answers normally must not be slowed")
}
func TestTooManyRequestsBacksOff(t *testing.T) {
g := New(Config{Step: 100 * time.Millisecond, Factor: 2, Max: time.Second})
g.Observe("acme.test", http.StatusTooManyRequests, 0, nil)
require.Equal(t, 100*time.Millisecond, g.Delay("acme.test"))
g.Observe("acme.test", http.StatusTooManyRequests, 0, nil)
require.Equal(t, 200*time.Millisecond, g.Delay("acme.test"), "a repeat complaint grows the delay")
}
func TestServiceUnavailableBacksOff(t *testing.T) {
g := New(Config{Step: 100 * time.Millisecond})
g.Observe("acme.test", http.StatusServiceUnavailable, 0, nil)
require.Equal(t, 100*time.Millisecond, g.Delay("acme.test"))
}
func TestTransportErrorBacksOff(t *testing.T) {
g := New(Config{Step: 100 * time.Millisecond})
g.Observe("acme.test", 0, 0, errors.New("connection reset"))
require.Equal(t, 100*time.Millisecond, g.Delay("acme.test"))
}
// A single 403 is ordinary: plenty of templates probe endpoints the scan is not
// authorised for. A run of them is the host refusing the scanner.
func TestForbiddenOnlyBacksOffAsAStreak(t *testing.T) {
g := New(Config{Step: 100 * time.Millisecond, ForbiddenStreak: 3})
g.Observe("acme.test", http.StatusForbidden, 0, nil)
g.Observe("acme.test", http.StatusForbidden, 0, nil)
require.Zero(t, g.Delay("acme.test"), "an occasional 403 is not a block")
g.Observe("acme.test", http.StatusForbidden, 0, nil)
require.Equal(t, 100*time.Millisecond, g.Delay("acme.test"), "a streak is")
}
func TestOkResetsTheForbiddenStreak(t *testing.T) {
g := New(Config{Step: 100 * time.Millisecond, ForbiddenStreak: 3})
g.Observe("acme.test", http.StatusForbidden, 0, nil)
g.Observe("acme.test", http.StatusForbidden, 0, nil)
g.Observe("acme.test", http.StatusOK, 0, nil)
g.Observe("acme.test", http.StatusForbidden, 0, nil)
require.Zero(t, g.Delay("acme.test"), "the streak must start again after a healthy response")
}
// The host's own figure is the only one that is not a guess, so it wins when it
// is longer than what the governor would have chosen.
func TestRetryAfterIsHonouredWhenLonger(t *testing.T) {
g := New(Config{Step: 100 * time.Millisecond, Max: time.Minute})
g.Observe("acme.test", http.StatusTooManyRequests, 5*time.Second, nil)
require.Equal(t, 5*time.Second, g.Delay("acme.test"))
// a shorter Retry-After does not undo a longer backoff already earned
g.Observe("acme.test", http.StatusTooManyRequests, time.Millisecond, nil)
require.Equal(t, 10*time.Second, g.Delay("acme.test"))
}
func TestDelayIsCapped(t *testing.T) {
g := New(Config{Step: time.Second, Factor: 10, Max: 2 * time.Second})
for i := 0; i < 5; i++ {
g.Observe("acme.test", http.StatusTooManyRequests, time.Hour, nil)
}
require.Equal(t, 2*time.Second, g.Delay("acme.test"), "neither growth nor Retry-After may exceed the cap")
}
func TestHealthyResponsesDecayTheDelay(t *testing.T) {
g := New(Config{Step: 100 * time.Millisecond, Factor: 2, Decay: 0.5})
g.Observe("acme.test", http.StatusTooManyRequests, 0, nil)
g.Observe("acme.test", http.StatusTooManyRequests, 0, nil)
require.Equal(t, 200*time.Millisecond, g.Delay("acme.test"))
g.Observe("acme.test", http.StatusOK, 0, nil)
require.Equal(t, 100*time.Millisecond, g.Delay("acme.test"))
// below the first step there is nothing useful left to wait for
g.Observe("acme.test", http.StatusOK, 0, nil)
require.Zero(t, g.Delay("acme.test"))
}
func TestHostsAreIndependent(t *testing.T) {
g := New(Config{Step: 100 * time.Millisecond})
g.Observe("blocked.test", http.StatusTooManyRequests, 0, nil)
require.Equal(t, 100*time.Millisecond, g.Delay("blocked.test"))
require.Zero(t, g.Delay("healthy.test"), "one host complaining must not slow another")
}
func TestWaitSleepsAndRespectsCancellation(t *testing.T) {
g := New(Config{Step: 50 * time.Millisecond})
start := time.Now()
require.NoError(t, g.Wait(context.Background(), "quiet.test"))
require.Less(t, time.Since(start), 10*time.Millisecond, "a healthy host is not waited on")
g.Observe("slow.test", http.StatusTooManyRequests, 0, nil)
start = time.Now()
require.NoError(t, g.Wait(context.Background(), "slow.test"))
require.GreaterOrEqual(t, time.Since(start), 50*time.Millisecond)
// a cancelled scan must not be held up by a long backoff
g.Observe("stuck.test", http.StatusTooManyRequests, time.Hour, nil)
ctx, cancel := context.WithCancel(context.Background())
cancel()
require.ErrorIs(t, g.Wait(ctx, "stuck.test"), context.Canceled)
}
func TestWaitSpacesConcurrentCallers(t *testing.T) {
const step = 60 * time.Millisecond
g := New(Config{Step: step, Max: time.Second})
g.Observe("burst.test", http.StatusTooManyRequests, 0, nil)
start := time.Now()
var wg sync.WaitGroup
wg.Add(2)
for i := 0; i < 2; i++ {
go func() {
defer wg.Done()
require.NoError(t, g.Wait(context.Background(), "burst.test"))
}()
}
wg.Wait()
require.GreaterOrEqual(t, time.Since(start), 2*step-10*time.Millisecond)
}
func TestForbiddenRetryAfterBacksOffImmediately(t *testing.T) {
g := New(Config{Step: 100 * time.Millisecond, ForbiddenStreak: 5, Max: time.Minute})
g.Observe("acme.test", http.StatusForbidden, 2*time.Second, nil)
require.Equal(t, 2*time.Second, g.Delay("acme.test"))
}
func TestNilGovernorIsInert(t *testing.T) {
var g *Governor
require.Zero(t, g.Delay("acme.test"))
require.NoError(t, g.Wait(context.Background(), "acme.test"))
require.NotPanics(t, func() { g.Observe("acme.test", http.StatusTooManyRequests, 0, nil) })
}
func TestRetryAfterParsing(t *testing.T) {
require.Equal(t, 5*time.Second, RetryAfter("5"))
require.Equal(t, 5*time.Second, RetryAfter(" 5 "))
require.Zero(t, RetryAfter(""))
require.Zero(t, RetryAfter("not-a-number"))
require.Zero(t, RetryAfter("-3"), "a negative wait is meaningless")
require.Zero(t, RetryAfter(time.Now().Add(-time.Hour).UTC().Format(http.TimeFormat)), "a date in the past is not a wait")
future := RetryAfter(time.Now().Add(30 * time.Second).UTC().Format(http.TimeFormat))
require.Greater(t, future, 25*time.Second)
require.LessOrEqual(t, future, 30*time.Second)
}
// The previous parser appended "s" and called time.ParseDuration, which accepts
// fractional seconds and Go duration strings, and returns an error (then zero)
// when the integer does not fit.
func TestRetryAfterRejectsDurationSyntax(t *testing.T) {
fractional, err := time.ParseDuration("1.5s")
require.NoError(t, err)
require.Equal(t, 1500*time.Millisecond, fractional)
require.Zero(t, RetryAfter("1.5"))
compound, err := time.ParseDuration("1m3s")
require.NoError(t, err)
require.Equal(t, 63*time.Second, compound)
require.Zero(t, RetryAfter("1m3"))
_, err = time.ParseDuration("999999999999999999999s")
require.Error(t, err)
require.Equal(t, time.Duration(1<<63-1), RetryAfter("999999999999999999999"))
}
func TestNotFoundDoesNotClearBackoff(t *testing.T) {
g := New(Config{Step: 100 * time.Millisecond, Decay: 0.5})
g.Observe("acme.test", http.StatusTooManyRequests, 0, nil)
g.Observe("acme.test", http.StatusNotFound, 0, nil)
require.Equal(t, 100*time.Millisecond, g.Delay("acme.test"))
g.Observe("acme.test", http.StatusInternalServerError, 0, nil)
require.Equal(t, 100*time.Millisecond, g.Delay("acme.test"))
g.Observe("acme.test", http.StatusOK, 0, nil)
require.Zero(t, g.Delay("acme.test"))
}
func TestForbiddenStreakSurvivesAMiss(t *testing.T) {
g := New(Config{Step: 100 * time.Millisecond, ForbiddenStreak: 3})
g.Observe("acme.test", http.StatusForbidden, 0, nil)
g.Observe("acme.test", http.StatusForbidden, 0, nil)
g.Observe("acme.test", http.StatusNotFound, 0, nil)
require.Zero(t, g.Delay("acme.test"))
g.Observe("acme.test", http.StatusForbidden, 0, nil)
require.Equal(t, 100*time.Millisecond, g.Delay("acme.test"))
}
func TestFailedDialBacksOffAndSuccessfulDialDecays(t *testing.T) {
g := New(Config{Step: 100 * time.Millisecond, Factor: 2, Decay: 0.5})
g.ObserveDial("acme.test:22", nil)
g.ObserveDial("acme.test:22", errors.New("connection refused"))
g.ObserveDial("acme.test:22", errors.New("connection refused"))
require.Equal(t, 200*time.Millisecond, g.Delay("acme.test:22"))
g.ObserveDial("acme.test:22", nil)
require.Equal(t, 100*time.Millisecond, g.Delay("acme.test:22"))
g.ObserveDial("acme.test:22", nil)
require.Zero(t, g.Delay("acme.test:22"))
}
// A port that never accepted is closed or filtered, not pushing back. Slowing
// it would cost every template aimed at it a growing wait for nothing.
func TestDialFailureOnNeverReachedHostIsIgnored(t *testing.T) {
g := New(Config{Step: 100 * time.Millisecond})
for i := 0; i < 3; i++ {
g.ObserveDial("acme.test:22", errors.New("connection refused"))
}
require.Zero(t, g.Delay("acme.test:22"))
_, err := g.hosts.GetIFPresent("acme.test:22")
require.Error(t, err, "a port that never accepted must not take a cache slot")
g.ObserveDial("ok.test:22", nil)
_, err = g.hosts.GetIFPresent("ok.test:22")
require.NoError(t, err, "a port that accepted is tracked so a later refusal can slow it")
}
func TestResponseMarksHostReachableForDials(t *testing.T) {
g := New(Config{Step: 100 * time.Millisecond})
g.Observe("acme.test:443", http.StatusNotFound, 0, nil)
g.ObserveDial("acme.test:443", errors.New("connection refused"))
require.Equal(t, 100*time.Millisecond, g.Delay("acme.test:443"))
}
@@ -242,7 +242,7 @@ func (c *Client) poll() error {
return nil
}
// requestShouldStopAtFirstMatch checks if further interactions should be stopped
// requestShouldStopAtFirstmatch checks if further interactions should be stopped
// note: extra care should be taken while using this function since internalEvent is
// synchronized all the time and if caller functions has already acquired lock its best to explicitly specify that
// we could use `TryLock()` but that may over complicate things and need to differentiate
@@ -21,14 +21,14 @@ func WithExecutionID(ctx context.Context, executionContext *ExecutionContext) co
return context.WithValue(ctx, executionIDKey, executionContext)
}
// HasExecutionContext checks if the context has an execution context.
// HasExecutionID checks if the context has an execution ID
func HasExecutionContext(ctx context.Context) bool {
_, ok := ctx.Value(executionIDKey).(*ExecutionContext)
return ok
}
// GetExecutionContext retrieves the execution context from the context.
// It returns nil if no execution context is set.
// GetExecutionID retrieves the execution ID from the context
// Returns empty string if no execution ID is set
func GetExecutionContext(ctx context.Context) *ExecutionContext {
if id, ok := ctx.Value(executionIDKey).(*ExecutionContext); ok {
return id
@@ -5,7 +5,6 @@ import (
"sync"
"time"
"github.com/docker/go-units"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/utils/env"
httputil "github.com/projectdiscovery/utils/http"
@@ -17,7 +16,6 @@ var (
MaxBytesBufferAllocOnLowMemory = env.GetEnvOrDefault("MEMGUARDIAN_ALLOC", 0)
memTimer *time.Ticker
cancelFunc context.CancelFunc
memWatcherDone chan struct{}
muGlobalChange sync.Mutex
)
@@ -27,18 +25,12 @@ func StartActiveMemGuardian(ctx context.Context) {
if memguardian.DefaultMemGuardian == nil || memTimer != nil {
return
}
if err := refreshMemoryWarning(); err != nil {
gologger.Warning().Msgf("could not check memory pressure: %s", err)
}
memTimer = time.NewTicker(memguardian.DefaultInterval)
ctx, cancelFunc = context.WithCancel(ctx)
memWatcherDone = make(chan struct{})
ticker := memTimer
done := memWatcherDone
go func(t *time.Ticker) {
defer close(done)
if t == nil {
return
}
@@ -47,9 +39,6 @@ func StartActiveMemGuardian(ctx context.Context) {
case <-ctx.Done():
return
case <-t.C:
if err := refreshMemoryWarning(); err != nil {
gologger.Warning().Msgf("could not check memory pressure: %s", err)
}
if IsLowOnMemory() {
_ = GlobalGuardBytesBufferAlloc()
} else {
@@ -60,24 +49,6 @@ func StartActiveMemGuardian(ctx context.Context) {
}(ticker)
}
func refreshMemoryWarning() error {
usedRatio, usedBytes, err := memguardian.UsedRam()
if err != nil {
return err
}
warning := usedRatio >= memguardian.DefaultMaxUsedRamRatio
if maxRam := env.GetEnvOrDefault(memguardian.MemGuardianMaxUsedMemoryENV, ""); maxRam != "" {
maxBytes, err := units.FromHumanSize(maxRam)
if err != nil {
return err
}
warning = warning || maxBytes > 0 && usedBytes >= uint64(maxBytes)
}
memguardian.DefaultMemGuardian.Warning.Store(warning)
return nil
}
func StopActiveMemGuardian() {
muGlobalChange.Lock()
defer muGlobalChange.Unlock()
@@ -90,10 +61,6 @@ func StopActiveMemGuardian() {
cancelFunc()
cancelFunc = nil
}
if memWatcherDone != nil {
<-memWatcherDone
memWatcherDone = nil
}
if memTimer != nil {
memTimer.Stop()
memTimer = nil
@@ -110,7 +77,7 @@ func IsLowOnMemory() bool {
// GuardThreads on caller
func GuardThreadsOrDefault(current int) int {
if MaxThreadsOnLowMemory > 0 {
return min(current, MaxThreadsOnLowMemory)
return MaxThreadsOnLowMemory
}
fraction := int(current / 5)
@@ -127,45 +127,3 @@ func TestMemGuardianReset(t *testing.T) {
StopActiveMemGuardian()
time.Sleep(10 * time.Millisecond) // Allow cleanup
}
func TestActiveMemGuardianUpdatesLowMemoryWarning(t *testing.T) {
StopActiveMemGuardian()
t.Setenv(memguardian.MemGuardianMaxUsedMemoryENV, "")
original := memguardian.DefaultMemGuardian
originalInterval := memguardian.DefaultInterval
originalRatio := memguardian.DefaultMaxUsedRamRatio
memguardian.DefaultInterval = 5 * time.Millisecond
memguardian.DefaultMaxUsedRamRatio = 1
guardian, err := memguardian.New(
memguardian.WitInterval(memguardian.DefaultInterval),
memguardian.WithMaxRamRatioWarning(1),
)
require.NoError(t, err)
memguardian.DefaultMemGuardian = guardian
t.Cleanup(func() {
StopActiveMemGuardian()
guardian.Close()
memguardian.DefaultMemGuardian = original
memguardian.DefaultInterval = originalInterval
memguardian.DefaultMaxUsedRamRatio = originalRatio
})
StartActiveMemGuardian(context.Background())
require.Eventually(t, IsLowOnMemory, 100*time.Millisecond, 5*time.Millisecond)
StopActiveMemGuardian()
memguardian.DefaultMaxUsedRamRatio = 100
require.NoError(t, refreshMemoryWarning())
require.False(t, IsLowOnMemory())
t.Setenv(memguardian.MemGuardianMaxUsedMemoryENV, "0")
require.NoError(t, refreshMemoryWarning())
require.False(t, IsLowOnMemory())
}
func TestGuardThreadsNeverIncreasesConcurrency(t *testing.T) {
previous := MaxThreadsOnLowMemory
MaxThreadsOnLowMemory = 5
t.Cleanup(func() { MaxThreadsOnLowMemory = previous })
require.Equal(t, 2, GuardThreadsOrDefault(2))
require.Equal(t, 5, GuardThreadsOrDefault(25))
}
+6 -7
View File
@@ -89,15 +89,13 @@ func newHttpClient(options *types.Options) (*http.Client, error) {
// causing "malformed HTTP response" errors. We use http2.Transport directly
// with utls dialing to properly handle h2 connections (see #6360).
if options.TlsImpersonate {
// utls conns are not *tls.Conn, so http.Transport speaks HTTP/1 on an h2 session.
// http2.Transport is deprecated in x/net 0.59, and it is still the dial that keeps h2. See #6360.
roundTripper = &http2.Transport{ //nolint:staticcheck // SA1019
DialTLSContext: func(ctx context.Context, network, addr string, _ *tls.Config) (net.Conn, error) { //nolint:staticcheck // SA1019
roundTripper = &http2.Transport{
DialTLSContext: func(ctx context.Context, network, addr string, _ *tls.Config) (net.Conn, error) {
return dialers.Fastdialer.DialTLSWithConfigImpersonate(ctx, network, addr, tlsConfig, impersonate.Random, nil)
},
TLSClientConfig: tlsConfig, //nolint:staticcheck // SA1019
AllowHTTP: false, //nolint:staticcheck // SA1019
DisableCompression: false, //nolint:staticcheck // SA1019
TLSClientConfig: tlsConfig,
AllowHTTP: false,
DisableCompression: false,
}
}
@@ -115,3 +113,4 @@ func newHttpClient(options *types.Options) (*http.Client, error) {
return httpclient, nil
}
@@ -230,9 +230,7 @@ func (p *Page) WaitVisible(act *Action, out ActionData) error {
}
func (p *Page) Sleeper(pollTimeout, timeout time.Duration) *Page {
p.mutex.RLock()
page := *p
p.mutex.RUnlock()
page.page = page.Page().Sleeper(func() utils.Sleeper {
return createBackOffSleeper(pollTimeout, timeout)
})
@@ -240,9 +238,7 @@ func (p *Page) Sleeper(pollTimeout, timeout time.Duration) *Page {
}
func (p *Page) Timeout(timeout time.Duration) *Page {
p.mutex.RLock()
page := *p
p.mutex.RUnlock()
page.page = page.Page().Timeout(timeout)
return &page
}
@@ -736,18 +732,6 @@ func (p *Page) SelectInputElement(act *Action, out ActionData) error {
// WaitPageLifecycleEvent waits for specified page lifecycle event name
func (p *Page) WaitPageLifecycleEvent(act *Action, out ActionData, event proto.PageLifecycleEventName) error {
if event == proto.PageLifecycleEventNameLoad {
dur, err := getTimeout(p, act)
if err != nil {
return err
}
if err := p.page.Timeout(dur).WaitLoad(); err != nil {
return err
}
p.updateLastNavigatedURL()
return nil
}
fn, err := getNavigationFunc(p, act, event)
if err != nil {
return err
@@ -18,8 +18,6 @@ import (
"testing"
"time"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/proto"
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testheadless"
@@ -155,26 +153,6 @@ func TestLocatorInteractshTracking(t *testing.T) {
})
}
func TestSleeperConcurrentHistoryUpdate(t *testing.T) {
page := &Page{page: &rod.Page{}, mutex: &sync.RWMutex{}}
start := make(chan struct{})
var done sync.WaitGroup
done.Add(1)
go func() {
defer done.Done()
<-start
for range 10000 {
page.addToHistory(HistoryData{})
}
}()
close(start)
for range 10000 {
page.Sleeper(time.Millisecond, time.Second)
}
done.Wait()
}
func TestActionNavigate(t *testing.T) {
response := `
<html>
@@ -664,47 +642,6 @@ func TestActionWaitLoad(t *testing.T) {
})
}
func TestActionWaitLoadAfterPageAlreadyLoaded(t *testing.T) {
actions := []*Action{{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}}
testHeadlessSimpleResponse(t, "<html><body>ready</body></html>", actions, 10*time.Second, func(page *Page, err error, _ ActionData) {
require.NoError(t, err)
require.NoError(t, page.Page().WaitLoad())
wait := &Action{ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}, Data: map[string]string{"timeout": "2s"}}
started := time.Now()
require.NoError(t, page.WaitPageLifecycleEvent(wait, nil, proto.PageLifecycleEventNameLoad))
require.Less(t, time.Since(started), 1500*time.Millisecond, "a completed load must not consume the full wait timeout")
})
}
func TestActionWaitLoadWaitsForPendingResource(t *testing.T) {
resourceDone := make(chan struct{})
actions := []*Action{
{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}},
}
testHeadless(t, actions, 10*time.Second, func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/slow.js" {
w.Header().Set("Content-Type", "text/javascript")
time.Sleep(300 * time.Millisecond)
_, _ = fmt.Fprint(w, "window.resourceLoaded = true")
close(resourceDone)
return
}
_, _ = fmt.Fprint(w, `<html><script src="/slow.js"></script></html>`)
}, func(page *Page, err error, _ ActionData) {
require.NoError(t, err)
select {
case <-resourceDone:
default:
t.Fatal("waitload returned before the pending resource completed")
}
loaded, evalErr := page.Page().Eval(`() => window.resourceLoaded`)
require.NoError(t, evalErr)
require.True(t, loaded.Value.Bool())
})
}
func TestActionGetResource(t *testing.T) {
response := `
<html>
@@ -1,98 +0,0 @@
package http
import (
"context"
"net/http"
"testing"
"time"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hostbackoff"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hostratelimit"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httputils"
"github.com/projectdiscovery/retryablehttp-go"
urlutil "github.com/projectdiscovery/utils/url"
"github.com/stretchr/testify/require"
)
func TestHostBackoffKeyKeepsHTTPSPort(t *testing.T) {
raw, err := retryablehttp.NewRequest(http.MethodGet, "https://example.com/secret", nil)
require.NoError(t, err)
gr := &generatedRequest{request: raw}
key := hostBackoffKey(gr, nil)
require.Equal(t, "example.com:443", hostratelimit.NormalizeHostPort(key))
// request.URL.Host drops the default port. That is what executeRequest used
// to observe, and it lands on port 80.
require.Equal(t, "example.com", raw.URL.Host)
require.Equal(t, "example.com:80", hostratelimit.NormalizeHostPort(raw.URL.Host))
}
func TestObserveIgnoresCanceledRequest(t *testing.T) {
g := hostbackoff.New(hostbackoff.Config{Step: 100 * time.Millisecond})
req := &Request{options: &protocols.ExecutorOptions{HostBackoff: g}}
target := "https://example.com/x"
key := hostratelimit.NormalizeHostPort(target)
req.observeHostBackoff(target, nil, context.Canceled)
require.Zero(t, g.Delay(key))
resp := &http.Response{StatusCode: http.StatusTooManyRequests, Header: http.Header{}}
req.observeHostBackoff(target, resp, context.Canceled)
require.Equal(t, 100*time.Millisecond, g.Delay(key))
}
func TestHostBackoffKeyKeepsIPv6OnHTTPS(t *testing.T) {
raw, err := retryablehttp.NewRequest(http.MethodGet, "https://[2001:db8::1]/secret", nil)
require.NoError(t, err)
key := hostratelimit.NormalizeHostPort(hostBackoffKey(&generatedRequest{request: raw}, nil))
// URL.Host has no port, so the old observation key took the http default.
require.NotEqual(t, key, hostratelimit.NormalizeHostPort(raw.URL.Host))
require.Contains(t, key, "443")
}
func TestHostBackoffKeyFallsBackToInput(t *testing.T) {
input := contextargs.NewWithInput(context.Background(), "https://example.com/from-input")
require.Equal(t, "https://example.com/from-input", hostBackoffKey(nil, input))
require.Equal(t, "example.com:443", hostratelimit.NormalizeHostPort(hostBackoffKey(nil, input)))
}
// A pipeline or unsafe request stored parsed.Host, which is empty when ParseURL
// fails or when it clears a single-label host. The wait used the original URL.
func TestParsedHostCanMissTheWaitKey(t *testing.T) {
_, err := urlutil.ParseURL("http:///no-host", true)
require.Error(t, err)
parsed, err := urlutil.ParseURL("http://intranet/admin", true)
require.NoError(t, err)
require.Empty(t, parsed.Host)
require.Equal(t, "intranet:80", hostratelimit.NormalizeHostPort("http://intranet/admin"))
}
func TestWaitHostBackoffStopsWhenHandlerTriggers(t *testing.T) {
g := hostbackoff.New(hostbackoff.Config{Step: 2 * time.Second, Max: 2 * time.Second})
req := &Request{options: &protocols.ExecutorOptions{HostBackoff: g}}
target := "https://example.com/x"
req.observeHostBackoff(target, &http.Response{StatusCode: http.StatusTooManyRequests, Header: http.Header{}}, nil)
parent, cancelParent := context.WithCancel(context.Background())
defer cancelParent()
handler := httputils.NewNonBlockingSPMHandler[struct{}](parent, 1, true)
waitCtx, cancelWait := stopWithHandler(parent, handler)
defer cancelWait()
done := make(chan error, 1)
go func() {
done <- req.waitHostBackoff(waitCtx, target)
}()
time.Sleep(20 * time.Millisecond)
handler.Trigger()
select {
case err := <-done:
require.ErrorIs(t, err, context.Canceled)
case <-time.After(500 * time.Millisecond):
t.Fatal("backoff wait outlived the handler")
}
}
+23 -34
View File
@@ -4,7 +4,6 @@ import (
"bytes"
"fmt"
"math"
"net/http"
"strings"
"time"
@@ -21,6 +20,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/network/networkclientpool"
"github.com/projectdiscovery/nuclei/v3/pkg/types/scanstrategy"
@@ -122,8 +122,7 @@ type Request struct {
// examples:
// - name: Send requests using 10 concurrent threads
// value: 10
Threads int `yaml:"threads,omitempty" json:"threads,omitempty" jsonschema:"title=threads for sending requests,description=Threads specifies number of threads to use sending requests. This enables Connection Pooling"`
PayloadThreads int `yaml:"payload-threads,omitempty" json:"payload-threads,omitempty" jsonschema:"title=dependent payload concurrency,description=Opt-in concurrency across independent payload groups while preserving request order inside each group"`
Threads int `yaml:"threads,omitempty" json:"threads,omitempty" jsonschema:"title=threads for sending requests,description=Threads specifies number of threads to use sending requests. This enables Connection Pooling"`
// description: |
// MaxSize is the maximum size of http response body to read in bytes.
// examples:
@@ -139,15 +138,13 @@ type Request struct {
CompiledOperators *operators.Operators `yaml:"-" json:"-"`
options *protocols.ExecutorOptions
connConfiguration *httpclientpool.Configuration
totalRequests int
customHeaders map[string]string
generator *generators.PayloadGenerator // optional, only enabled when using payloads
dependentPayloadThreads int
rawhttpClient *rawhttp.Client
replayClient *http.Client // set when -replay-proxy is given
dialer *fastdialer.Dialer
options *protocols.ExecutorOptions
connConfiguration *httpclientpool.Configuration
totalRequests int
customHeaders map[string]string
generator *generators.PayloadGenerator // optional, only enabled when using payloads
rawhttpClient *rawhttp.Client
dialer *fastdialer.Dialer
// description: |
// SelfContained specifies if the request is self-contained.
@@ -434,13 +431,6 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
}
request.rawhttpClient = httpclientpool.GetRawHTTP(options)
}
if options.Options != nil && options.Options.ReplayProxy != "" {
client, err := newReplayClient(options.Options)
if err != nil {
return err
}
request.replayClient = client
}
if len(request.Matchers) > 0 || len(request.Extractors) > 0 {
compiled := &request.Operators
compiled.ExcludeMatchers = options.ExcludeMatchers
@@ -522,7 +512,6 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
delete(request.Payloads, payload)
}
request.dependentPayloadThreads = 0
if len(request.Payloads) > 0 {
request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Catalog, request.options.Options.AttackType, request.options.Options)
if err != nil {
@@ -549,8 +538,12 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
}
}
if len(request.Payloads) > 0 {
// Multi-request matchers and dynamic extractors depend on ordered responses
// across a payload iteration, so they use the standard engine.
// Due to a known issue (https://github.com/projectdiscovery/nuclei/issues/5015),
// dynamic extractors cannot be used with payloads. To address this,
// execution is handled by the standard engine without concurrency,
// achieved by setting the thread count to 0.
// this limitation will be removed once we have a better way to handle dynamic extractors with payloads
hasMultipleRequests := false
if len(request.Raw)+len(request.Path) > 1 {
hasMultipleRequests = true
@@ -563,21 +556,17 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
break
}
}
if hasMultipleRequests && (hasNamedInternalExtractor || request.NeedsRequestCondition()) {
if !hasNamedInternalExtractor && request.NeedsRequestCondition() && request.PayloadThreads > 0 && request.canRunDependentPayloadsConcurrently() {
request.dependentPayloadThreads = request.PayloadThreads
}
if request.dependentPayloadThreads == 0 {
stats.Increment(SetThreadToCountZero)
}
if hasNamedInternalExtractor && hasMultipleRequests {
stats.Increment(SetThreadToCountZero)
request.Threads = 0
} else {
// specifically for http requests high concurrency and threads will lead to memory exhaustion, hence reduce the maximum parallelism
if protocolstate.IsLowOnMemory() {
request.Threads = protocolstate.GuardThreadsOrDefault(request.Threads)
}
request.Threads = options.GetThreadsForNPayloadRequests(request.Requests(), request.Threads)
}
}
if request.PayloadThreads > 0 && request.dependentPayloadThreads == 0 {
return errors.New("payload-threads requires multiple raw requests, indexed response conditions, and independent payload groups without iterate-all, interactsh, or @once")
}
// Avoid reusing client-side HTTP proxy connections for the legacy non-threaded spray path.
if shouldDisableKeepAliveForHTTPProxy(request, options) {
@@ -624,7 +613,7 @@ const (
)
func init() {
stats.NewEntry(SetThreadToCountZero, "Setting thread count to 0 for %d templates with payloads and dependent requests")
stats.NewEntry(SetThreadToCountZero, "Setting thread count to 0 for %d templates, dynamic extractors are not supported with payloads yet")
}
// UpdateOptions replaces this request's options with a new copy
@@ -643,7 +632,7 @@ func shouldDisableKeepAliveForHTTPProxy(request *Request, options *protocols.Exe
if request == nil || options == nil || options.Options == nil || options.Options.AliveHttpProxy == "" {
return false
}
return request.Threads <= 0 && request.dependentPayloadThreads <= 0 && options.Options.ScanStrategy != scanstrategy.HostSpray.String()
return request.Threads <= 0 && options.Options.ScanStrategy != scanstrategy.HostSpray.String()
}
// AnalyzeConnectionReuse determines if a request can safely reuse connections.
+14 -25
View File
@@ -394,9 +394,21 @@ func wrappedGet(options *types.Options, configuration *Configuration, host strin
transportKey := transportHash(host, disableKeepAlives, maxIdleConns, maxIdleConnsPerHost, maxConnsPerHost, responseHeaderTimeout)
createTransport := func() (http.RoundTripper, error) {
tlsConfig, err := NewTLSConfig(options)
// Set the base TLS configuration definition
tlsConfig := &tls.Config{
Renegotiation: tls.RenegotiateOnceAsClient,
InsecureSkipVerify: true,
MinVersion: tls.VersionTLS10,
ClientSessionCache: sharedTLSSessionCache,
}
if options.SNI != "" {
tlsConfig.ServerName = options.SNI
}
tlsConfig, err := utils.AddConfiguredClientCertToRequest(tlsConfig, options)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "could not create client certificate")
}
transport := &http.Transport{
@@ -741,26 +753,3 @@ func RecordHTTPToHTTPSPortMismatch(options *types.Options, hostname string) {
tracker.RecordHTTPToHTTPSPort(hostname)
}
// NewTLSConfig returns the TLS configuration HTTP requests to scan targets use,
// honoring -sni and client certificates.
func NewTLSConfig(options *types.Options) (*tls.Config, error) {
tlsConfig := &tls.Config{
Renegotiation: tls.RenegotiateOnceAsClient,
// Scan targets present untrusted certificates. This is the same configuration the HTTP client used inline.
InsecureSkipVerify: true, // codeql[go/disabled-certificate-check]
// Keep the scan client's existing minimum so older targets still connect.
MinVersion: tls.VersionTLS10, // codeql[go/insecure-tls]
ClientSessionCache: sharedTLSSessionCache,
}
if options.SNI != "" {
tlsConfig.ServerName = options.SNI
}
tlsConfig, err := utils.AddConfiguredClientCertToRequest(tlsConfig, options)
if err != nil {
return nil, errors.Wrap(err, "could not create client certificate")
}
return tlsConfig, nil
}
+10 -20
View File
@@ -58,7 +58,7 @@ func NewBlockingSPMHandler[T comparable](ctx context.Context, size int, maxResul
maxResults: maxResults,
}
s.internalWg.Add(1)
go s.run(ctx1)
go s.run(ctx)
return s
}
@@ -77,7 +77,7 @@ func NewNonBlockingSPMHandler[T comparable](ctx context.Context, maxResults int,
maxResults: maxResults,
}
s.internalWg.Add(1)
go s.run(ctx1)
go s.run(ctx)
return s
}
@@ -132,25 +132,18 @@ func (h *StopAtFirstMatchHandler[T]) run(ctx context.Context) {
if !ok {
return
}
h.recordResult(val)
if h.onResult != nil {
h.onResult(val)
}
if len(h.results) >= h.maxResults {
// skip or do not store the result
continue
}
h.results[val] = struct{}{}
}
}
}
func (h *StopAtFirstMatchHandler[T]) recordResult(val T) {
if h.onResult != nil {
h.onResult(val)
}
if len(h.results) < h.maxResults {
h.results[val] = struct{}{}
}
}
// Context is canceled when the handler stops.
func (h *StopAtFirstMatchHandler[T]) Context() context.Context {
return h.ctx
}
// Done returns a channel with the context done signal when stop at first match is detected
func (h *StopAtFirstMatchHandler[T]) Done() <-chan struct{} {
return h.ctx.Done()
@@ -212,9 +205,6 @@ func (h *StopAtFirstMatchHandler[T]) Wait() {
// after waiting it closes the error channel
close(h.ResultChan)
h.internalWg.Wait()
for val := range h.ResultChan {
h.recordResult(val)
}
}
// CombinedResults returns the combined results
-79
View File
@@ -1,79 +0,0 @@
package httputils
import (
"context"
"testing"
"time"
)
func TestCancelStopsResultReceiver(t *testing.T) {
constructors := map[string]func(context.Context) *StopAtFirstMatchHandler[int]{
"blocking": func(ctx context.Context) *StopAtFirstMatchHandler[int] {
return NewBlockingSPMHandler[int](ctx, 1, 1, true)
},
"nonblocking": func(ctx context.Context) *StopAtFirstMatchHandler[int] {
return NewNonBlockingSPMHandler[int](ctx, 1, true)
},
}
for name, construct := range constructors {
t.Run(name, func(t *testing.T) {
parent, cancelParent := context.WithCancel(context.Background())
defer cancelParent()
handler := construct(parent)
done := make(chan struct{})
go func() {
handler.internalWg.Wait()
close(done)
}()
handler.Cancel()
select {
case <-done:
case <-time.After(time.Second):
t.Fatal("result receiver remained alive after cancellation")
}
})
}
}
func TestCancelPreservesQueuedResults(t *testing.T) {
constructors := map[string]func() *StopAtFirstMatchHandler[int]{
"blocking": func() *StopAtFirstMatchHandler[int] {
return NewBlockingSPMHandler[int](context.Background(), 1, 2, false)
},
"nonblocking": func() *StopAtFirstMatchHandler[int] {
return NewNonBlockingSPMHandler[int](context.Background(), 2, false)
},
}
for name, construct := range constructors {
t.Run(name, func(t *testing.T) {
for range 25 {
handler := construct()
entered := make(chan struct{})
release := make(chan struct{})
handler.SetOnResultCallback(func(value int) {
if value == 1 {
close(entered)
<-release
}
})
handler.ResultChan <- 1
<-entered
handler.ResultChan <- 2
handler.Cancel()
close(release)
handler.Wait()
found := make(map[int]bool)
for _, result := range handler.CombinedResults() {
found[result] = true
}
if !found[1] || !found[2] {
t.Fatalf("queued results lost after cancellation: %v", found)
}
}
})
}
}
@@ -13,8 +13,6 @@ func TestShouldDisableKeepAliveForHTTPProxy(t *testing.T) {
opts.AliveHttpProxy = "http://127.0.0.1:8080"
opts.ScanStrategy = scanstrategy.TemplateSpray.String()
require.True(t, shouldDisableKeepAliveForHTTPProxy(&Request{}, &protocols.ExecutorOptions{Options: opts}))
require.False(t, shouldDisableKeepAliveForHTTPProxy(&Request{Threads: 10}, &protocols.ExecutorOptions{Options: opts}))
require.False(t, shouldDisableKeepAliveForHTTPProxy(&Request{dependentPayloadThreads: 2}, &protocols.ExecutorOptions{Options: opts}))
opts.ScanStrategy = scanstrategy.HostSpray.String()
require.False(t, shouldDisableKeepAliveForHTTPProxy(&Request{}, &protocols.ExecutorOptions{Options: opts}))
-77
View File
@@ -1,77 +0,0 @@
package http
import (
"bufio"
"bytes"
"io"
"net/http"
"net/url"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/utils/errkit"
)
// newReplayClient builds the client that re-sends findings through proxy. It
// uses the scan's TLS configuration so a replay reaches the target the same way.
func newReplayClient(options *types.Options) (*http.Client, error) {
proxyURL, err := url.Parse(options.ReplayProxy)
if err != nil {
return nil, errkit.Wrapf(err, "invalid replay proxy %q", options.ReplayProxy)
}
tlsConfig, err := httpclientpool.NewTLSConfig(options)
if err != nil {
return nil, err
}
return &http.Client{
Timeout: options.GetTimeouts().HttpTimeout,
Transport: &http.Transport{
Proxy: http.ProxyURL(proxyURL),
TLSClientConfig: tlsConfig,
// keep the request as sent, without an added Accept-Encoding
DisableCompression: true,
},
// the proxy must record the request that matched, not where it redirects
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
}, nil
}
// replayRequest re-sends a request that produced a finding through the replay
// proxy, so it lands in a tool such as Burp without routing the whole scan
// through it. The response is discarded and failures are only logged: a replay
// must never affect the scan.
func (request *Request) replayRequest(dumpedRequest []byte, formedURL string) {
if request.replayClient == nil {
return
}
dump := bufio.NewReader(bytes.NewReader(dumpedRequest))
replay, err := http.ReadRequest(dump)
if err != nil {
gologger.Debug().Msgf("[%s] could not parse request to replay: %s", request.options.TemplateID, err)
return
}
// unsafe raw requests are dumped as written and may carry a body without
// declaring its length; ReadRequest would then treat them as bodyless
if replay.ContentLength <= 0 && len(replay.TransferEncoding) == 0 {
if rest, _ := io.ReadAll(dump); len(rest) > 0 {
replay.Body = io.NopCloser(bytes.NewReader(rest))
replay.ContentLength = int64(len(rest))
}
}
target, err := url.Parse(formedURL)
if err != nil {
gologger.Debug().Msgf("[%s] could not parse URL to replay %q: %s", request.options.TemplateID, formedURL, err)
return
}
replay.RequestURI = ""
replay.URL = target
resp, err := request.replayClient.Do(replay)
if err != nil {
gologger.Debug().Msgf("[%s] could not replay %s through %s: %s", request.options.TemplateID, formedURL, request.options.Options.ReplayProxy, err)
return
}
_, _ = io.Copy(io.Discard, resp.Body)
_ = resp.Body.Close()
}
-359
View File
@@ -1,359 +0,0 @@
package http
import (
"context"
"io"
"net/http"
"net/http/httptest"
neturl "net/url"
"sync"
"testing"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
"github.com/projectdiscovery/nuclei/v3/pkg/authprovider"
"github.com/projectdiscovery/nuclei/v3/pkg/authprovider/authx"
"github.com/projectdiscovery/nuclei/v3/pkg/model"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v3/pkg/operators"
"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/globalmatchers"
urlutil "github.com/projectdiscovery/utils/url"
"github.com/stretchr/testify/require"
)
var _ authprovider.AuthProvider = staticReplayAuth{}
type replayedRequest struct {
method, url, header, body string
}
func TestReplayProxyResendsOnlyMatchedRequests(t *testing.T) {
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/hit" {
_, _ = w.Write([]byte("match-me"))
return
}
_, _ = w.Write([]byte("nothing here"))
}))
defer target.Close()
// a forward proxy receives absolute-form request targets for http URLs
var mu sync.Mutex
var replayed []replayedRequest
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, _ := io.ReadAll(r.Body)
mu.Lock()
replayed = append(replayed, replayedRequest{method: r.Method, url: r.URL.String(), header: r.Header.Get("X-Marker"), body: string(body)})
mu.Unlock()
}))
defer proxy.Close()
options := testutils.DefaultOptions.Copy()
options.SetExecutionID(t.Name())
options.ReplayProxy = proxy.URL
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
templateID := "replay-proxy"
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: templateID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
request := &Request{
ID: templateID,
Method: HTTPMethodTypeHolder{MethodType: HTTPPost},
Path: []string{"{{BaseURL}}/hit", "{{BaseURL}}/miss"},
Body: "a=1&b=2",
Headers: map[string]string{
"X-Marker": "replayed",
},
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
Words: []string{"match-me"},
}},
},
}
require.NoError(t, request.Compile(executerOpts))
var findings int
input := contextargs.NewWithInput(context.Background(), target.URL)
err := request.ExecuteWithResults(input, output.InternalEvent{}, output.InternalEvent{}, func(event *output.InternalWrappedEvent) {
if event.OperatorsResult != nil && event.OperatorsResult.Matched {
findings++
}
})
require.NoError(t, err)
require.Equal(t, 1, findings)
mu.Lock()
defer mu.Unlock()
require.Equal(t, []replayedRequest{{method: http.MethodPost, url: target.URL + "/hit", header: "replayed", body: "a=1&b=2"}}, replayed)
}
func TestReplayProxyUnsetSendsNothing(t *testing.T) {
options := testutils.DefaultOptions.Copy()
options.SetExecutionID(t.Name())
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ID: "replay-unset"})
request := &Request{ID: "replay-unset", Path: []string{"{{BaseURL}}/"}}
require.NoError(t, request.Compile(executerOpts))
require.Nil(t, request.replayClient)
}
func TestReplayRequestKeepsBodyWithoutContentLength(t *testing.T) {
var got replayedRequest
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, _ := io.ReadAll(r.Body)
got = replayedRequest{method: r.Method, url: r.URL.String(), body: string(body)}
}))
defer proxy.Close()
options := testutils.DefaultOptions.Copy()
options.ReplayProxy = proxy.URL
client, err := newReplayClient(options)
require.NoError(t, err)
request := &Request{replayClient: client, options: &protocols.ExecutorOptions{Options: options}}
// unsafe raw requests are dumped as written, here without Content-Length
request.replayRequest([]byte("POST /hit HTTP/1.1\r\nHost: target.example\r\n\r\nx=1"), "http://target.example/hit")
require.Equal(t, replayedRequest{method: http.MethodPost, url: "http://target.example/hit", body: "x=1"}, got)
}
func TestReplayProxySendsRedirectChainOnce(t *testing.T) {
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/start" {
w.Header().Set("Location", "/next")
w.WriteHeader(http.StatusFound)
_, _ = w.Write([]byte("match-me"))
return
}
_, _ = w.Write([]byte("match-me"))
}))
defer target.Close()
var mu sync.Mutex
var replayed []string
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mu.Lock()
replayed = append(replayed, r.URL.String())
mu.Unlock()
}))
defer proxy.Close()
options := testutils.DefaultOptions.Copy()
options.SetExecutionID(t.Name())
options.ReplayProxy = proxy.URL
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: "replay-redirect",
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
request := &Request{
ID: "replay-redirect",
Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
Path: []string{"{{BaseURL}}/start"},
Redirects: true,
MaxRedirects: 5,
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
Words: []string{"match-me"},
}},
},
}
require.NoError(t, request.Compile(executerOpts))
input := contextargs.NewWithInput(context.Background(), target.URL)
require.NoError(t, request.ExecuteWithResults(input, output.InternalEvent{}, output.InternalEvent{}, func(*output.InternalWrappedEvent) {}))
mu.Lock()
defer mu.Unlock()
require.Equal(t, []string{target.URL + "/start"}, replayed)
}
func TestReplayProxyReplaysRaceRequest(t *testing.T) {
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte("match-me"))
}))
defer target.Close()
var mu sync.Mutex
var replayed []replayedRequest
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, _ := io.ReadAll(r.Body)
mu.Lock()
replayed = append(replayed, replayedRequest{method: r.Method, url: r.URL.String(), body: string(body)})
mu.Unlock()
}))
defer proxy.Close()
options := testutils.DefaultOptions.Copy()
options.SetExecutionID(t.Name())
options.ReplayProxy = proxy.URL
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: "replay-race",
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
request := &Request{
ID: "replay-race",
Method: HTTPMethodTypeHolder{MethodType: HTTPPost},
Path: []string{"{{BaseURL}}/race"},
Body: "race=1",
Race: true,
RaceNumberRequests: 1,
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
Words: []string{"match-me"},
}},
},
}
require.NoError(t, request.Compile(executerOpts))
input := contextargs.NewWithInput(context.Background(), target.URL)
require.NoError(t, request.ExecuteWithResults(input, output.InternalEvent{}, output.InternalEvent{}, func(*output.InternalWrappedEvent) {}))
mu.Lock()
defer mu.Unlock()
require.Equal(t, []replayedRequest{{method: http.MethodPost, url: target.URL + "/race", body: "race=1"}}, replayed)
}
func TestReplayRequestAddsNoHeaders(t *testing.T) {
var got http.Header
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
got = r.Header.Clone()
}))
defer proxy.Close()
options := testutils.DefaultOptions.Copy()
options.ReplayProxy = proxy.URL
client, err := newReplayClient(options)
require.NoError(t, err)
request := &Request{replayClient: client, options: &protocols.ExecutorOptions{Options: options}}
request.replayRequest([]byte("GET /hit HTTP/1.1\r\nHost: target.example\r\nUser-Agent: nuclei\r\n\r\n"), "http://target.example/hit")
require.Empty(t, got.Get("Accept-Encoding"))
}
func TestReplayProxyReplaysGlobalMatcher(t *testing.T) {
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte("match-me"))
}))
defer target.Close()
var mu sync.Mutex
var replayed []string
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mu.Lock()
replayed = append(replayed, r.URL.String())
mu.Unlock()
}))
defer proxy.Close()
options := testutils.DefaultOptions.Copy()
options.SetExecutionID(t.Name())
options.ReplayProxy = proxy.URL
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: "replay-global",
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
global := &operators.Operators{Matchers: []*matchers.Matcher{{
Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
Words: []string{"match-me"},
}}}
require.NoError(t, global.Compile())
executerOpts.GlobalMatchers = globalmatchers.New()
executerOpts.GlobalMatchers.AddOperator(&globalmatchers.Item{
TemplateID: "global",
Operators: []*operators.Operators{global},
})
request := &Request{
ID: "replay-global",
Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
Path: []string{"{{BaseURL}}/hit"},
}
require.NoError(t, request.Compile(executerOpts))
input := contextargs.NewWithInput(context.Background(), target.URL)
require.NoError(t, request.ExecuteWithResults(input, output.InternalEvent{}, output.InternalEvent{}, func(*output.InternalWrappedEvent) {}))
mu.Lock()
defer mu.Unlock()
require.Equal(t, []string{target.URL + "/hit"}, replayed)
}
func TestReplayProxyReplaysRaceRequestWithAuth(t *testing.T) {
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte("match-me"))
}))
defer target.Close()
var mu sync.Mutex
var auth []string
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mu.Lock()
auth = append(auth, r.Header.Get("Authorization"))
mu.Unlock()
}))
defer proxy.Close()
options := testutils.DefaultOptions.Copy()
options.SetExecutionID(t.Name())
options.ReplayProxy = proxy.URL
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: "replay-race-auth",
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
executerOpts.AuthProvider = staticReplayAuth{authx.NewBearerTokenAuthStrategy(&authx.Secret{Token: "session-token"})}
request := &Request{
ID: "replay-race-auth",
Method: HTTPMethodTypeHolder{MethodType: HTTPPost},
Path: []string{"{{BaseURL}}/race"},
Body: "race=1",
Race: true,
RaceNumberRequests: 1,
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Type: matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
Words: []string{"match-me"},
}},
},
}
require.NoError(t, request.Compile(executerOpts))
input := contextargs.NewWithInput(context.Background(), target.URL)
require.NoError(t, request.ExecuteWithResults(input, output.InternalEvent{}, output.InternalEvent{}, func(*output.InternalWrappedEvent) {}))
mu.Lock()
defer mu.Unlock()
require.Equal(t, []string{"Bearer session-token"}, auth)
}
type staticReplayAuth struct {
strategy authx.AuthStrategy
}
func (s staticReplayAuth) LookupAddr(string) []authx.AuthStrategy { return nil }
func (s staticReplayAuth) LookupURL(*neturl.URL) []authx.AuthStrategy {
return []authx.AuthStrategy{s.strategy}
}
func (s staticReplayAuth) LookupURLX(*urlutil.URL) []authx.AuthStrategy {
return []authx.AuthStrategy{s.strategy}
}
func (s staticReplayAuth) GetTemplatePaths() []string { return nil }
func (s staticReplayAuth) PreFetchSecrets() error { return nil }
+51 -378
View File
@@ -4,7 +4,6 @@ import (
"bytes"
"context"
"encoding/hex"
stderrors "errors"
"fmt"
"io"
"maps"
@@ -19,7 +18,6 @@ import (
"github.com/pkg/errors"
"go.uber.org/multierr"
"golang.org/x/sync/semaphore"
"moul.io/http2curl"
"github.com/projectdiscovery/fastdialer/fastdialer"
@@ -36,8 +34,6 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hostbackoff"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hostratelimit"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
@@ -52,6 +48,7 @@ import (
convUtil "github.com/projectdiscovery/utils/conversion"
"github.com/projectdiscovery/utils/errkit"
httpUtils "github.com/projectdiscovery/utils/http"
"github.com/projectdiscovery/utils/reader"
sliceutil "github.com/projectdiscovery/utils/slice"
stringsutil "github.com/projectdiscovery/utils/strings"
unitutils "github.com/projectdiscovery/utils/unit"
@@ -78,91 +75,6 @@ func (request *Request) Type() templateTypes.ProtocolType {
return templateTypes.HTTPProtocol
}
// hostBackoffKey is the identity shared by the pre-request wait and the
// post-response observation. It keeps the scheme, or an explicit port, so
// NormalizeHostPort does not record an https target under port 80.
func hostBackoffKey(req *generatedRequest, input *contextargs.Context) string {
if req != nil {
if key := req.URL(); key != "" {
return key
}
if req.request != nil && req.request.Request != nil && req.request.Request.URL != nil {
return req.request.Request.URL.String()
}
}
if input != nil && input.MetaInput != nil {
return input.MetaInput.Input
}
return ""
}
// observeHostBackoff reports one request outcome to the per-host governor.
func (request *Request) observeHostBackoff(host string, resp *http.Response, err error) {
if request.options == nil || request.options.HostBackoff == nil {
return
}
// A cancelled sibling, or a scan that is stopping, is not the host asking
// for less traffic. A real status on the response is still worth recording.
if resp == nil && err != nil && stderrors.Is(err, context.Canceled) {
return
}
statusCode := 0
var retryAfter time.Duration
if resp != nil {
statusCode = resp.StatusCode
retryAfter = hostbackoff.RetryAfter(resp.Header.Get("Retry-After"))
}
request.options.HostBackoff.Observe(hostratelimit.NormalizeHostPort(host), statusCode, retryAfter, err)
}
// waitHostBackoff pauses for the delay this host has already earned.
func (request *Request) waitHostBackoff(ctx context.Context, host string) error {
if request.options == nil {
return nil
}
return request.options.HostBackoffWait(ctx, host)
}
type httpRateLimitWaitObserver interface {
ObserveHTTPRateLimitWait(time.Duration)
}
type httpRequestTimingObserver interface {
ObserveHTTPRequest(time.Duration, int, error)
}
// rateLimitTake first honours any backoff the host earned by signalling it is
// overloaded, so a blocking target is paced regardless of which limiter is in play.
func (request *Request) rateLimitTake(ctx context.Context, hostname string) error {
if err := request.waitHostBackoff(ctx, hostname); err != nil {
return err
}
started := time.Now()
err := request.options.RateLimitTakeContext(ctx, hostname)
if observer, ok := request.options.Progress.(httpRateLimitWaitObserver); ok {
observer.ObserveHTTPRateLimitWait(time.Since(started))
}
return err
}
// stopWithHandler ends when parent ends or the handler stops, so a backoff
// wait does not outlive stop-at-first-match.
func stopWithHandler[T comparable](parent context.Context, handler *httputils.StopAtFirstMatchHandler[T]) (context.Context, context.CancelFunc) {
ctx, cancel := context.WithCancel(parent)
if handler == nil {
return ctx, cancel
}
if handler.Context().Err() != nil {
cancel()
return ctx, func() {}
}
stop := context.AfterFunc(handler.Context(), cancel)
return ctx, func() {
stop()
cancel()
}
}
// executeRaceRequest executes race condition request for a URL
func (request *Request) executeRaceRequest(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
reqURL := input.MetaInput.Input
@@ -182,11 +94,6 @@ func (request *Request) executeRaceRequest(input *contextargs.Context, dynamicVa
return err
}
request.setCustomHeaders(requestForDump)
// Auth is applied again on each raced request. The dump is what replay
// sends, so it has to include the same headers.
if !request.SkipSecretFile {
requestForDump.ApplyAuth(request.options.AuthProvider)
}
dumpedRequest, err := dump(requestForDump, reqURL)
if err != nil {
return err
@@ -260,20 +167,8 @@ func (request *Request) executeRaceRequest(input *contextargs.Context, dynamicVa
// stop sending more requests condition is met
return
}
waitCtx, cancelWait := stopWithHandler(requestInput.Context(), spmHandler)
err := request.waitHostBackoff(waitCtx, hostBackoffKey(httpRequest, requestInput))
cancelWait()
if err != nil || spmHandler.FoundFirstMatch() || spmHandler.Cancelled() {
if err != nil {
select {
case <-spmHandler.Done():
case spmHandler.ResultChan <- err:
}
}
return
}
err = request.executeRequest(requestInput, httpRequest, previous, false, wrappedCallback, 0)
err := request.executeRequest(requestInput, httpRequest, previous, false, wrappedCallback, 0)
select {
case <-spmHandler.Done():
return
@@ -362,12 +257,15 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
spmHandler.Release()
continue
}
// Full URL, so https and an explicit port stay on the same key the observation uses.
// The handler context ends the wait when stop-at-first-match fires.
waitCtx, cancelWait := stopWithHandler(ctx, spmHandler)
err := request.rateLimitTake(waitCtx, hostBackoffKey(t.req, t.updatedInput))
cancelWait()
if err != nil {
// Extract hostname for per-host rate limiting (use full URL - normalization happens in rateLimitTake)
hostname := t.updatedInput.MetaInput.Input
if t.req != nil && t.req.URL() != "" {
hostname = t.req.URL()
} else if t.req != nil && t.req.request != nil && t.req.request.Request != nil && t.req.request.Request.URL != nil {
// Extract from request URL if available
hostname = t.req.request.Request.URL.String()
}
if err := request.options.RateLimitTakeContext(ctx, hostname); err != nil {
select {
case <-spmHandler.Done():
spmHandler.Release()
@@ -386,7 +284,7 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
}
hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators)
needsRequestEvent := hasInteractMatchers && request.NeedsRequestCondition()
err = request.executeRequest(t.updatedInput, t.req, make(map[string]interface{}), hasInteractMatchers, func(event *output.InternalWrappedEvent) {
err := request.executeRequest(t.updatedInput, t.req, make(map[string]interface{}), hasInteractMatchers, func(event *output.InternalWrappedEvent) {
if (t.hasInteractMarkers || needsRequestEvent) && request.options.Interactsh != nil {
requestData := &interactsh.RequestData{
MakeResultFunc: request.MakeResultEvent,
@@ -433,12 +331,8 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
}
// resize check point - nop if there are no changes
desiredWorkers := request.options.Options.PayloadConcurrency
if protocolstate.IsLowOnMemory() {
desiredWorkers = protocolstate.GuardThreadsOrDefault(desiredWorkers)
}
if shouldFollowGlobal && spmHandler.Size() != desiredWorkers {
if err := spmHandler.Resize(input.Context(), desiredWorkers); err != nil {
if shouldFollowGlobal && spmHandler.Size() != request.options.Options.PayloadConcurrency {
if err := spmHandler.Resize(input.Context(), request.options.Options.PayloadConcurrency); err != nil {
close(tasks)
workersWg.Wait()
return err
@@ -611,19 +505,7 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
// skip if first match is found
return
}
waitCtx, cancelWait := stopWithHandler(requestInput.Context(), spmHandler)
err := request.waitHostBackoff(waitCtx, hostBackoffKey(httpRequest, requestInput))
cancelWait()
if err != nil || spmHandler.FoundFirstMatch() || spmHandler.Cancelled() {
if err != nil {
select {
case <-spmHandler.Done():
case spmHandler.ResultChan <- err:
}
}
return
}
err = request.executeRequest(requestInput, httpRequest, previous, false, wrappedCallback, 0)
err := request.executeRequest(requestInput, httpRequest, previous, false, wrappedCallback, 0)
select {
case <-spmHandler.Done():
return
@@ -662,62 +544,29 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
if request.Threads > 0 && (len(request.Payloads) > 0 || request.Race) {
return request.executeParallelHTTP(input, dynamicValues, callback)
}
if request.dependentPayloadThreads > 0 {
return request.executeDependentPayloads(input, dynamicValues, previous, callback)
}
return request.executeSequentialHTTP(input, dynamicValues, previous, callback, request.newGenerator(false), nil, nil)
}
func (request *Request) executeSequentialHTTP(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback, generator *requestGenerator, accounted *atomic.Int64, stopOnMatch *atomic.Bool) error {
var recordedCurrent bool
countAccounted := func(count int64) {
if accounted != nil {
accounted.Add(count)
}
}
recordSkipped := func(count int64) {
countAccounted(count)
request.options.Progress.SetRequests(uint64(count))
}
recordSent := func() {
recordedCurrent = true
countAccounted(1)
request.options.Progress.IncrementRequests()
}
recordFailed := func() {
recordedCurrent = true
countAccounted(1)
request.options.Progress.IncrementFailedRequestsBy(1)
}
recordStopped := func() {
remaining := int64(generator.Remaining())
if !recordedCurrent {
remaining++
}
recordSkipped(remaining)
}
generator := request.newGenerator(false)
initialPrevious := cloneInternalEvent(previous)
initialContext := dynamicValueContext{
values: generators.CopyMap(dynamicValues),
previous: cloneInternalEvent(initialPrevious),
previous: cloneInternalEvent(previous),
}
dynamicValueContexts := []dynamicValueContext{initialContext}
if request.IterateAll {
dynamicValueContexts = expandDynamicValueContexts(initialContext, generators.MergeMapsMany(dynamicValues), true)
}
oncePrevious := make(output.InternalEvent)
lastRequestIndex := 0
var requestErr error
executionLoop:
for {
executeFunc := func(data string, payloads, dynamicValue map[string]interface{}, previousEvent output.InternalEvent) (skipBranch, stop bool, extracted map[string][]string, err error) {
recordedCurrent = false
hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators)
ctx := request.newContext(input)
generatedHttpRequest, err := generator.Make(ctx, input, data, payloads, dynamicValue)
ctxWithTimeout, cancel := context.WithTimeoutCause(ctx, request.options.Options.GetTimeouts().HttpTimeout, ErrHttpEngineRequestDeadline)
defer cancel()
generatedHttpRequest, err := generator.Make(ctxWithTimeout, input, data, payloads, dynamicValue)
if err != nil {
if err == types.ErrNoMoreRequests {
return false, true, nil, nil
@@ -730,22 +579,19 @@ executionLoop:
// a copy and using it to check for host errors etc
// but this should be replaced once templateCtx is refactored properly
updatedInput := contextargs.GetCopyIfHostOutdated(input, generatedHttpRequest.URL())
if generatedHttpRequest.customCancelFunction != nil {
defer generatedHttpRequest.customCancelFunction()
}
if request.isUnresponsiveAddress(updatedInput) {
return false, true, nil, nil
}
// Full URL, so https and an explicit port stay on the same key the observation uses.
if err := request.rateLimitTake(ctx, hostBackoffKey(generatedHttpRequest, input)); err != nil {
// Extract hostname for per-host rate limiting (use generated request URL - normalization happens in rateLimitTake)
hostname := input.MetaInput.Input
if generatedHttpRequest.URL() != "" {
// Use the generated URL directly - the normalization function will extract host:port correctly
hostname = generatedHttpRequest.URL()
}
if err := request.options.RateLimitTakeContext(ctx, hostname); err != nil {
return false, true, nil, err
}
if generatedHttpRequest.request != nil && generatedHttpRequest.customCancelFunction == nil {
// Rate admission must not consume the network request's timeout.
timedCtx, cancel := context.WithTimeoutCause(generatedHttpRequest.request.Context(), request.options.Options.GetTimeouts().HttpTimeout, ErrHttpEngineRequestDeadline)
defer cancel()
generatedHttpRequest.request.WithContext(timedCtx)
if generatedHttpRequest.customCancelFunction != nil {
defer generatedHttpRequest.customCancelFunction()
}
hasInteractMarkers := interactsh.HasMarkers(data) || len(generatedHttpRequest.interactshURLs) > 0
@@ -796,9 +642,6 @@ executionLoop:
if errors.Is(execReqErr, ErrMissingVars) {
return true, false, nil, nil
}
if stopOnMatch != nil && stopOnMatch.Load() && errors.Is(execReqErr, context.Canceled) {
return false, true, nil, nil
}
request.recordHostResult(updatedInput, execReqErr)
if execReqErr != nil {
@@ -806,9 +649,9 @@ executionLoop:
reqKitErr.Msgf("got err while executing %v", generatedHttpRequest.URL())
requestErr = reqKitErr
recordFailed()
request.options.Progress.IncrementFailedRequestsBy(1)
} else {
recordSent()
request.options.Progress.IncrementRequests()
}
// If this was a match, and we want to stop at first match, skip all further requests.
@@ -823,13 +666,6 @@ executionLoop:
if !ok {
break
}
if generator.payloadIterator != nil && generator.currentIndex <= lastRequestIndex {
for i := range dynamicValueContexts {
dynamicValueContexts[i].previous = cloneInternalEvent(initialPrevious)
maps.Copy(dynamicValueContexts[i].previous, oncePrevious)
}
}
lastRequestIndex = generator.currentIndex
select {
case <-input.Context().Done():
@@ -857,11 +693,11 @@ executionLoop:
if requestErr == nil {
requestErr = gotErr
}
recordSkipped(int64(generator.Remaining() + 1))
request.options.Progress.SetRequests(uint64(generator.Remaining() + 1))
break executionLoop
}
if stop {
recordStopped()
request.options.Progress.SetRequests(uint64(generator.Remaining() + 1))
break executionLoop
}
if skipBranch {
@@ -876,7 +712,6 @@ executionLoop:
dynamicValueContexts = nil
continue
}
maps.Copy(oncePrevious, oncePreviousDelta)
nextDynamicValueContexts := make([]dynamicValueContext, 0, len(dynamicValueContexts))
for _, branch := range dynamicValueContexts {
@@ -900,11 +735,11 @@ executionLoop:
if requestErr == nil {
requestErr = gotErr
}
recordSkipped(int64(generator.Remaining() + 1))
request.options.Progress.SetRequests(uint64(generator.Remaining() + 1))
break executionLoop
}
if stop {
recordStopped()
request.options.Progress.SetRequests(uint64(generator.Remaining() + 1))
break executionLoop
}
if skipBranch {
@@ -917,119 +752,6 @@ executionLoop:
return requestErr
}
func (request *Request) canRunDependentPayloadsConcurrently() bool {
if request.IterateAll || request.Race || request.Pipeline || len(request.Fuzzing) > 0 || len(request.Raw) < 2 || interactsh.HasMatchers(request.CompiledOperators) {
return false
}
for _, raw := range request.Raw {
if interactsh.HasMarkers(raw) {
return false
}
if mark, hasMark := parseFlowAnnotations(raw); hasMark && mark == Once {
return false
}
}
return true
}
func (request *Request) executeDependentPayloads(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
workers := min(request.dependentPayloadThreads, defaultMaxWorkers)
if request.options.Options.PayloadConcurrency > 0 {
workers = min(workers, request.options.Options.PayloadConcurrency)
}
workerCtx, cancel := context.WithCancel(input.Context())
defer cancel()
permits := semaphore.NewWeighted(int64(workers))
shouldStop := request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch || request.options.StopAtFirstMatch
var found atomic.Bool
var callbackMu sync.Mutex
wrappedCallback := func(event *output.InternalWrappedEvent) {
callbackMu.Lock()
defer callbackMu.Unlock()
if shouldStop && found.Load() {
return
}
callback(event)
if shouldStop && event.OperatorsResult != nil && event.OperatorsResult.Matched {
found.Store(true)
cancel()
}
}
tasks := make(chan map[string]interface{}, workers)
var workersWG sync.WaitGroup
var errorsMu sync.Mutex
var executionErrors []error
var accounted atomic.Int64
for i := 0; i < workers; i++ {
workersWG.Add(1)
go func() {
defer workersWG.Done()
for payloads := range tasks {
if workerCtx.Err() != nil {
return
}
weight := int64(1)
if protocolstate.IsLowOnMemory() {
allowed := max(1, protocolstate.GuardThreadsOrDefault(workers))
weight = int64((workers + allowed - 1) / allowed)
}
if err := permits.Acquire(workerCtx, weight); err != nil {
return
}
generator := &requestGenerator{
request: request,
options: request.options,
currentPayloads: payloads,
onceFlow: make(map[string]struct{}),
}
groupInput := input.CloneWithContext(workerCtx)
err := request.executeSequentialHTTP(groupInput, dynamicValues, previous, wrappedCallback, generator, &accounted, &found)
permits.Release(weight)
if err != nil {
if found.Load() && errors.Is(err, context.Canceled) {
continue
}
errorsMu.Lock()
executionErrors = append(executionErrors, err)
errorsMu.Unlock()
}
}
}()
}
iterator := request.generator.NewIterator()
enqueue:
for {
payloads, ok := iterator.Value()
if !ok {
break
}
select {
case <-workerCtx.Done():
break enqueue
case tasks <- payloads:
}
}
close(tasks)
workersWG.Wait()
missing := int64(request.Requests()) - accounted.Load()
if missing < 0 {
return fmt.Errorf("dependent payload request accounting exceeded planned requests by %d", -missing)
}
if missing > 0 {
request.options.Progress.SetRequests(uint64(missing))
}
if found.Load() {
return nil
}
if err := input.Context().Err(); err != nil {
return err
}
return multierr.Combine(executionErrors...)
}
type dynamicValueContext struct {
values map[string]interface{}
seen map[string]map[string]struct{}
@@ -1165,25 +887,6 @@ func extractorAppliesToRequest(part string, requestIndex int) bool {
const drainReqSize = int64(8 * unitutils.Kilo)
// reuseRequestBody reads the body once and installs GetBody copies. The
// post-send dump reads one while the transport may still be writing another.
// GET, HEAD, and chunked requests keep their existing content length.
func reuseRequestBody(req *retryablehttp.Request) error {
body, err := io.ReadAll(req.Body)
if err != nil {
return err
}
keepLength := stringsutil.EqualFoldAny(req.Method, http.MethodGet, http.MethodHead) || req.Header.Get("Transfer-Encoding") == "chunked"
contentLength := req.ContentLength
if err = req.SetBody(body); err != nil {
return err
}
if keepLength {
req.ContentLength = contentLength
}
return nil
}
// executeRequest executes the actual generated request and returns error if occurred
func (request *Request) executeRequest(input *contextargs.Context, generatedRequest *generatedRequest, previousEvent output.InternalEvent, hasInteractMatchers bool, processEvent protocols.OutputEventCallback, requestCount int) (err error) {
// Check if hosts keep erroring
@@ -1200,10 +903,6 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
processEvent(event)
}
// Captured before auth, scheme correction, or the error path can rewrite
// the URL. The caller waited on this same key.
backoffKey := hostBackoffKey(generatedRequest, input)
request.setCustomHeaders(generatedRequest)
fuzzed := &generatedRequest.fuzzGeneratedRequest
cookieFuzzing := generatedRequest.request != nil && fuzzed.Component != nil && fuzzed.Component.Name() == component.RequestCookieComponent && len(fuzzed.FuzzedKeys) > 0
@@ -1223,10 +922,20 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
// For race conditions we can't dump the request body at this point as it's already waiting the open-gate event, already handled with a similar code within the race function
if !generatedRequest.original.Race {
// Buffer every body so the post-send dump reads its own copy. GET, HEAD,
// and chunked requests keep the content length they already had.
if generatedRequest.request != nil && generatedRequest.request.Body != nil {
if err = reuseRequestBody(generatedRequest.request); err != nil {
// change encoding type to content-length unless transfer-encoding header is manually set
if generatedRequest.request != nil && !stringsutil.EqualFoldAny(generatedRequest.request.Method, http.MethodGet, http.MethodHead) && generatedRequest.request.Body != nil && generatedRequest.request.Header.Get("Transfer-Encoding") != "chunked" {
var newReqBody *reader.ReusableReadCloser
newReqBody, ok := generatedRequest.request.Body.(*reader.ReusableReadCloser)
if !ok {
newReqBody, err = reader.NewReusableReadCloser(generatedRequest.request.Body)
}
if err == nil {
// update the request body with the reusable reader
generatedRequest.request.SetBodyReader(newReqBody)
// get content length
length, _ := io.Copy(io.Discard, newReqBody)
generatedRequest.request.ContentLength = length
} else {
// log error and continue
gologger.Verbose().Msgf("[%v] Could not read request body while forcing transfer encoding: %s\n", request.options.TemplateID, err)
err = nil
@@ -1277,7 +986,6 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
var formedURL string
var hostname string
var sent bool
timeStart := time.Now()
if generatedRequest.original.Pipeline {
// if request is a pipeline request, use the pipelined client
@@ -1287,10 +995,8 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
hostname = parsed.Host
}
resp, err = generatedRequest.pipelinedClient.DoRaw(generatedRequest.rawRequest.Method, input.MetaInput.Input, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)))
sent = true
} else if generatedRequest.request != nil {
resp, err = generatedRequest.pipelinedClient.Dor(generatedRequest.request)
sent = true
}
} else if generatedRequest.original.Unsafe && generatedRequest.rawRequest != nil {
// if request is a unsafe request, use the rawhttp client
@@ -1334,7 +1040,6 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
Body: io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)),
Options: &options,
})
sent = true
} else {
//** For Normal requests **//
// Use the dial target (URL.Host) rather than the optional Host-header
@@ -1411,7 +1116,6 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
}
resp, err = executingClient.Do(generatedRequest.request)
sent = true
// If we forced http->https from a previous detection and the corrected
// request failed (e.g. a false positive where the port actually speaks
@@ -1426,18 +1130,6 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
}
}
}
// A project-file hit never sets sent, so a stored response cannot move the delay.
// Record before dump or the dialer lookup can return and drop the outcome.
if sent {
request.observeHostBackoff(backoffKey, resp, err)
if observer, ok := request.options.Progress.(httpRequestTimingObserver); ok {
statusCode := 0
if resp != nil {
statusCode = resp.StatusCode
}
observer.ObserveHTTPRequest(time.Since(timeStart), statusCode, err)
}
}
// use request url as matched url if empty
if formedURL == "" {
formedURL = input.MetaInput.Input
@@ -1543,17 +1235,6 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
// we only intend to log/save the final redirected response
// i.e why we have to use sync.Once to ensure it's only done once
var errx error
// Race skips the dumps above because the body is already waiting on the
// gate. executeRaceRequest stored those bytes for DSL use.
if generatedRequest.original != nil && generatedRequest.original.Race && len(dumpedRequest) == 0 {
if raw, ok := previousEvent["request"].(string); ok {
dumpedRequest = []byte(raw)
}
}
// One executed request is replayed once, even when several responses in
// the redirect chain match.
replayed := false
onceFunc := sync.OnceFunc(func() {
// if nuclei-project is enabled store the response if not previously done
if request.options.ProjectFile != nil && !fromCache {
@@ -1704,10 +1385,8 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
event.UsesInteractsh = true
}
globalMatched := false
if request.options.GlobalMatchers.HasMatchers() {
request.options.GlobalMatchers.Match(interimEvent, request.Match, request.Extract, isDebug, func(event output.InternalEvent, result *operators.Result) {
globalMatched = true
callback(eventcreator.CreateEventWithOperatorResults(request, event, result))
})
}
@@ -1718,12 +1397,6 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
callback(event)
localMatched := event.OperatorsResult != nil && event.OperatorsResult.Matched
if !replayed && (localMatched || globalMatched) {
replayed = true
request.replayRequest(dumpedRequest, formedURL)
}
if request.options.FuzzStatsDB != nil && generatedRequest.fuzzGeneratedRequest.Request != nil {
request.options.FuzzStatsDB.RecordResultEvent(fuzzStats.FuzzingEvent{
URL: input.MetaInput.Target(),
@@ -2,15 +2,7 @@ package http
import (
"context"
"crypto/md5"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/require"
@@ -21,595 +13,9 @@ import (
extractorpkg "github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors"
matcherpkg "github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/progress"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/utils/memguardian"
)
func TestIndexedResponsePayloadCannotUseIndependentRequestThreads(t *testing.T) {
options := testutils.DefaultOptions.Copy()
options.ExecutionId = t.Name()
options.RestrictLocalNetworkAccess = false
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
var requests atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
requests.Add(1)
if request.URL.Path == "/first" {
writer.WriteHeader(http.StatusCreated)
return
}
writer.WriteHeader(http.StatusOK)
}))
defer server.Close()
request := &Request{
ID: "indexed-response-payload-threading",
Raw: []string{
"GET /first?probe={{probe}} HTTP/1.1\nHost: {{Hostname}}",
"GET /second?probe={{probe}} HTTP/1.1\nHost: {{Hostname}}",
},
Threads: 10,
Payloads: map[string]interface{}{"probe": []string{"one", "two"}},
Operators: operatorsWithMatchers(&matcherpkg.Matcher{
Type: matcherpkg.MatcherTypeHolder{MatcherType: matcherpkg.DSLMatcher},
DSL: []string{"status_code_1 == 201 && status_code_2 == 200"},
}),
}
executerOptions := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: request.ID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
executerOptions.RateLimiter = nil
require.NoError(t, request.Compile(executerOptions))
require.Zero(t, request.Threads)
run := func() int {
matched := 0
err := request.ExecuteWithResults(
contextargs.NewWithInput(t.Context(), server.URL),
make(output.InternalEvent),
make(output.InternalEvent),
func(event *output.InternalWrappedEvent) {
if event.OperatorsResult != nil && event.OperatorsResult.Matched {
matched++
}
},
)
require.NoError(t, err)
return matched
}
require.Equal(t, 2, run())
require.Equal(t, int32(4), requests.Load())
request.Threads = 10
requests.Store(0)
require.Zero(t, run())
require.Equal(t, int32(4), requests.Load())
}
func TestDependentPayloadThreadsRejectSharedRequestState(t *testing.T) {
newRequest := func() *Request {
return &Request{Raw: []string{
"GET /first HTTP/1.1\nHost: {{Hostname}}",
"GET /second HTTP/1.1\nHost: {{Hostname}}",
}}
}
base := newRequest()
require.True(t, base.canRunDependentPayloadsConcurrently())
iterating := newRequest()
iterating.IterateAll = true
require.False(t, iterating.canRunDependentPayloadsConcurrently())
racing := newRequest()
racing.Race = true
require.False(t, racing.canRunDependentPayloadsConcurrently())
pipelined := newRequest()
pipelined.Pipeline = true
require.False(t, pipelined.canRunDependentPayloadsConcurrently())
once := newRequest()
once.Raw[0] = "@once\n" + once.Raw[0]
require.False(t, once.canRunDependentPayloadsConcurrently())
}
func TestIndexedResponsePayloadThreadsKeepPairsOrdered(t *testing.T) {
options := testutils.DefaultOptions.Copy()
options.ExecutionId = t.Name()
options.RestrictLocalNetworkAccess = false
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
require.NotNil(t, memguardian.DefaultMemGuardian)
previousWarning := memguardian.DefaultMemGuardian.Warning.Swap(false)
t.Cleanup(func() { memguardian.DefaultMemGuardian.Warning.Store(previousWarning) })
var firstStarted atomic.Int32
var firstActive atomic.Int32
var maximumFirstActive atomic.Int32
var outOfOrder atomic.Int32
firstGate := make(chan struct{})
firstDone := make(map[string]bool)
var firstDoneMu sync.Mutex
negativeProbe := fmt.Sprintf("%x", md5.Sum([]byte("two")))
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
probe := request.URL.Query().Get("probe")
switch request.URL.Path {
case "/first":
active := firstActive.Add(1)
for old := maximumFirstActive.Load(); active > old && !maximumFirstActive.CompareAndSwap(old, active); old = maximumFirstActive.Load() {
}
if firstStarted.Add(1) == 2 {
close(firstGate)
}
select {
case <-firstGate:
case <-time.After(250 * time.Millisecond):
}
firstDoneMu.Lock()
firstDone[probe] = true
firstDoneMu.Unlock()
firstActive.Add(-1)
writer.WriteHeader(http.StatusCreated)
case "/second":
firstDoneMu.Lock()
if !firstDone[probe] {
outOfOrder.Add(1)
}
firstDoneMu.Unlock()
if probe == negativeProbe {
writer.WriteHeader(http.StatusNotFound)
return
}
writer.WriteHeader(http.StatusOK)
}
}))
defer server.Close()
request := &Request{
ID: "indexed-response-payload-pairs",
Raw: []string{
"GET /first?probe={{md5(probe)}} HTTP/1.1\nHost: {{Hostname}}",
"GET /second?probe={{md5(probe)}} HTTP/1.1\nHost: {{Hostname}}",
},
PayloadThreads: 2,
Payloads: map[string]interface{}{"probe": []string{"one", "two"}},
Operators: operatorsWithMatchers(&matcherpkg.Matcher{
Type: matcherpkg.MatcherTypeHolder{MatcherType: matcherpkg.DSLMatcher},
DSL: []string{"status_code_1 == 201 && status_code_2 == 200"},
}),
}
executerOptions := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: request.ID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
executerOptions.RateLimiter = nil
require.NoError(t, request.Compile(executerOptions))
var matches atomic.Int32
require.NoError(t, request.ExecuteWithResults(
contextargs.NewWithInput(t.Context(), server.URL),
make(output.InternalEvent),
make(output.InternalEvent),
func(event *output.InternalWrappedEvent) {
if event.OperatorsResult != nil && event.OperatorsResult.Matched {
matches.Add(1)
}
},
))
require.Equal(t, int32(1), matches.Load())
require.Zero(t, outOfOrder.Load())
require.Equal(t, int32(2), maximumFirstActive.Load())
require.Len(t, firstDone, 2)
}
func TestDependentPayloadRateLimitWaitDoesNotExpireHTTPTimeout(t *testing.T) {
options := testutils.DefaultOptions.Copy()
options.ExecutionId = t.Name()
options.RestrictLocalNetworkAccess = false
options.PerHostRateLimit = true
options.RateLimit = 1
options.RateLimitDuration = 80 * time.Millisecond
options.SetTimeouts(&types.Timeouts{HttpTimeout: 50 * time.Millisecond})
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
var sent atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
sent.Add(1)
writer.WriteHeader(http.StatusOK)
}))
defer server.Close()
request := &Request{
ID: "dependent-payload-rate-wait",
Raw: []string{
"GET /first?probe={{probe}} HTTP/1.1\nHost: {{Hostname}}",
"GET /second?probe={{probe}} HTTP/1.1\nHost: {{Hostname}}",
},
PayloadThreads: 2,
Payloads: map[string]interface{}{
"probe": []string{"one", "two"},
},
Operators: operatorsWithMatchers(&matcherpkg.Matcher{
Type: matcherpkg.MatcherTypeHolder{MatcherType: matcherpkg.DSLMatcher},
DSL: []string{"status_code_1 == 200 && status_code_2 == 200"},
}),
}
executerOptions := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: request.ID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
executerOptions.RateLimiter = nil
require.NoError(t, request.Compile(executerOptions))
var matches atomic.Int32
require.NoError(t, request.ExecuteWithResults(
contextargs.NewWithInput(t.Context(), server.URL),
make(output.InternalEvent),
make(output.InternalEvent),
func(event *output.InternalWrappedEvent) {
if event.OperatorsResult != nil && event.OperatorsResult.Matched {
matches.Add(1)
}
},
))
require.Equal(t, int32(4), sent.Load())
require.Equal(t, int32(2), matches.Load())
}
func TestDependentPayloadRateLimitedLastMatchAccounting(t *testing.T) {
options := testutils.DefaultOptions.Copy()
options.ExecutionId = t.Name()
options.RestrictLocalNetworkAccess = false
options.PerHostRateLimit = true
options.RateLimit = 1
options.RateLimitDuration = 20 * time.Millisecond
options.SetTimeouts(&types.Timeouts{HttpTimeout: 15 * time.Millisecond})
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
var sent atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
sent.Add(1)
if request.URL.Path == "/first" {
writer.WriteHeader(http.StatusCreated)
return
}
if request.URL.Query().Get("probe") == "key-50" {
writer.WriteHeader(http.StatusOK)
return
}
writer.WriteHeader(http.StatusNotFound)
}))
defer server.Close()
keys := make([]string, 51)
for i := range keys {
keys[i] = fmt.Sprintf("key-%d", i)
}
request := &Request{
ID: "dependent-payload-last-match",
Raw: []string{
"GET /first?probe={{probe}} HTTP/1.1\nHost: {{Hostname}}",
"GET /second?probe={{probe}} HTTP/1.1\nHost: {{Hostname}}",
},
PayloadThreads: 10,
StopAtFirstMatch: true,
Payloads: map[string]interface{}{"probe": keys},
Operators: operatorsWithMatchers(&matcherpkg.Matcher{
Type: matcherpkg.MatcherTypeHolder{MatcherType: matcherpkg.DSLMatcher},
DSL: []string{"status_code_1 == 201 && status_code_2 == 200"},
}),
}
executerOptions := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: request.ID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
executerOptions.RateLimiter = nil
progressCounter := &payloadRequestProgress{Progress: executerOptions.Progress}
executerOptions.Progress = progressCounter
require.NoError(t, request.Compile(executerOptions))
var matches atomic.Int32
require.NoError(t, request.ExecuteWithResults(
contextargs.NewWithInput(t.Context(), server.URL),
make(output.InternalEvent),
make(output.InternalEvent),
func(event *output.InternalWrappedEvent) {
if event.OperatorsResult != nil && event.OperatorsResult.Matched {
matches.Add(1)
}
},
))
require.Equal(t, int32(1), matches.Load())
require.GreaterOrEqual(t, sent.Load(), int32(2))
require.LessOrEqual(t, sent.Load(), int32(102))
require.Equal(t, int64(sent.Load()), progressCounter.sent.Load())
require.Equal(t, int64(102), progressCounter.sent.Load()+progressCounter.skipped.Load()+progressCounter.failed.Load())
require.Zero(t, progressCounter.failed.Load())
}
func TestIndexedResponsePayloadThreadsStopAfterFirstMatch(t *testing.T) {
options := testutils.DefaultOptions.Copy()
options.ExecutionId = t.Name()
options.RestrictLocalNetworkAccess = false
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
require.NotNil(t, memguardian.DefaultMemGuardian)
previousWarning := memguardian.DefaultMemGuardian.Warning.Swap(false)
t.Cleanup(func() { memguardian.DefaultMemGuardian.Warning.Store(previousWarning) })
var requests atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
requests.Add(1)
if request.URL.Path == "/first" {
writer.WriteHeader(http.StatusCreated)
return
}
if request.URL.Query().Get("probe") == "match" {
writer.WriteHeader(http.StatusOK)
return
}
writer.WriteHeader(http.StatusNotFound)
}))
defer server.Close()
request := &Request{
ID: "indexed-response-stop-first-match",
Raw: []string{
"GET /first?probe={{probe}} HTTP/1.1\nHost: {{Hostname}}",
"GET /second?probe={{probe}} HTTP/1.1\nHost: {{Hostname}}",
},
PayloadThreads: 2,
StopAtFirstMatch: true,
Payloads: map[string]interface{}{
"probe": []string{"match", "miss-a", "miss-b", "miss-c"},
},
Operators: operatorsWithMatchers(&matcherpkg.Matcher{
Type: matcherpkg.MatcherTypeHolder{MatcherType: matcherpkg.DSLMatcher},
DSL: []string{"status_code_1 == 201 && status_code_2 == 200"},
}),
}
executerOptions := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: request.ID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
executerOptions.RateLimiter = nil
progressCounter := &payloadRequestProgress{Progress: executerOptions.Progress}
executerOptions.Progress = progressCounter
require.NoError(t, request.Compile(executerOptions))
var matches atomic.Int32
require.NoError(t, request.ExecuteWithResults(
contextargs.NewWithInput(t.Context(), server.URL),
make(output.InternalEvent),
make(output.InternalEvent),
func(event *output.InternalWrappedEvent) {
if event.OperatorsResult != nil && event.OperatorsResult.Matched {
matches.Add(1)
}
},
))
require.Equal(t, int32(1), matches.Load())
require.Less(t, requests.Load(), int32(8))
require.Equal(t, int64(request.Requests()), progressCounter.sent.Load()+progressCounter.skipped.Load()+progressCounter.failed.Load())
require.Zero(t, progressCounter.failed.Load())
}
func TestIndexedResponsePayloadThreadsCoverAllFiveHundredResponses(t *testing.T) {
options := testutils.DefaultOptions.Copy()
options.ExecutionId = t.Name()
options.RestrictLocalNetworkAccess = false
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
require.NotNil(t, memguardian.DefaultMemGuardian)
previousWarning := memguardian.DefaultMemGuardian.Warning.Swap(false)
t.Cleanup(func() { memguardian.DefaultMemGuardian.Warning.Store(previousWarning) })
var active atomic.Int32
var maximum atomic.Int32
var served atomic.Int32
var keysMu sync.Mutex
keys := make(map[string]int)
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
current := active.Add(1)
for old := maximum.Load(); current > old && !maximum.CompareAndSwap(old, current); old = maximum.Load() {
}
key := strings.SplitN(request.URL.RawQuery, "=", 2)[0]
keysMu.Lock()
keys[key]++
keysMu.Unlock()
time.Sleep(5 * time.Millisecond)
active.Add(-1)
served.Add(1)
writer.WriteHeader(http.StatusInternalServerError)
}))
defer server.Close()
buckets := make([]string, 17)
for i := range buckets {
buckets[i] = fmt.Sprintf("bucket-%d", i)
}
for _, tc := range []struct {
name string
payloadThreads int
minimumActive int32
maximumActive int32
lowMemory bool
}{
{name: "serial", minimumActive: 1, maximumActive: 1},
{name: "guarded", payloadThreads: 10, minimumActive: 1, maximumActive: 2, lowMemory: true},
{name: "grouped", payloadThreads: 10, minimumActive: 5},
} {
t.Run(tc.name, func(t *testing.T) {
memguardian.DefaultMemGuardian.Warning.Store(tc.lowMemory)
maximum.Store(0)
served.Store(0)
keysMu.Lock()
clear(keys)
keysMu.Unlock()
request := &Request{
ID: "indexed-response-5xx-coverage-" + tc.name,
Raw: []string{
"GET /first?{{rand}}-{{md5(escape)}}-{{bucket}}=x HTTP/1.1\nHost: {{Hostname}}",
"GET /second?{{rand}}-{{md5(escape)}}-{{bucket}}=x HTTP/1.1\nHost: {{Hostname}}",
},
PayloadThreads: tc.payloadThreads,
AttackType: generators.AttackTypeHolder{Value: generators.ClusterBombAttack},
Payloads: map[string]interface{}{
"escape": []string{"\v", "\f", "\x1c", "\x1d", "\x1e", "\x1f"},
"bucket": buckets,
},
Operators: operatorsWithMatchers(&matcherpkg.Matcher{
Type: matcherpkg.MatcherTypeHolder{MatcherType: matcherpkg.DSLMatcher},
DSL: []string{"status_code_2 == 200"},
}),
StopAtFirstMatch: true,
}
executerOptions := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: request.ID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
executerOptions.RateLimiter = nil
require.NoError(t, executerOptions.Variables.UnmarshalJSON([]byte(`{"rand":"{{rand_base(5)}}"}`)))
progressCounter := &payloadRequestProgress{Progress: executerOptions.Progress}
executerOptions.Progress = progressCounter
require.NoError(t, request.Compile(executerOptions))
var matches atomic.Int32
require.NoError(t, request.ExecuteWithResults(
contextargs.NewWithInput(t.Context(), server.URL),
make(output.InternalEvent),
make(output.InternalEvent),
func(event *output.InternalWrappedEvent) {
if event.OperatorsResult != nil && event.OperatorsResult.Matched {
matches.Add(1)
}
},
))
require.Zero(t, matches.Load())
require.Equal(t, int32(204), served.Load())
require.Equal(t, int64(204), progressCounter.sent.Load())
require.Zero(t, progressCounter.skipped.Load())
require.Zero(t, progressCounter.failed.Load())
require.GreaterOrEqual(t, maximum.Load(), tc.minimumActive)
if tc.maximumActive > 0 {
require.LessOrEqual(t, maximum.Load(), tc.maximumActive)
}
keysMu.Lock()
require.Len(t, keys, 102)
for _, count := range keys {
require.Equal(t, 2, count)
}
keysMu.Unlock()
})
}
}
func TestIndexedResponsePayloadThreadsPreserveCachePoisoningDetection(t *testing.T) {
options := testutils.DefaultOptions.Copy()
options.ExecutionId = t.Name()
options.RestrictLocalNetworkAccess = false
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
require.NotNil(t, memguardian.DefaultMemGuardian)
previousWarning := memguardian.DefaultMemGuardian.Warning.Swap(false)
t.Cleanup(func() { memguardian.DefaultMemGuardian.Warning.Store(previousWarning) })
var mu sync.Mutex
cache := make(map[string]string)
var served atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
served.Add(1)
key := strings.SplitN(request.URL.RawQuery, "=", 2)[0]
mu.Lock()
defer mu.Unlock()
if request.URL.Path == "/first" {
if request.Header.Get("X-Bucket") == "positive" {
cache[key] = "alert(document.domain)"
}
writer.WriteHeader(http.StatusBadGateway)
return
}
if value, ok := cache[key]; ok {
writer.WriteHeader(http.StatusOK)
_, _ = writer.Write([]byte(value))
return
}
writer.WriteHeader(http.StatusNotFound)
}))
defer server.Close()
request := &Request{
ID: "indexed-response-cache-poisoning",
Raw: []string{
"GET /first?{{rand}}-{{md5(escape)}}-{{bucket}}=x HTTP/1.1\nHost: {{Hostname}}\nX-Bucket: {{bucket}}",
"GET /second?{{rand}}-{{md5(escape)}}-{{bucket}}=x HTTP/1.1\nHost: {{Hostname}}",
},
PayloadThreads: 2,
AttackType: generators.AttackTypeHolder{Value: generators.ClusterBombAttack},
Payloads: map[string]interface{}{
"escape": []string{"\v"},
"bucket": []string{"positive", "negative"},
},
Operators: operatorsWithMatchers(&matcherpkg.Matcher{
Type: matcherpkg.MatcherTypeHolder{MatcherType: matcherpkg.DSLMatcher},
DSL: []string{`contains(body_2, "alert(document.domain)") && status_code_2 == 200`},
}),
}
executerOptions := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: request.ID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
executerOptions.RateLimiter = nil
require.NoError(t, executerOptions.Variables.UnmarshalJSON([]byte(`{"rand":"{{rand_base(5)}}"}`)))
require.NoError(t, request.Compile(executerOptions))
var matches atomic.Int32
require.NoError(t, request.ExecuteWithResults(
contextargs.NewWithInput(t.Context(), server.URL),
make(output.InternalEvent),
make(output.InternalEvent),
func(event *output.InternalWrappedEvent) {
if event.OperatorsResult != nil && event.OperatorsResult.Matched {
matches.Add(1)
}
},
))
require.Equal(t, int32(4), served.Load())
require.Equal(t, int32(1), matches.Load())
require.Len(t, cache, 1)
}
type payloadRequestProgress struct {
progress.Progress
sent atomic.Int64
skipped atomic.Int64
failed atomic.Int64
}
func (p *payloadRequestProgress) IncrementRequests() {
p.sent.Add(1)
p.Progress.IncrementRequests()
}
func (p *payloadRequestProgress) SetRequests(count uint64) {
p.skipped.Add(int64(count))
p.Progress.SetRequests(count)
}
func (p *payloadRequestProgress) IncrementFailedRequestsBy(count int64) {
p.failed.Add(count)
p.Progress.IncrementFailedRequestsBy(count)
}
func TestNeedsRequestConditionDurationReferences(t *testing.T) {
tests := []struct {
name string
+9 -4
View File
@@ -190,6 +190,15 @@ func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest,
if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(request.options.ProtocolType.String(), input) {
return false, nil
}
// Extract hostname for per-host rate limiting: prefer the concrete fuzzed
// request URL (rules may change host/port), fall back to the input target
hostname := input.MetaInput.Input
if gr.Request != nil && gr.Request.Request != nil && gr.Request.Request.URL != nil {
hostname = gr.Request.Request.URL.String()
}
if err := request.options.RateLimitTakeContext(input.Context(), hostname); err != nil {
return false, err
}
req := &generatedRequest{
request: gr.Request,
dynamicValues: gr.DynamicValues,
@@ -197,10 +206,6 @@ func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest,
original: request,
fuzzGeneratedRequest: gr,
}
// Same key executeRequest will observe, including a host or port the fuzz rule rewrote.
if err := request.rateLimitTake(input.Context(), hostBackoffKey(req, input)); err != nil {
return false, err
}
var gotMatches bool
requestErr := request.executeRequest(input, req, gr.DynamicValues, hasInteractMatchers, func(event *output.InternalWrappedEvent) {
for _, result := range event.Results {
-387
View File
@@ -21,16 +21,9 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/progress"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hostbackoff"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/ratelimit"
"github.com/projectdiscovery/retryablehttp-go"
"github.com/projectdiscovery/utils/memguardian"
"io"
)
func executeDynamicHTTPTestRequest(t *testing.T, request *Request, target string) {
@@ -1301,291 +1294,6 @@ func TestExecuteParallelHTTP_StopAtFirstMatchCancelsQueuedRateLimitWaiters(t *te
require.Equal(t, int32(1), requests.Load())
}
func TestPayloadConcurrencyIsGuardedOnceUnderMemoryPressure(t *testing.T) {
options := testutils.DefaultOptions.Copy()
options.ExecutionId = t.Name()
options.RestrictLocalNetworkAccess = false
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
require.NotNil(t, memguardian.DefaultMemGuardian)
previousWarning := memguardian.DefaultMemGuardian.Warning.Swap(true)
t.Cleanup(func() { memguardian.DefaultMemGuardian.Warning.Store(previousWarning) })
var active atomic.Int32
var maximum atomic.Int32
var total atomic.Int32
var matched atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
current := active.Add(1)
for previous := maximum.Load(); current > previous && !maximum.CompareAndSwap(previous, current); previous = maximum.Load() {
}
time.Sleep(40 * time.Millisecond)
active.Add(-1)
total.Add(1)
writer.WriteHeader(http.StatusOK)
}))
defer server.Close()
request := &Request{
ID: "memory-guard-payload-concurrency",
Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
Path: []string{"{{BaseURL}}/test?payload={{v}}"},
Threads: 20,
Payloads: map[string]interface{}{
"v": []string{"1", "2", "3", "4", "5", "6", "7", "8"},
},
Operators: operators.Operators{Matchers: []*matchers.Matcher{{
Type: matchers.MatcherTypeHolder{MatcherType: matchers.DSLMatcher},
DSL: []string{"status_code == 200"},
}}},
}
executerOptions := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: request.ID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
executerOptions.RateLimiter = nil
require.NoError(t, request.Compile(executerOptions))
compiledThreads := request.Threads
require.NoError(t, request.ExecuteWithResults(
contextargs.NewWithInput(t.Context(), server.URL),
make(output.InternalEvent),
make(output.InternalEvent),
func(event *output.InternalWrappedEvent) {
if event.OperatorsResult != nil && event.OperatorsResult.Matched {
matched.Add(1)
}
},
))
require.Equal(t, int32(8), total.Load())
require.Equal(t, int32(8), matched.Load())
if compiledThreads != 20 || maximum.Load() <= 1 {
t.Fatalf("memory guard applied more than once: compiled threads=%d, maximum concurrent requests=%d", compiledThreads, maximum.Load())
}
memguardian.DefaultMemGuardian.Warning.Store(false)
maximum.Store(0)
total.Store(0)
matched.Store(0)
require.NoError(t, request.ExecuteWithResults(
contextargs.NewWithInput(t.Context(), server.URL),
make(output.InternalEvent),
make(output.InternalEvent),
func(event *output.InternalWrappedEvent) {
if event.OperatorsResult != nil && event.OperatorsResult.Matched {
matched.Add(1)
}
},
))
require.Equal(t, int32(8), total.Load())
require.Equal(t, int32(8), matched.Load())
require.Greater(t, maximum.Load(), int32(4))
}
func TestDefaultPayloadConcurrencyRespectsMemoryPressure(t *testing.T) {
options := testutils.DefaultOptions.Copy()
options.ExecutionId = t.Name()
options.RestrictLocalNetworkAccess = false
options.PayloadConcurrency = 25
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
require.NotNil(t, memguardian.DefaultMemGuardian)
previousWarning := memguardian.DefaultMemGuardian.Warning.Swap(true)
t.Cleanup(func() { memguardian.DefaultMemGuardian.Warning.Store(previousWarning) })
var active atomic.Int32
var maximum atomic.Int32
var total atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
current := active.Add(1)
for previous := maximum.Load(); current > previous && !maximum.CompareAndSwap(previous, current); previous = maximum.Load() {
}
time.Sleep(40 * time.Millisecond)
active.Add(-1)
total.Add(1)
writer.WriteHeader(http.StatusOK)
}))
defer server.Close()
request := &Request{
ID: "default-payload-concurrency-under-memory-pressure",
Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
Path: []string{"{{BaseURL}}/test?payload={{v}}"},
Threads: options.PayloadConcurrency,
Payloads: map[string]interface{}{
"v": []string{"1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16"},
},
Operators: operators.Operators{Matchers: []*matchers.Matcher{{
Type: matchers.MatcherTypeHolder{MatcherType: matchers.DSLMatcher},
DSL: []string{"status_code == 200"},
}}},
}
executerOptions := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: request.ID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
executerOptions.RateLimiter = nil
require.NoError(t, request.Compile(executerOptions))
require.NoError(t, request.ExecuteWithResults(
contextargs.NewWithInput(t.Context(), server.URL),
make(output.InternalEvent),
make(output.InternalEvent),
func(*output.InternalWrappedEvent) {},
))
require.Equal(t, int32(16), total.Load())
require.LessOrEqual(t, maximum.Load(), int32(protocolstate.GuardThreadsOrDefault(options.PayloadConcurrency)))
require.Greater(t, maximum.Load(), int32(1))
}
type timingProgressObserver struct {
progress.Progress
mu sync.Mutex
requestDuration time.Duration
requestStatus int
requestError error
requestCount int
rateWaitCount int
rateWait time.Duration
}
func (o *timingProgressObserver) ObserveHTTPRateLimitWait(duration time.Duration) {
o.mu.Lock()
defer o.mu.Unlock()
o.rateWaitCount++
o.rateWait = duration
}
func (o *timingProgressObserver) ObserveHTTPRequest(duration time.Duration, status int, err error) {
o.mu.Lock()
defer o.mu.Unlock()
o.requestDuration = duration
o.requestStatus = status
o.requestError = err
o.requestCount++
}
func TestHTTPExecutionObservesRequestTiming(t *testing.T) {
options := testutils.DefaultOptions.Copy()
options.ExecutionId = t.Name()
options.RestrictLocalNetworkAccess = false
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
time.Sleep(20 * time.Millisecond)
writer.WriteHeader(http.StatusTeapot)
}))
defer server.Close()
request := &Request{
ID: "http-timing-observer",
Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
Path: []string{"{{BaseURL}}/"},
Operators: operators.Operators{Matchers: []*matchers.Matcher{{
Type: matchers.MatcherTypeHolder{MatcherType: matchers.StatusMatcher},
Status: []int{http.StatusTeapot},
}}},
}
executerOptions := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: request.ID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
observer := &timingProgressObserver{Progress: executerOptions.Progress}
executerOptions.Progress = observer
require.NoError(t, request.Compile(executerOptions))
require.NoError(t, request.ExecuteWithResults(
contextargs.NewWithInput(t.Context(), server.URL),
make(output.InternalEvent),
make(output.InternalEvent),
func(*output.InternalWrappedEvent) {},
))
observer.mu.Lock()
defer observer.mu.Unlock()
require.Equal(t, 1, observer.rateWaitCount)
require.Equal(t, 1, observer.requestCount)
require.Equal(t, http.StatusTeapot, observer.requestStatus)
require.NoError(t, observer.requestError)
require.GreaterOrEqual(t, observer.requestDuration, 20*time.Millisecond)
}
func TestHTTPRateLimitWaitExcludesHostBackoff(t *testing.T) {
backoff := hostbackoff.New(hostbackoff.Config{Step: 200 * time.Millisecond, Max: 200 * time.Millisecond})
observer := &timingProgressObserver{}
request := &Request{options: &protocols.ExecutorOptions{HostBackoff: backoff, Progress: observer}}
target := "https://example.com/path"
request.observeHostBackoff(target, &http.Response{StatusCode: http.StatusTooManyRequests, Header: http.Header{}}, nil)
started := time.Now()
require.NoError(t, request.rateLimitTake(t.Context(), target))
elapsed := time.Since(started)
observer.mu.Lock()
defer observer.mu.Unlock()
require.Equal(t, 1, observer.rateWaitCount)
require.GreaterOrEqual(t, elapsed, 150*time.Millisecond)
require.Less(t, observer.rateWait, elapsed/2)
}
func TestHTTPRateLimitWaitNotObservedWhenBackoffCanceled(t *testing.T) {
backoff := hostbackoff.New(hostbackoff.Config{Step: time.Second, Max: time.Second})
observer := &timingProgressObserver{}
request := &Request{options: &protocols.ExecutorOptions{HostBackoff: backoff, Progress: observer}}
target := "https://example.com/path"
request.observeHostBackoff(target, &http.Response{StatusCode: http.StatusTooManyRequests, Header: http.Header{}}, nil)
ctx, cancel := context.WithCancel(t.Context())
cancel()
require.ErrorIs(t, request.rateLimitTake(ctx, target), context.Canceled)
observer.mu.Lock()
defer observer.mu.Unlock()
require.Zero(t, observer.rateWaitCount)
}
func TestSerialHTTPUnresponsiveHostSkipsBeforeRateLimit(t *testing.T) {
options := testutils.DefaultOptions.Copy()
options.ExecutionId = t.Name()
options.RateLimit = 1
options.RateLimitDuration = 2 * time.Second
options.RestrictLocalNetworkAccess = false
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
var requests atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
requests.Add(1)
writer.WriteHeader(http.StatusOK)
}))
defer server.Close()
request := &Request{
ID: "serial-skip-before-rate-limit",
Method: HTTPMethodTypeHolder{MethodType: HTTPGet},
Path: []string{"{{BaseURL}}/"},
}
executerOptions := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: request.ID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
executerOptions.HostErrorsCache = &fakeHostErrorsCache{}
observer := &timingProgressObserver{Progress: executerOptions.Progress}
executerOptions.Progress = observer
executerOptions.RateLimiter = ratelimit.New(context.Background(), 1, 2*time.Second)
t.Cleanup(executerOptions.RateLimiter.Stop)
require.NoError(t, executerOptions.RateLimiter.TakeContext(t.Context()))
require.NoError(t, request.Compile(executerOptions))
ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
defer cancel()
started := time.Now()
err := request.ExecuteWithResults(contextargs.NewWithInput(ctx, server.URL), make(output.InternalEvent), make(output.InternalEvent), func(*output.InternalWrappedEvent) {})
require.NoError(t, err)
require.Less(t, time.Since(started), 250*time.Millisecond)
require.Zero(t, requests.Load())
observer.mu.Lock()
defer observer.mu.Unlock()
require.Zero(t, observer.rateWaitCount)
}
func TestExecuteParallelHTTP_SkipOnUnresponsiveFromCache(t *testing.T) {
options := testutils.DefaultOptions
testutils.Init(options)
@@ -1854,98 +1562,3 @@ func TestPerHostRateLimitHonorsRequestContextCancellation(t *testing.T) {
require.ErrorIs(t, err, context.Canceled)
require.Less(t, cancellationDelay, 250*time.Millisecond)
}
func TestDumpedRequestKeepsBodyServerDidNotRead(t *testing.T) {
// The server answers in full, then holds the connection without reading
// the body, so the transport is still writing it when the response arrives.
release := make(chan struct{})
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Length", "2")
_, _ = w.Write([]byte("ok"))
w.(http.Flusher).Flush()
<-release
}))
defer target.Close()
defer close(release)
options := testutils.DefaultOptions.Copy()
options.SetExecutionID(t.Name())
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: "dump-body",
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
body := strings.Repeat("A", 4<<20)
requests := []*Request{
{
ID: "post",
Method: HTTPMethodTypeHolder{MethodType: HTTPPost},
Path: []string{"{{BaseURL}}/"},
Body: body,
},
{
ID: "get",
Raw: []string{"GET / HTTP/1.1\nHost: {{Hostname}}\n\n" + body},
},
{
ID: "chunked",
Raw: []string{"POST / HTTP/1.1\nHost: {{Hostname}}\nTransfer-Encoding: chunked\n\n" + body},
},
}
for _, request := range requests {
require.NoError(t, request.Compile(executerOpts))
// how much the transport has written when the response lands varies, so
// a few rounds make a lost body show up reliably
for range 3 {
var dumped string
input := contextargs.NewWithInput(context.Background(), target.URL)
err := request.ExecuteWithResults(input, output.InternalEvent{}, output.InternalEvent{}, func(event *output.InternalWrappedEvent) {
dumped, _ = event.InternalEvent["request"].(string)
})
require.NoError(t, err)
require.True(t, strings.HasSuffix(dumped, "\r\n\r\n"+body), "%s recorded request lost its body: %d of %d bytes", request.ID, len(dumped), len(body))
}
}
}
func TestReuseRequestBodyKeepsExcludedEncoding(t *testing.T) {
body := []byte("original")
for _, tc := range []struct {
name string
method string
header string
contentLength int64
wantLength int64
}{
{name: "post", method: http.MethodPost, contentLength: 1, wantLength: int64(len(body))},
{name: "get", method: http.MethodGet, contentLength: 3, wantLength: 3},
{name: "head", method: http.MethodHead, contentLength: 0, wantLength: 0},
{name: "chunked", method: http.MethodPost, header: "chunked", contentLength: -1, wantLength: -1},
} {
t.Run(tc.name, func(t *testing.T) {
req, err := retryablehttp.NewRequest(tc.method, "https://example.com/", body)
require.NoError(t, err)
req.ContentLength = tc.contentLength
if tc.header != "" {
req.Header.Set("Transfer-Encoding", tc.header)
}
require.NoError(t, reuseRequestBody(req))
require.Equal(t, tc.wantLength, req.ContentLength)
if tc.header != "" {
require.Equal(t, tc.header, req.Header.Get("Transfer-Encoding"))
}
got, err := io.ReadAll(req.Body)
require.NoError(t, err)
require.Equal(t, body, got)
fresh, err := req.GetBody()
require.NoError(t, err)
got, err = io.ReadAll(fresh)
require.NoError(t, err)
require.Equal(t, body, got)
})
}
}
+2 -10
View File
@@ -13,16 +13,8 @@ import (
// dump creates a dump of the http request in form of a byte slice
func dump(req *generatedRequest, reqURL string) ([]byte, error) {
if req.request != nil {
// Use a clone to avoid a race condition with the http transport. The
// clone shares the body, which the transport may still be sending, so
// dump a fresh copy of it where one is available.
clone := req.request.Clone(req.request.Context())
if clone.GetBody != nil {
if body, err := clone.GetBody(); err == nil {
clone.Body = body
}
}
bin, err := clone.Dump()
// Use a clone to avoid a race condition with the http transport
bin, err := req.request.Clone(req.request.Context()).Dump()
if err != nil {
return nil, errkit.Wrapf(err, "could not dump request: %v", req.request.String())
}
-29
View File
@@ -1,9 +1,7 @@
package network
import (
"crypto/tls"
"encoding/hex"
stderrors "errors"
"fmt"
"maps"
"net"
@@ -71,11 +69,7 @@ func (request *Request) getOpenPorts(target *contextargs.Context) ([]string, err
request.dialer, _ = networkclientpool.Get(request.options.Options, &networkclientpool.Configuration{})
}
if err := request.options.HostBackoffWait(target.Context(), addr); err != nil {
return nil, err
}
conn, err := request.dialer.Dial(target.Context(), "tcp", addr)
request.options.HostBackoffObserveDial(addr, err)
if err != nil {
errs = append(errs, err)
continue
@@ -332,9 +326,6 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
return nil
}
if err := request.options.HostBackoffWait(input.Context(), actualAddress); err != nil {
return err
}
if err := request.options.RateLimitTakeContext(input.Context(), actualAddress); err != nil {
request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err)
request.options.Progress.IncrementFailedRequestsBy(1)
@@ -346,9 +337,6 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
} else {
conn, err = request.dialer.Dial(input.Context(), "tcp", actualAddress)
}
// DialTLS fails the handshake after TCP has connected. Backoff follows the
// TCP result; the request still returns the handshake error.
request.options.HostBackoffObserveDial(actualAddress, tcpDialError(err))
// adds it to unresponsive address list if applicable
request.markHostError(updatedTarget, err)
if err != nil {
@@ -558,23 +546,6 @@ func displayCompactHexView(event *output.InternalWrappedEvent, response string,
}
}
// tcpDialError reports a failure to open TCP. A TLS handshake error means the
// port already accepted the connection.
func tcpDialError(err error) error {
if err == nil {
return nil
}
var record tls.RecordHeaderError
if stderrors.As(err, &record) {
return nil
}
message := err.Error()
if strings.Contains(message, "tls:") || strings.Contains(message, "tls handshake") {
return nil
}
return err
}
// getAddress returns the address of the host to make request to
func getAddress(toTest string) (string, error) {
if strings.Contains(toTest, "://") {
@@ -1,221 +0,0 @@
package network
import (
"context"
"net"
"net/http"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hostbackoff"
)
func newBackoffRequest(t *testing.T, address string, g *hostbackoff.Governor) (*Request, *protocols.ExecutorOptions) {
t.Helper()
options := testutils.DefaultOptions.Copy()
options.SetExecutionID(t.Name())
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ID: "network-host-backoff"})
executerOpts.HostBackoff = g
request := &Request{ID: "network-host-backoff", Address: []string{address}, ReadSize: 2}
require.NoError(t, request.Compile(executerOpts))
return request, executerOpts
}
func execute(request *Request, address string) error {
input := contextargs.NewWithInput(context.Background(), address)
return request.ExecuteWithResults(input, output.InternalEvent{}, output.InternalEvent{}, func(*output.InternalWrappedEvent) {})
}
func TestFailedDialBacksOffHost(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
address := listener.Addr().String()
go func() {
conn, err := listener.Accept()
if err == nil {
_, _ = conn.Write([]byte("ok"))
_ = conn.Close()
}
}()
g := hostbackoff.New(hostbackoff.Config{Step: 100 * time.Millisecond})
request, _ := newBackoffRequest(t, address, g)
require.NoError(t, execute(request, address))
require.NoError(t, listener.Close())
require.Error(t, execute(request, address))
require.Equal(t, 100*time.Millisecond, g.Delay(address), "a port that accepted and now refuses is pushing back")
}
func TestClosedPortIsNotBackedOff(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
address := listener.Addr().String()
require.NoError(t, listener.Close())
g := hostbackoff.New(hostbackoff.Config{Step: 100 * time.Millisecond})
request, _ := newBackoffRequest(t, address, g)
require.Error(t, execute(request, address))
require.Error(t, execute(request, address))
require.Zero(t, g.Delay(address), "a port that never accepted is closed, not blocking")
}
func TestSuccessfulDialWaitsThenDecays(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
t.Cleanup(func() { _ = listener.Close() })
go func() {
for {
conn, err := listener.Accept()
if err != nil {
return
}
_, _ = conn.Write([]byte("ok"))
_ = conn.Close()
}
}()
address := listener.Addr().String()
const step = 150 * time.Millisecond
g := hostbackoff.New(hostbackoff.Config{Step: step})
g.Observe(address, http.StatusTooManyRequests, 0, nil)
request, _ := newBackoffRequest(t, address, g)
start := time.Now()
require.NoError(t, execute(request, address))
require.GreaterOrEqual(t, time.Since(start), step, "a host that earned a delay is paced")
require.Zero(t, g.Delay(address), "a connection that succeeds is recovery")
}
func TestMultiPortProbeWaitsThenDecays(t *testing.T) {
openLn, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
t.Cleanup(func() { _ = openLn.Close() })
go func() {
for {
conn, err := openLn.Accept()
if err != nil {
return
}
_ = conn.Close()
}
}()
openAddr := openLn.Addr().String()
_, openPort, err := net.SplitHostPort(openAddr)
require.NoError(t, err)
closedLn, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
closedAddr := closedLn.Addr().String()
_, closedPort, err := net.SplitHostPort(closedAddr)
require.NoError(t, err)
require.NoError(t, closedLn.Close())
const step = 150 * time.Millisecond
g := hostbackoff.New(hostbackoff.Config{Step: step})
g.Observe(openAddr, http.StatusTooManyRequests, 0, nil)
options := testutils.DefaultOptions.Copy()
options.SetExecutionID(t.Name())
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ID: "network-host-backoff"})
executerOpts.HostBackoff = g
request := &Request{
ID: "network-host-backoff",
Address: []string{"{{Hostname}}"},
Port: openPort + "," + closedPort,
ReadSize: 2,
}
require.NoError(t, request.Compile(executerOpts))
start := time.Now()
require.NoError(t, execute(request, "127.0.0.1"))
require.GreaterOrEqual(t, time.Since(start), step, "a multi-port probe waits on a host:port that already has a delay")
require.Zero(t, g.Delay(openAddr), "a probe that connects is recovery")
require.Zero(t, g.Delay(closedAddr), "a port that never accepted stays unpaced")
}
func TestMultiPortProbeFailureBacksOffPreviouslyOpenPort(t *testing.T) {
closedLn, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
closedAddr := closedLn.Addr().String()
_, closedPort, err := net.SplitHostPort(closedAddr)
require.NoError(t, err)
require.NoError(t, closedLn.Close())
otherLn, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
otherAddr := otherLn.Addr().String()
_, otherPort, err := net.SplitHostPort(otherAddr)
require.NoError(t, err)
require.NoError(t, otherLn.Close())
const step = 100 * time.Millisecond
g := hostbackoff.New(hostbackoff.Config{Step: step})
g.ObserveDial(closedAddr, nil)
options := testutils.DefaultOptions.Copy()
options.SetExecutionID(t.Name())
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ID: "network-host-backoff"})
executerOpts.HostBackoff = g
request := &Request{
ID: "network-host-backoff",
Address: []string{"{{Hostname}}"},
Port: closedPort + "," + otherPort,
}
require.NoError(t, request.Compile(executerOpts))
require.Error(t, execute(request, "127.0.0.1"))
require.Equal(t, step, g.Delay(closedAddr), "a probe that fails on a port that accepted before slows it")
require.Zero(t, g.Delay(otherAddr), "a port that never accepted stays unpaced")
}
func TestTLSHandshakeFailureDoesNotBackOff(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
t.Cleanup(func() { _ = listener.Close() })
go func() {
for {
conn, err := listener.Accept()
if err != nil {
return
}
_, _ = conn.Write([]byte("not tls"))
_ = conn.Close()
}
}()
address := listener.Addr().String()
const step = 100 * time.Millisecond
g := hostbackoff.New(hostbackoff.Config{Step: step})
g.ObserveDial(address, nil)
options := testutils.DefaultOptions.Copy()
options.SetExecutionID(t.Name())
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ID: "network-host-backoff"})
executerOpts.HostBackoff = g
request := &Request{ID: "network-host-backoff", Address: []string{"tls://" + address}, ReadSize: 2}
require.NoError(t, request.Compile(executerOpts))
err = execute(request, address)
require.Error(t, err)
require.Zero(t, g.Delay(address), "a TLS handshake failure after TCP connected stays with the template: %v", err)
}
+5 -21
View File
@@ -3,7 +3,6 @@ package protocols
import (
"context"
"encoding/base64"
"errors"
"sync/atomic"
"github.com/projectdiscovery/fastdialer/fastdialer"
@@ -31,7 +30,6 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/projectfile"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/globalmatchers"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hostbackoff"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hostratelimit"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
@@ -117,9 +115,6 @@ type ExecutorOptions struct {
InteractshScope *interactsh.RequestScope
// HostErrorsCache is an optional cache for handling host errors
HostErrorsCache hosterrorscache.CacheInterface
// HostBackoff paces a host that signals it is being overloaded, so a scan
// slows down for it instead of being blocked outright. Nil disables it.
HostBackoff *hostbackoff.Governor
// Stop execution once first match is found (Assigned while parsing templates)
// Note: this is different from Options.StopAtFirstMatch (Assigned from CLI option)
StopAtFirstMatch bool
@@ -148,6 +143,9 @@ type ExecutorOptions struct {
ProtocolType templateTypes.ProtocolType
// Flow is execution flow for the template (written in javascript)
Flow string
// MaxFlowProtocolRequests bounds protocol requests issued by one Flow execution.
// Zero leaves the execution unbounded for callers that trust their templates.
MaxFlowProtocolRequests int64
// IsMultiProtocol is true if template has more than one protocol
IsMultiProtocol bool
// templateStore is a map which contains template context for each scan (i.e input * template-id pair)
@@ -241,20 +239,6 @@ func (e *ExecutorOptions) RateLimitTakeContext(ctx context.Context, host string)
return nil
}
// HostBackoffWait pauses for the delay address has earned with HostBackoff.
func (e *ExecutorOptions) HostBackoffWait(ctx context.Context, address string) error {
return e.HostBackoff.Wait(ctx, hostratelimit.NormalizeHostPort(address))
}
// HostBackoffObserveDial reports a connection attempt to address to HostBackoff.
func (e *ExecutorOptions) HostBackoffObserveDial(address string, err error) {
// a canceled dial is the scan stopping, not the host asking for less traffic
if errors.Is(err, context.Canceled) {
return
}
e.HostBackoff.ObserveDial(hostratelimit.NormalizeHostPort(address), err)
}
// GetThreadsForNPayloadRequests returns the number of threads to use as default for
// given max-request of payloads
func (e *ExecutorOptions) GetThreadsForNPayloadRequests(totalRequests int, currentThreads int) int {
@@ -397,7 +381,6 @@ func (e *ExecutorOptions) Copy() *ExecutorOptions {
Interactsh: e.Interactsh,
InteractshScope: e.InteractshScope,
HostErrorsCache: e.HostErrorsCache,
HostBackoff: e.HostBackoff,
StopAtFirstMatch: e.StopAtFirstMatch,
Variables: e.Variables,
Constants: e.Constants,
@@ -412,6 +395,7 @@ func (e *ExecutorOptions) Copy() *ExecutorOptions {
ResumeCfg: e.ResumeCfg,
ProtocolType: e.ProtocolType,
Flow: e.Flow,
MaxFlowProtocolRequests: e.MaxFlowProtocolRequests,
IsMultiProtocol: e.IsMultiProtocol,
JsCompiler: e.JsCompiler,
AuthProvider: e.AuthProvider,
@@ -626,7 +610,6 @@ func (e *ExecutorOptions) ApplyNewEngineOptions(n *ExecutorOptions) {
e.Interactsh = n.Interactsh
e.InteractshScope = n.InteractshScope
e.HostErrorsCache = n.HostErrorsCache
e.HostBackoff = n.HostBackoff
e.InputHelper = n.InputHelper
e.FuzzParamsFrequency = n.FuzzParamsFrequency
e.FuzzStatsDB = n.FuzzStatsDB
@@ -643,6 +626,7 @@ func (e *ExecutorOptions) ApplyNewEngineOptions(n *ExecutorOptions) {
e.Logger = n.Logger
e.CustomFastdialer = n.CustomFastdialer
e.ClusterMappings = n.ClusterMappings.Copy()
e.MaxFlowProtocolRequests = n.MaxFlowProtocolRequests
}
// GetOptions returns scan options, or nil for an unbound executor.
+7 -4
View File
@@ -13,9 +13,10 @@ func TestExecutorOptionsCopyPreservesInteractshScope(t *testing.T) {
require.NoError(t, err)
scope := client.NewRequestScope()
copy := (&ExecutorOptions{Interactsh: client, InteractshScope: scope}).Copy()
copy := (&ExecutorOptions{Interactsh: client, InteractshScope: scope, MaxFlowProtocolRequests: 1000}).Copy()
require.Same(t, client, copy.Interactsh)
require.Same(t, scope, copy.InteractshScope)
require.Equal(t, int64(1000), copy.MaxFlowProtocolRequests)
}
func TestExecutorOptionsApplyNewEngineOptionsReplacesInteractshScope(t *testing.T) {
@@ -26,10 +27,12 @@ func TestExecutorOptionsApplyNewEngineOptionsReplacesInteractshScope(t *testing.
reused := &ExecutorOptions{Interactsh: client, InteractshScope: previous}
reused.ApplyNewEngineOptions(&ExecutorOptions{
Options: &types.Options{},
Interactsh: client,
InteractshScope: next,
Options: &types.Options{},
Interactsh: client,
InteractshScope: next,
MaxFlowProtocolRequests: 1000,
})
require.Same(t, next, reused.InteractshScope)
require.Equal(t, int64(1000), reused.MaxFlowProtocolRequests)
}
-4
View File
@@ -256,9 +256,6 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
hostIp = host
}
if err := request.options.HostBackoffWait(input.Context(), addressToDial); err != nil {
return err
}
if err := request.options.RateLimitTakeContext(input.Context(), addressToDial); err != nil {
requestOptions.Output.Request(requestOptions.TemplateID, input.MetaInput.Input, request.Type().String(), err)
requestOptions.Progress.IncrementFailedRequestsBy(1)
@@ -268,7 +265,6 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
timeStart := time.Now()
response, err := request.tlsx.Connect(host, hostIp, port)
duration := time.Since(timeStart)
request.options.HostBackoffObserveDial(addressToDial, err)
if err != nil {
requestOptions.Output.Request(requestOptions.TemplateID, input.MetaInput.Input, request.Type().String(), err)
requestOptions.Progress.IncrementFailedRequestsBy(1)
+3 -20
View File
@@ -1,7 +1,6 @@
package websocket
import (
"context"
"crypto/tls"
"fmt"
"io"
@@ -221,20 +220,10 @@ func (request *Request) executeRequestWithPayloads(target *contextargs.Context,
tlsConfig.ServerName = requestOptions.Options.SNI
}
// Dial reports both a refused TCP connection and a rejected HTTP upgrade.
// Only the TCP result belongs to host backoff: an upgrade failure means
// the port already accepted the connection.
var dialed bool
var dialErr error
websocketDialer := ws.Dialer{
Header: ws.HandshakeHeaderHTTP(header),
Timeout: time.Duration(requestOptions.Options.Timeout) * time.Second,
NetDial: func(ctx context.Context, network, address string) (net.Conn, error) {
dialed = true
conn, err := request.dialer.Dial(ctx, network, address)
dialErr = err
return conn, err
},
Header: ws.HandshakeHeaderHTTP(header),
Timeout: time.Duration(requestOptions.Options.Timeout) * time.Second,
NetDial: request.dialer.Dial,
TLSConfig: tlsConfig,
}
@@ -266,9 +255,6 @@ func (request *Request) executeRequestWithPayloads(target *contextargs.Context,
addressToDial = parsedAddress.String()
if err := requestOptions.HostBackoffWait(target.Context(), addressToDial); err != nil {
return err
}
if err := requestOptions.RateLimitTakeContext(target.Context(), addressToDial); err != nil {
requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err)
requestOptions.Progress.IncrementFailedRequestsBy(1)
@@ -279,9 +265,6 @@ func (request *Request) executeRequestWithPayloads(target *contextargs.Context,
timeStart := time.Now()
conn, readBuffer, _, err := websocketDialer.Dial(target.Context(), addressToDial)
handshakeDuration := time.Since(timeStart)
if dialed {
requestOptions.HostBackoffObserveDial(addressToDial, dialErr)
}
if err != nil {
requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err)
requestOptions.Progress.IncrementFailedRequestsBy(1)
@@ -1,71 +0,0 @@
package websocket
import (
"context"
"net"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hostbackoff"
)
func newBackoffWebsocket(t *testing.T, address string, g *hostbackoff.Governor) *Request {
t.Helper()
options := testutils.DefaultOptions.Copy()
options.SetExecutionID(t.Name())
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ID: "websocket-host-backoff"})
executerOpts.HostBackoff = g
request := &Request{ID: "websocket-host-backoff", Address: address}
require.NoError(t, request.Compile(executerOpts))
return request
}
func executeWebsocket(request *Request, address string) error {
input := contextargs.NewWithInput(context.Background(), address)
return request.ExecuteWithResults(input, output.InternalEvent{}, output.InternalEvent{}, func(*output.InternalWrappedEvent) {})
}
func TestWebsocketHandshakeFailureDoesNotBackOff(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.NotFound(w, r)
}))
t.Cleanup(server.Close)
address := strings.Replace(server.URL, "http://", "ws://", 1)
host := server.Listener.Addr().String()
const step = 100 * time.Millisecond
g := hostbackoff.New(hostbackoff.Config{Step: step})
g.ObserveDial(host, nil)
request := newBackoffWebsocket(t, address, g)
require.Error(t, executeWebsocket(request, address))
require.Zero(t, g.Delay(host), "a rejected upgrade means the port accepted the connection")
}
func TestWebsocketDialFailureBacksOffPreviouslyOpenPort(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
host := listener.Addr().String()
require.NoError(t, listener.Close())
address := "ws://" + host
const step = 100 * time.Millisecond
g := hostbackoff.New(hostbackoff.Config{Step: step})
g.ObserveDial(host, nil)
request := newBackoffWebsocket(t, address, g)
require.Error(t, executeWebsocket(request, address))
require.Equal(t, step, g.Delay(host), "a refused TCP dial on a port that accepted before slows it")
}
+18 -123
View File
@@ -7,8 +7,6 @@ import (
"fmt"
"io"
"net/http"
"sort"
"sync"
"github.com/shurcooL/graphql"
@@ -28,9 +26,6 @@ type Integration struct {
url string
httpclient *http.Client
options *Options
closedStateMu sync.Mutex
closedStateID string
}
// Options contains the configuration options for linear issue tracker client
@@ -52,8 +47,6 @@ type Options struct {
// OpenStateID is the id of the open state for the project
OpenStateID string `yaml:"open-state-id"`
// ClosedStateID is the id of the closed state for the project, discovered from the team's completed workflow states when empty
ClosedStateID string `yaml:"closed-state-id"`
HttpClient *retryablehttp.Client `yaml:"-"`
OmitRaw bool `yaml:"-"`
@@ -122,8 +115,7 @@ func (i *Integration) CreateIssue(event *output.ResultEvent) (*filters.CreateIss
}
var resp struct {
IssueUpdate struct {
LastSyncID int `json:"lastSyncId"`
Success bool `json:"success"`
LastSyncID int `json:"lastSyncId"`
}
}
err := i.doGraphqlRequest(ctx, existingIssueUpdateStateMutation, &resp, variables, "IssueUpdate")
@@ -176,8 +168,10 @@ type createIssueMutation struct {
ID graphql.ID
Title graphql.String
Identifier graphql.String
State linearState
URL graphql.String
State struct {
Name graphql.String
}
URL graphql.String
}
}
}
@@ -191,7 +185,6 @@ const (
identifier
state {
name
type
}
url
}
@@ -210,28 +203,16 @@ const (
identifier
state {
name
type
}
url
}
}
}
`
completedWorkflowStatesQuery = `query ($teamID: ID) {
workflowStates(filter: { team: { id: { eq: $teamID } }, type: { eq: "completed" } }) {
nodes {
id
position
}
}
}
`
existingIssueUpdateStateMutation = `mutation IssueUpdate($issueUpdateInput: IssueUpdateInput!, $issueID: String!) {
issueUpdate(input: $issueUpdateInput, id: $issueID) {
lastSyncId
success
}
}
`
@@ -271,8 +252,12 @@ func (i *Integration) createIssueLinear(ctx context.Context, title, description
ID: mutation.IssueCreate.Issue.ID,
Title: mutation.IssueCreate.Issue.Title,
Identifier: mutation.IssueCreate.Issue.Identifier,
State: mutation.IssueCreate.Issue.State,
URL: mutation.IssueCreate.Issue.URL,
State: struct {
Name graphql.String
}{
Name: mutation.IssueCreate.Issue.State.Name,
},
URL: mutation.IssueCreate.Issue.URL,
}, nil
}
@@ -282,7 +267,7 @@ func (i *Integration) findIssueByTitle(ctx context.Context, title string) (*line
"title": graphql.String(title),
}
if i.options.TeamID != "" {
variables["teamID"] = graphql.ID(i.options.TeamID)
variables["teamId"] = graphql.ID(i.options.TeamID)
}
if i.options.ProjectID != "" {
variables["projectID"] = graphql.ID(i.options.ProjectID)
@@ -304,84 +289,12 @@ func (i *Integration) Name() string {
}
func (i *Integration) CloseIssue(event *output.ResultEvent) error {
ctx := context.Background()
existingIssue, err := i.findIssueByTitle(ctx, format.Summary(event))
if errors.Is(err, io.EOF) {
return nil
}
if err != nil {
return fmt.Errorf("could not find linear issue: %w", err)
}
if isClosedStateType(string(existingIssue.State.Type)) {
return nil
}
stateID, err := i.resolveClosedStateID(ctx)
if err != nil {
return fmt.Errorf("could not resolve linear closed state: %w", err)
}
variables := map[string]interface{}{
"issueUpdateInput": map[string]interface{}{"stateId": stateID},
"issueID": types.ToString(existingIssue.ID),
}
var resp struct {
IssueUpdate struct {
LastSyncID int `json:"lastSyncId"`
Success bool `json:"success"`
}
}
if err := i.doGraphqlRequest(ctx, existingIssueUpdateStateMutation, &resp, variables, "IssueUpdate"); err != nil {
return fmt.Errorf("error closing issue %s: %w", existingIssue.ID, err)
}
if !resp.IssueUpdate.Success {
return fmt.Errorf("error closing issue %s: Linear rejected the update", existingIssue.ID)
}
// TODO: Unimplemented for now as not used in many places
// and overhead of maintaining our own API for this.
// This is too much code as it is :(
return nil
}
func isClosedStateType(stateType string) bool {
switch stateType {
case linearStateTypeCompleted, linearStateTypeCanceled, linearStateTypeDuplicate:
return true
}
return false
}
func (i *Integration) resolveClosedStateID(ctx context.Context) (string, error) {
if i.options.ClosedStateID != "" {
return i.options.ClosedStateID, nil
}
i.closedStateMu.Lock()
cached := i.closedStateID
i.closedStateMu.Unlock()
if cached != "" {
return cached, nil
}
if i.options.TeamID == "" {
return "", errors.New("team-id or closed-state-id is required to close issues")
}
var result completedWorkflowStatesQueryResult
variables := map[string]interface{}{"teamID": graphql.ID(i.options.TeamID)}
if err := i.doGraphqlRequest(ctx, completedWorkflowStatesQuery, &result, variables, ""); err != nil {
return "", err
}
nodes := result.WorkflowStates.Nodes
if len(nodes) == 0 {
return "", fmt.Errorf("no completed workflow state found for team %s", i.options.TeamID)
}
sort.SliceStable(nodes, func(a, b int) bool { return nodes[a].Position < nodes[b].Position })
stateID := types.ToString(nodes[0].ID)
i.closedStateMu.Lock()
i.closedStateID = stateID
i.closedStateMu.Unlock()
return stateID, nil
}
// ShouldFilter determines if an issue should be logged to this tracker
func (i *Integration) ShouldFilter(event *output.ResultEvent) bool {
if i.options.AllowList != nil && !i.options.AllowList.GetMatch(event) {
@@ -395,26 +308,14 @@ func (i *Integration) ShouldFilter(event *output.ResultEvent) bool {
return true
}
type linearState struct {
Name graphql.String
Type graphql.String
}
type linearIssue struct {
ID graphql.ID
Title graphql.String
Identifier graphql.String
State linearState
URL graphql.String
}
type completedWorkflowStatesQueryResult struct {
WorkflowStates struct {
Nodes []struct {
ID graphql.ID
Position graphql.Float
}
State struct {
Name graphql.String
}
URL graphql.String
}
type findExistingIssuesSearch struct {
@@ -434,12 +335,6 @@ func (adt *addHeaderTransport) RoundTrip(req *http.Request) (*http.Response, err
return adt.T.RoundTrip(req)
}
const (
linearStateTypeCompleted = "completed"
linearStateTypeCanceled = "canceled"
linearStateTypeDuplicate = "duplicate"
)
const (
linearPriorityNone = float64(0)
linearPriorityCritical = float64(1)
@@ -1,140 +0,0 @@
package linear
import (
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v3/pkg/model"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
)
type fakeLinear struct {
mu sync.Mutex
stateType string
rejectUpdates bool
holdStates chan struct{}
mutations []map[string]any
searches []map[string]any
stateQuery int
}
func (f *fakeLinear) ServeHTTP(w http.ResponseWriter, r *http.Request) {
var body struct {
Query string `json:"query"`
Variables map[string]any `json:"variables"`
}
_ = json.NewDecoder(r.Body).Decode(&body)
f.mu.Lock()
defer f.mu.Unlock()
w.Header().Set("Content-Type", "application/json")
switch {
case strings.Contains(body.Query, "workflowStates"):
f.stateQuery++
if f.holdStates != nil {
f.mu.Unlock()
<-f.holdStates
f.mu.Lock()
}
_, _ = w.Write([]byte(`{"data":{"workflowStates":{"nodes":[{"id":"state-late","position":9},{"id":"state-done","position":2}]}}}`))
case strings.Contains(body.Query, "issueUpdate"):
f.mutations = append(f.mutations, body.Variables)
success := "true"
if f.rejectUpdates {
success = "false"
}
_, _ = w.Write([]byte(`{"data":{"issueUpdate":{"lastSyncId":1,"success":` + success + `}}}`))
default:
f.searches = append(f.searches, body.Variables)
_, _ = w.Write([]byte(`{"data":{"issues":{"nodes":[{"id":"issue-1","title":"t","identifier":"ENG-1","state":{"name":"Todo","type":"` + f.stateType + `"},"url":"u"}]}}}`))
}
}
func newTestIntegration(t *testing.T, fake *fakeLinear) *Integration {
t.Helper()
server := httptest.NewServer(fake)
t.Cleanup(server.Close)
integration, err := New(&Options{APIKey: "key", TeamID: "team-1"})
require.NoError(t, err)
integration.url = server.URL
return integration
}
func testEvent() *output.ResultEvent {
return &output.ResultEvent{TemplateID: "tpl", Host: "example.com", Info: model.Info{Name: "finding"}}
}
func TestCloseIssueMovesToDiscoveredCompletedState(t *testing.T) {
fake := &fakeLinear{stateType: "unstarted"}
integration := newTestIntegration(t, fake)
require.NoError(t, integration.CloseIssue(testEvent()))
require.NoError(t, integration.CloseIssue(testEvent()))
require.Len(t, fake.mutations, 2)
require.Equal(t, 1, fake.stateQuery)
require.Equal(t, "team-1", fake.searches[0]["teamID"])
require.Equal(t, "issue-1", fake.mutations[0]["issueID"])
require.Equal(t, map[string]any{"stateId": "state-done"}, fake.mutations[0]["issueUpdateInput"])
}
func TestCloseIssueDoesNotHoldTheLockDuringTheStateLookup(t *testing.T) {
fake := &fakeLinear{stateType: "unstarted", holdStates: make(chan struct{})}
integration := newTestIntegration(t, fake)
release := sync.OnceFunc(func() { close(fake.holdStates) })
t.Cleanup(release)
errs := make(chan error, 2)
for range 2 {
go func() { errs <- integration.CloseIssue(testEvent()) }()
}
require.Eventually(t, func() bool {
fake.mu.Lock()
defer fake.mu.Unlock()
return fake.stateQuery == 2
}, 2*time.Second, 10*time.Millisecond, "a second close waits on the first one's Linear request")
release()
require.NoError(t, <-errs)
require.NoError(t, <-errs)
require.Len(t, fake.mutations, 2)
require.Equal(t, map[string]any{"stateId": "state-done"}, fake.mutations[1]["issueUpdateInput"])
}
func TestCloseIssueUsesConfiguredClosedStateID(t *testing.T) {
fake := &fakeLinear{stateType: "started"}
integration := newTestIntegration(t, fake)
integration.options.ClosedStateID = "configured"
require.NoError(t, integration.CloseIssue(testEvent()))
require.Len(t, fake.mutations, 1)
require.Zero(t, fake.stateQuery)
require.Equal(t, map[string]any{"stateId": "configured"}, fake.mutations[0]["issueUpdateInput"])
}
func TestCloseIssueSkipsAlreadyClosedIssue(t *testing.T) {
for _, stateType := range []string{"completed", "canceled", "duplicate"} {
fake := &fakeLinear{stateType: stateType}
integration := newTestIntegration(t, fake)
require.NoError(t, integration.CloseIssue(testEvent()))
require.Empty(t, fake.mutations, stateType)
}
}
func TestCloseIssueFailsWhenLinearRejectsTheUpdate(t *testing.T) {
fake := &fakeLinear{stateType: "started", rejectUpdates: true}
integration := newTestIntegration(t, fake)
require.ErrorContains(t, integration.CloseIssue(testEvent()), "Linear rejected the update")
}
+67 -20
View File
@@ -2,11 +2,13 @@ package flow
import (
"bufio"
"context"
"fmt"
"io"
"strconv"
"strings"
"sync/atomic"
"time"
"github.com/projectdiscovery/goja"
"github.com/projectdiscovery/nuclei/v3/pkg/js/compiler"
@@ -26,8 +28,18 @@ import (
var (
// ErrInvalidRequestID is a request id error
ErrInvalidRequestID = errkit.New("invalid request id provided")
// ErrFlowRuntimeTerminationTimeout is returned when a flow VM does not stop after cancellation.
ErrFlowRuntimeTerminationTimeout = errkit.New("flow runtime failed to terminate after cancellation")
// ErrFlowProtocolRequestLimit is returned when Flow exceeds its configured protocol request budget.
ErrFlowProtocolRequestLimit = errkit.New("flow protocol request limit exceeded")
)
const flowRuntimeTerminationGrace = time.Second
type flowRunResult struct {
err error
}
// ProtoOptions are options that can be passed to flow protocol callback
// ex: dns(protoOptions) <- protoOptions are optional and can be anything
type ProtoOptions struct {
@@ -55,7 +67,8 @@ type FlowExecutor struct {
flattenKeys []string
// executions counts how many times the flow ran each request
executions map[protocols.Request]*atomic.Int64
executions map[protocols.Request]*atomic.Int64
protocolRequestCount atomic.Int64
}
// NewFlowExecutor creates a new flow executor from a list of requests
@@ -232,12 +245,8 @@ func (f *FlowExecutor) ExecuteWithResults(ctx *scan.ScanContext) error {
// get a new runtime from pool
runtime := GetJSRuntime(f.options.Options)
defer func() {
// whether to reuse or not depends on the whether script modifies
// global scope or not,
PutJSRuntime(runtime, compiler.CanRunAsIIFE(f.options.Flow))
}()
defer func() {
cleanupRuntime := func() {
runtime.ClearInterrupt()
// remove set builtin
_ = runtime.GlobalObject().Delete("set")
_ = runtime.GlobalObject().Delete("template")
@@ -245,16 +254,14 @@ func (f *FlowExecutor) ExecuteWithResults(ctx *scan.ScanContext) error {
_ = runtime.GlobalObject().Delete(proto)
}
runtime.RemoveContextValue("executionId")
}()
// TODO(dwisiswant0): remove this once we get the RCA.
// whether to reuse or not depends on the whether script modifies
// global scope or not,
PutJSRuntime(runtime, compiler.CanRunAsIIFE(f.options.Flow))
}
cleanupOnReturn := true
defer func() {
if ci.IsCI() {
return
}
if r := recover(); r != nil {
f.ctx.LogError(fmt.Errorf("panic occurred while executing flow: %v", r))
if cleanupOnReturn {
cleanupRuntime()
}
}()
@@ -288,11 +295,19 @@ func (f *FlowExecutor) ExecuteWithResults(ctx *scan.ScanContext) error {
runtime.SetContextValue("executionId", f.options.Options.ExecutionId)
// pass flow and execute the js vm and handle errors
_, err := runtime.RunProgram(f.program)
runResult, abandoned := runFlowProgram(ctx.Context(), runtime, f.program)
if abandoned != nil {
cleanupOnReturn = false
go func() {
<-abandoned
cleanupRuntime()
}()
return errkit.Wrapf(ErrFlowRuntimeTerminationTimeout, "%v", ctx.Context().Err())
}
f.reconcileProgress()
if err != nil {
ctx.LogError(err)
return errkit.Wrapf(err, "failed to execute flow\n%v\n", f.options.Flow)
if runResult.err != nil {
ctx.LogError(runResult.err)
return errkit.Wrapf(runResult.err, "failed to execute flow\n%v\n", f.options.Flow)
}
runtimeErr := f.GetRuntimeErrors()
if runtimeErr != nil {
@@ -303,6 +318,38 @@ func (f *FlowExecutor) ExecuteWithResults(ctx *scan.ScanContext) error {
return nil
}
func runFlowProgram(ctx context.Context, runtime *goja.Runtime, program *goja.Program) (flowRunResult, <-chan flowRunResult) {
result := make(chan flowRunResult, 1)
go func() {
outcome := flowRunResult{}
defer func() {
if recovered := recover(); recovered != nil {
if ci.IsCI() {
panic(recovered)
}
outcome.err = fmt.Errorf("panic occurred while executing flow: %v", recovered)
}
result <- outcome
}()
_, outcome.err = runtime.RunProgram(program)
}()
select {
case outcome := <-result:
return outcome, nil
case <-ctx.Done():
runtime.Interrupt(ctx.Err())
timer := time.NewTimer(flowRuntimeTerminationGrace)
defer timer.Stop()
select {
case outcome := <-result:
return outcome, nil
case <-timer.C:
return flowRunResult{}, result
}
}
}
// reconcileProgress settles progress, whose total assumed each request runs
// once: requests the flow never ran count as processed, as protocols do for
// skipped requests, and extra runs of a request are added to the total
+73 -23
View File
@@ -51,6 +51,79 @@ func setup() {
executerOpts.WorkflowLoader = workflowLoader
}
func TestFlowStopsWhenContextCanceled(t *testing.T) {
setup()
templatePath := filepath.Join(t.TempDir(), "infinite-flow.yaml")
require.NoError(t, os.WriteFile(templatePath, []byte(`id: infinite-flow
info:
name: Infinite Flow
author: pdteam
severity: info
flow: while (true) {}
http:
- method: GET
path:
- "{{BaseURL}}"
`), 0o600))
tmpl, err := templates.Parse(templatePath, nil, executerOpts)
require.NoError(t, err)
require.NoError(t, tmpl.Executer.Compile())
executionContext, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
defer cancel()
input := contextargs.NewWithInput(executionContext, "https://example.com")
scanContext := scan.NewScanContext(executionContext, input)
result := make(chan error, 1)
go func() {
_, executeErr := tmpl.Executer.Execute(scanContext)
result <- executeErr
}()
select {
case executeErr := <-result:
require.ErrorIs(t, executeErr, context.DeadlineExceeded)
case <-time.After(time.Second):
t.Fatal("flow execution did not stop after context cancellation")
}
}
func TestFlowProtocolRequestLimit(t *testing.T) {
setup()
executerOpts.MaxFlowProtocolRequests = 2
var requests atomic.Int64
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
requests.Add(1)
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
templatePath := filepath.Join(t.TempDir(), "bounded-flow.yaml")
require.NoError(t, os.WriteFile(templatePath, []byte(`id: bounded-flow
info:
name: Bounded Flow
author: pdteam
severity: info
flow: for (let i = 0; i < 3; i++) { http(); }
http:
- method: GET
path:
- "{{BaseURL}}"
`), 0o600))
tmpl, err := templates.Parse(templatePath, nil, executerOpts)
require.NoError(t, err)
require.NoError(t, tmpl.Executer.Compile())
input := contextargs.NewWithInput(context.Background(), server.URL)
scanContext := scan.NewScanContext(context.Background(), input)
_, err = tmpl.Executer.Execute(scanContext)
require.ErrorContains(t, err, "flow protocol request limit exceeded")
require.Equal(t, int64(2), requests.Load())
}
func TestFlowTemplateWithIndex(t *testing.T) {
// test
setup()
@@ -287,10 +360,6 @@ func newMultiPayloadServer() *httptest.Server {
// survive across multiple pitchfork payload iterations (generator restarts).
func TestFlowMultiPayloadIteration(t *testing.T) {
setup()
options := executerOpts.Options
previousConcurrency := options.PayloadConcurrency
options.PayloadConcurrency = 2
t.Cleanup(func() { options.PayloadConcurrency = previousConcurrency })
ts := newMultiPayloadServer()
defer ts.Close()
@@ -299,7 +368,6 @@ func TestFlowMultiPayloadIteration(t *testing.T) {
err = tmpl.Executer.Compile()
require.Nil(t, err, "could not compile template")
require.Zero(t, tmpl.RequestsHTTP[0].Threads, "request-condition payloads require ordered execution")
input := contextargs.NewWithInput(context.Background(), ts.URL)
ctx := scan.NewScanContext(context.Background(), input)
@@ -308,24 +376,6 @@ func TestFlowMultiPayloadIteration(t *testing.T) {
require.True(t, gotresults, "expected match with flow + multi payload iteration")
}
func TestIndependentMultiPayloadRequestsRemainParallel(t *testing.T) {
setup()
options := executerOpts.Options
previousConcurrency := options.PayloadConcurrency
options.PayloadConcurrency = 2
t.Cleanup(func() { options.PayloadConcurrency = previousConcurrency })
tmpl, err := templates.Parse("testcases/flow-multi-payload-iteration.yaml", nil, executerOpts)
require.NoError(t, err)
for _, matcher := range tmpl.RequestsHTTP[0].Matchers {
matcher.Part = "body"
}
require.False(t, tmpl.RequestsHTTP[0].NeedsRequestCondition())
require.NoError(t, tmpl.Executer.Compile())
require.Positive(t, tmpl.RequestsHTTP[0].Threads)
}
// TestFlowRequestConditionWithPayloads is the same as above but with
// pitchfork payloads and header-based extraction, matching the exact
// pattern from issue #5095.
+32
View File
@@ -34,6 +34,9 @@ func (f *FlowExecutor) requestExecutor(runtime *goja.Runtime, reqMap mapsutil.Ma
// execution logic for http()/dns() etc
for index := range f.allProtocols[opts.protoName] {
req := f.allProtocols[opts.protoName][index]
if !f.reserveProtocolRequests(runtime, req) {
return matcherStatus.Load()
}
// transform input if required
inputItem := f.ctx.Input.Clone()
if f.options.InputHelper != nil && f.ctx.Input.MetaInput.Input != "" && !utils.HasOfflineHTTPResponse(f.ctx.Input.MetaInput, req.Type()) {
@@ -72,6 +75,9 @@ func (f *FlowExecutor) requestExecutor(runtime *goja.Runtime, reqMap mapsutil.Ma
}
return matcherStatus.Load()
}
if !f.reserveProtocolRequests(runtime, req) {
return matcherStatus.Load()
}
// transform input if required
inputItem := f.ctx.Input.Clone()
if f.options.InputHelper != nil && f.ctx.Input.MetaInput.Input != "" && !utils.HasOfflineHTTPResponse(f.ctx.Input.MetaInput, req.Type()) {
@@ -93,6 +99,32 @@ func (f *FlowExecutor) requestExecutor(runtime *goja.Runtime, reqMap mapsutil.Ma
return matcherStatus.Load()
}
func (f *FlowExecutor) reserveProtocolRequests(runtime *goja.Runtime, req protocols.Request) bool {
limit := f.options.MaxFlowProtocolRequests
if limit <= 0 {
return true
}
cost := int64(req.Requests())
if cost < 1 {
cost = 1
}
for {
current := f.protocolRequestCount.Load()
if cost > limit-current {
err := errkit.Wrapf(ErrFlowProtocolRequestLimit, "maximum is %d", limit)
if storeErr := f.allErrs.Set("flow:request-limit", err); storeErr != nil {
f.ctx.LogError(fmt.Errorf("failed to store flow runtime errors got %v", storeErr))
}
runtime.Interrupt(err)
return false
}
if f.protocolRequestCount.CompareAndSwap(current, current+cost) {
return true
}
}
}
// protocolResultCallback returns a callback that is executed
// after execution of each protocol request
func (f *FlowExecutor) protocolResultCallback(req protocols.Request, matcherStatus *atomic.Bool, _ *ProtoOptions) func(result *output.InternalWrappedEvent) {
-9
View File
@@ -103,8 +103,6 @@ type Options struct {
ListDslSignatures bool
// List of HTTP(s)/SOCKS5 proxy to use (comma separated or file input)
Proxy goflags.StringSlice
// ReplayProxy re-sends the requests that produced a finding through this proxy
ReplayProxy string
// AliveProxy is the alive proxy to use
AliveHttpProxy string
// AliveSocksProxy is the alive socks proxy to use
@@ -211,12 +209,6 @@ type Options struct {
// PreflightPortScan enables a preflight resolve + TCP portscan and filters targets
// before running templates. Disabled by default.
PreflightPortScan bool
// HostBackoff slows a host down when it signals it is being overloaded
// (429, a run of 403s, or transport failures) instead of scanning it at full
// rate until it blocks the scanner. Opt-in, since it changes scan pacing.
HostBackoff bool
// HostBackoffMax caps the per-host backoff delay.
HostBackoffMax time.Duration
// MaxURLsPerPattern caps how many targets sharing one structural shape
// (/user/1, /user/2, ...) are scanned. Zero, the default, scans every
// target: dropping one means never scanning it.
@@ -629,7 +621,6 @@ func (options *Options) Copy() *Options {
Resume: options.Resume,
Output: options.Output,
ProxyInternal: options.ProxyInternal,
ReplayProxy: options.ReplayProxy,
ListDslSignatures: options.ListDslSignatures,
Proxy: options.Proxy,
AliveHttpProxy: options.AliveHttpProxy,