Compare commits

..
Author SHA1 Message Date
Nakul Bharti 649697c9af fix(headless): synchronize page wrapper copies with history updates 2026-10-19 22:55:28 +05:30
Nakul Bharti d2b4564681 fix(http): keep memory guard from raising template threads 2026-10-19 22:32:31 +05:30
Nakul Bharti f438963e83 fix(http): preserve memory limit when resizing payload workers 2026-10-19 22:17:03 +05:30
Nakul Bharti aaa6827409 fix(http): apply memory concurrency guard once 2026-10-19 22:09:37 +05:30
Nakul Bharti 5915f21f50 fix(http): keep dependent payload requests ordered 2026-10-19 17:37:15 +05:30
Nakul Bharti b76d8de501 fix(http): preserve queued results on cancellation
(cherry picked from commit 190fb46436)
2026-10-19 16:25:19 +05:30
Nakul Bharti bfff4f0d2c fix(http): stop result receiver on handler cancellation 2026-10-19 15:15:56 +05:30
Nakul Bharti f238298505 fix(http): exclude host backoff from rate-limit wait timing 2026-10-19 15:00:56 +05:30
Nakul Bharti 108c8e28c1 fix(http): skip unresponsive hosts before rate limiting 2026-10-19 14:30:35 +05:30
Nakul Bharti 342994fe2f Merge Nuclei dev while retaining headless timing and memory guard 2026-10-19 14:30:25 +05:30
Nakul Bharti 7211cb35fb Merge remote-tracking branch 'origin/dev' into codex/headless-timing-output 2026-10-18 13:54:59 +05:30
Nakul Bharti 4a2c5c2be6 fix(headless): preserve selected timing across failed actions and custom loggers 2026-10-18 13:54:06 +05:30
Nakul Bharti 2ae0db918a chore(headless): add bounded opt-in action timing for slow templates 2026-10-18 04:35:04 +05:30
Nakul Bharti ef4eb1dcf2 Merge commit 'ef4ad4f4c77a61010d1176efb80b6a3992878fc0' into codex/aurora-nuclei-memory-integration 2026-10-18 02:41:36 +05:30
Nakul Bharti ef4ad4f4c7 fix(protocolstate): use cgroup-aware memory guardian 2026-10-18 02:40:10 +05:30
Nakul Bharti f8a9087ccf fix(nuclei): update low-memory warning used by protocol guards 2026-10-18 01:29:19 +05:30
Nakul Bharti 857cc67702 fix(headless): avoid waiting for an already-fired load event 2026-10-17 09:03:40 +05:30
30 changed files with 536 additions and 1628 deletions

No files matched your search

+20 -19
View File
@@ -76,7 +76,7 @@ require (
github.com/praetorian-inc/fingerprintx v1.1.15
github.com/projectdiscovery/clistats v0.1.7
github.com/projectdiscovery/dsl v0.8.24
github.com/projectdiscovery/fastdialer v0.5.26
github.com/projectdiscovery/fastdialer v0.5.25
github.com/projectdiscovery/fasttemplate v0.0.2
github.com/projectdiscovery/gcache v0.0.0-20241015120333-12546c6e3f4c
github.com/projectdiscovery/goflags v0.2.1
@@ -92,18 +92,18 @@ require (
github.com/projectdiscovery/interactsh v1.4.1
github.com/projectdiscovery/mapcidr v1.1.97
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5
github.com/projectdiscovery/networkpolicy v0.1.55
github.com/projectdiscovery/ratelimit v0.0.94
github.com/projectdiscovery/rawhttp v0.2.0
github.com/projectdiscovery/networkpolicy v0.1.54
github.com/projectdiscovery/ratelimit v0.0.93
github.com/projectdiscovery/rawhttp v0.1.92
github.com/projectdiscovery/rdap v0.9.0
github.com/projectdiscovery/retryabledns v1.0.116
github.com/projectdiscovery/retryablehttp-go v1.3.31
github.com/projectdiscovery/retryablehttp-go v1.3.30
github.com/projectdiscovery/sarif v0.1.0
github.com/projectdiscovery/tlsx v1.4.3-0.20261020052910-acb80c2c7c75
github.com/projectdiscovery/tlsx v1.4.1
github.com/projectdiscovery/uncover v1.2.1
github.com/projectdiscovery/useragent v0.0.109
github.com/projectdiscovery/utils v0.11.11-0.20261020211236-cde16c1381ec
github.com/projectdiscovery/wappalyzergo v0.3.5
github.com/projectdiscovery/utils v0.11.10-0.20261019085724-c5c64e928d17
github.com/projectdiscovery/wappalyzergo v0.3.4
github.com/projectdiscovery/yamldoc-go v1.0.7
github.com/redis/go-redis/v9 v9.19.0
github.com/remeh/sizedwaitgroup v1.0.0
@@ -124,12 +124,12 @@ require (
go.mongodb.org/mongo-driver v1.17.9
go.uber.org/multierr v1.11.0
golang.org/x/exp v0.0.0-20260527015227-08cc5374adb3
golang.org/x/net v0.59.0
golang.org/x/net v0.58.0
golang.org/x/oauth2 v0.36.0
golang.org/x/sync v0.23.0
golang.org/x/sys v0.48.0
golang.org/x/term v0.46.0
golang.org/x/text v0.42.0
golang.org/x/sync v0.22.0
golang.org/x/sys v0.47.0
golang.org/x/term v0.45.0
golang.org/x/text v0.41.0
golang.org/x/time v0.15.0
golang.org/x/tools v0.49.0
google.golang.org/grpc v1.83.2
@@ -375,7 +375,7 @@ require (
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
github.com/projectdiscovery/asnmap v1.1.1 // indirect
github.com/projectdiscovery/blackrock v0.0.2 // indirect
github.com/projectdiscovery/cdncheck v1.3.2 // indirect
github.com/projectdiscovery/cdncheck v1.3.1 // indirect
github.com/projectdiscovery/freeport v0.0.7 // indirect
github.com/projectdiscovery/jarm-go v0.0.0-20260910160638-430e2bae86f6 // indirect
github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb // indirect
@@ -389,7 +389,7 @@ require (
github.com/sergi/go-diff v1.4.0 // indirect
github.com/shirou/gopsutil v3.21.11+incompatible // indirect
github.com/shirou/gopsutil/v4 v4.26.3 // indirect
github.com/sirupsen/logrus v1.10.2 // indirect
github.com/sirupsen/logrus v1.9.4 // indirect
github.com/skeema/knownhosts v1.3.1 // indirect
github.com/sorairolake/lzip-go v0.3.8 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
@@ -418,7 +418,7 @@ require (
github.com/vmihailenco/tagparser v0.1.2 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/vulncheck-oss/go-exploit v1.51.0 // indirect
github.com/weppos/publicsuffix-go v0.50.4-0.20260918082853-3c3d7dc4a0cb // indirect
github.com/weppos/publicsuffix-go v0.50.3 // indirect
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
@@ -439,7 +439,8 @@ require (
github.com/yusufpapurcu/wmi v1.2.4 // indirect
github.com/zcalusic/sysinfo v1.1.3 // indirect
github.com/zeebo/blake3 v0.2.4 // indirect
github.com/zmap/zcrypto v0.0.0-20261015050653-aa1de3126e87 // indirect
github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect
github.com/zmap/zcrypto v0.0.0-20240803002437-3a861682ac77 // indirect
go.etcd.io/bbolt v1.4.3 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect
@@ -457,8 +458,8 @@ require (
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
goftp.io/server/v2 v2.0.1 // indirect
golang.org/x/arch v0.3.0 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/mod v0.41.0 // indirect
golang.org/x/crypto v0.55.0 // indirect
golang.org/x/mod v0.39.0 // indirect
google.golang.org/api v0.264.0 // indirect
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect
+53 -38
View File
@@ -130,6 +130,7 @@ github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEV
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk=
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g=
github.com/ProtonMail/go-crypto v1.1.6 h1:ZcV+Ropw6Qn0AX9brlQLAUXfqLBc7Bl+f/DmNxpLfdw=
github.com/ProtonMail/go-crypto v1.1.6/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE=
github.com/PuerkitoBio/goquery v1.13.0 h1:mqHbjD7Jmnul4DTR24LKTjo1uUmHUh072kteGV+xpFM=
@@ -266,6 +267,7 @@ github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/
github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU5DoEw9xY/FUi1c=
github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk=
github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M=
github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM=
github.com/bytedance/sonic v1.15.3 h1:P3akjLPBtV/i6bHC6LbcLjY3KuoOvfiqF8wFHeP5IhY=
@@ -323,6 +325,7 @@ github.com/clbanning/mxj/v2 v2.7.0/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cloudflare/cfssl v1.6.5 h1:46zpNkm6dlNkMZH/wMW22ejih6gIaJbzL2du6vD7ZeI=
github.com/cloudflare/cfssl v1.6.5/go.mod h1:Bk1si7sq8h2+yVEDrFJiz3d7Aw+pfjjJSZVaD+Taky4=
github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I=
github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8=
github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4=
github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M=
@@ -589,6 +592,7 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
github.com/google/go-github/v50 v50.2.0/go.mod h1:VBY8FB6yPIjrtKhozXv4FQupxKLS6H4m6xFZlT43q8Q=
github.com/google/go-github/v92 v92.0.0 h1:4vW4RVffwIvoEfIA4RX09mRSB47qQX557/2+HgeWRRg=
github.com/google/go-github/v92 v92.0.0/go.mod h1:w3CH62ZcmRfvW1cdXpyTztSOVMtdjxtKpgo0GouLmjY=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
@@ -931,14 +935,14 @@ github.com/projectdiscovery/asnmap v1.1.1 h1:ImJiKIaACOT7HPx4Pabb5dksolzaFYsD1kI
github.com/projectdiscovery/asnmap v1.1.1/go.mod h1:QT7jt9nQanj+Ucjr9BqGr1Q2veCCKSAVyUzLXfEcQ60=
github.com/projectdiscovery/blackrock v0.0.2 h1:mxXdu0uM8P2L2Qi210COlU8QiICPFW/Rxk5QUhlPO2k=
github.com/projectdiscovery/blackrock v0.0.2/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss=
github.com/projectdiscovery/cdncheck v1.3.2 h1:I453ZWNBeG0WWIy3wi9412l4PTzorLgebEnqmbcpSfo=
github.com/projectdiscovery/cdncheck v1.3.2/go.mod h1:5IkUMMlcGaeg1DWW5oJVd3r9X3EpJ88j5HxBeF4Me9M=
github.com/projectdiscovery/cdncheck v1.3.1 h1:+qPKwqdf3mFtrw3ZoVti7hembmVaapnJBjRosYTNdRo=
github.com/projectdiscovery/cdncheck v1.3.1/go.mod h1:5IkUMMlcGaeg1DWW5oJVd3r9X3EpJ88j5HxBeF4Me9M=
github.com/projectdiscovery/clistats v0.1.7 h1:NXf4lUzx1QcbJd6QycNwwYHTv2FsW9MchQ2LsgSL6C0=
github.com/projectdiscovery/clistats v0.1.7/go.mod h1:RxRGxhMzkT344VTTvZWIrojurUXdQQRN1yfwgg/88I8=
github.com/projectdiscovery/dsl v0.8.24 h1:rN2w6LSIjA16qKGRjyj70oxdwXGaspe6sAjEkP7F2fk=
github.com/projectdiscovery/dsl v0.8.24/go.mod h1:V+EludvS5PS19UNMlM3CuBwZ+sFnMaQhCFGxysDUanQ=
github.com/projectdiscovery/fastdialer v0.5.26 h1:wCCsopLsO1EF1V1+9Wqkxj1bMm6sRo8NdlzwE2Ib2Ss=
github.com/projectdiscovery/fastdialer v0.5.26/go.mod h1:dZzEPnpsxqCiD3IkM5CN3RglW3l1gXJNyTkxqjPZIbs=
github.com/projectdiscovery/fastdialer v0.5.25 h1:xGFpc5B7cmogqnBrcVYV5yXh6UFlqRA6u5KqautKgD0=
github.com/projectdiscovery/fastdialer v0.5.25/go.mod h1:CZZ0PwdTw2KCI/PrZTW9wyBY88MIjpT8WhsKWSw3bkw=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
github.com/projectdiscovery/freeport v0.0.7 h1:Q6uXo/j8SaV/GlAHkEYQi8WQoPXyJWxyspx+aFmz9Qk=
@@ -977,32 +981,32 @@ github.com/projectdiscovery/mapcidr v1.1.97 h1:7FkxNNVXp+m1rIu5Nv/2SrF9k4+LwP8Qu
github.com/projectdiscovery/mapcidr v1.1.97/go.mod h1:9dgTJh1SP02gYZdpzMjm6vtYFkEHQHoTyaVNvaeJ7lA=
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw1pfT6bg35NiN7yd1XKtJap5Nk6lMwQ0RNi8=
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc=
github.com/projectdiscovery/networkpolicy v0.1.55 h1:D30pipxOdWnSq3r7sX0kQvGyb18sUgopuda1j9NAmnQ=
github.com/projectdiscovery/networkpolicy v0.1.55/go.mod h1:N5bXm8L3UsDw51CeA1nhTsGEbQMbgL5lXTiIU013xd0=
github.com/projectdiscovery/ratelimit v0.0.94 h1:2qciSuUxarn73YGICZeprZtjrwyAH+0iVpdKzKCXoXE=
github.com/projectdiscovery/ratelimit v0.0.94/go.mod h1:GAerX6s+eWRuACedHRv2/xmMtlW+VahZmbRstQh0sS0=
github.com/projectdiscovery/rawhttp v0.2.0 h1:TGiHC5hydL97/zoi0WjEuNrfO6Gjtg9LeXndUf1Cq3M=
github.com/projectdiscovery/rawhttp v0.2.0/go.mod h1:Waz+K5y6r44ElpqAoOfYtY03zrGGZ12nx0yDKq7Ot1E=
github.com/projectdiscovery/networkpolicy v0.1.54 h1:+v7yyuRBu2S7dzeIJToDENy1i+HjRhNzqnQrDlm00z8=
github.com/projectdiscovery/networkpolicy v0.1.54/go.mod h1:tN+YAFVlJjHv/znPEhkcSAjmBMkMhslC9XpWKTrKEHw=
github.com/projectdiscovery/ratelimit v0.0.93 h1:k2rRpCR31DqeqrORSgGzJ+z0T7+QGCt0Z0+Ov/TVrgk=
github.com/projectdiscovery/ratelimit v0.0.93/go.mod h1:Tqpx1QKdFI63L01T2gpSvV5Uro9CnZkZWr80ciSXSBY=
github.com/projectdiscovery/rawhttp v0.1.92 h1:38GRsVLNllES/dZcEcA6s6P+rYBp34H9mDphMJMDvvg=
github.com/projectdiscovery/rawhttp v0.1.92/go.mod h1:spfXTPjxCj+bfWLynMVB7uOb+LS3PDNAI9HexMEV1Zg=
github.com/projectdiscovery/rdap v0.9.0 h1:wPhHx5pQ2QI+WGhyNb2PjhTl0NtB39Nk7YFZ9cp8ZGA=
github.com/projectdiscovery/rdap v0.9.0/go.mod h1:zk4yrJFQ2Hy36Aqk+DvotYQxYAeALaCJ5ORySkff36Q=
github.com/projectdiscovery/retryabledns v1.0.116 h1:tK28PJ4tLO7bgM89RpIFjkfTPMSbRegwvg9mN/q6z78=
github.com/projectdiscovery/retryabledns v1.0.116/go.mod h1:r3vH0AKYtJ9fculcqRFrdVYe2ABcHwZeR4C6j3rqCbY=
github.com/projectdiscovery/retryablehttp-go v1.3.31 h1:0hSEZ5+jj6Oo1IL8qrytD36TK/qCxfe543jEaRqSTjs=
github.com/projectdiscovery/retryablehttp-go v1.3.31/go.mod h1:/WEa7YejxTCXEMLCGfGYaC3JsgXgzuqw8rv8cb4buEc=
github.com/projectdiscovery/retryablehttp-go v1.3.30 h1:YnDCZP04/AKTiIBT5vA1nVhy5cRW5dPy1Du5nCjxHy4=
github.com/projectdiscovery/retryablehttp-go v1.3.30/go.mod h1:MXn1MzThNmHc05KldUbfks1MvwAnUKR4GGJcFhVIc08=
github.com/projectdiscovery/sarif v0.1.0 h1:O541T+a448nSJsmIMnXXSOeDQEzpnCAYvRfe0eG5h74=
github.com/projectdiscovery/sarif v0.1.0/go.mod h1:LBC+reM3bkI3qIIhE0rZaINaYX6VG+En6u2hHa5mA7E=
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0=
github.com/projectdiscovery/tlsx v1.4.3-0.20261020052910-acb80c2c7c75 h1:3qJgfWzzwtFXc/vdUIurSzPzyjBH9NQe0wATkZMoQGI=
github.com/projectdiscovery/tlsx v1.4.3-0.20261020052910-acb80c2c7c75/go.mod h1:8jqB9aHT6U8NkUsENvaSjD/iBEGB4Ta+CbJBMcvzP9c=
github.com/projectdiscovery/tlsx v1.4.1 h1:QuECHVJERBirNrmpjgIOAFQScjy/DxVjzXeL+L81mH0=
github.com/projectdiscovery/tlsx v1.4.1/go.mod h1:WCrUXAHjdKbDNjsu5EbVBS7qvzap3fxi/E85v/Flyx8=
github.com/projectdiscovery/uncover v1.2.1 h1:8U46T/96CLT7BPoXBgkTvWqB06lOyeTSLvh5+UjzATE=
github.com/projectdiscovery/uncover v1.2.1/go.mod h1:0p8onrWxfpXQEYs90ZDzTSpu1107gWmodX1NWqu/+z4=
github.com/projectdiscovery/useragent v0.0.109 h1:b86BPdZvmgbJousa04jAugDRY4Y+rAvEamE0TFCPmeE=
github.com/projectdiscovery/useragent v0.0.109/go.mod h1:sRyBqDAcD31RKYrO8fyXGowLd36fNwY1g0MNW9mp7zk=
github.com/projectdiscovery/utils v0.11.11-0.20261020211236-cde16c1381ec h1:VhJiivJR8ljv5Qk3QZQC9og50wspEBB5qkNfdyqNhws=
github.com/projectdiscovery/utils v0.11.11-0.20261020211236-cde16c1381ec/go.mod h1:T7KzyElivWJiPP+B4sXqkp1CcFe6O/P0qBn/MjDNa74=
github.com/projectdiscovery/wappalyzergo v0.3.5 h1:UZradYF1GIcruSIV5wwo/Coph9/VnM17oajLD9eXR2U=
github.com/projectdiscovery/wappalyzergo v0.3.5/go.mod h1:4E6+fh/ckaY3wo5tlM7lNYSkhuTYRHzu2YmGIRraXP0=
github.com/projectdiscovery/utils v0.11.10-0.20261019085724-c5c64e928d17 h1:aSNbcIUA6wZyHaGdZjFzmZ5irbMMg5Yc5WmuXdmnvyI=
github.com/projectdiscovery/utils v0.11.10-0.20261019085724-c5c64e928d17/go.mod h1:d3BcbsHhuxW9T3jpvrgzwshvqhaL9qPSkoi+3h/DY8E=
github.com/projectdiscovery/wappalyzergo v0.3.4 h1:yO+xel+C0d/vOFWcUKVZRpIDAfcEmsjpVDjQ0jifAUw=
github.com/projectdiscovery/wappalyzergo v0.3.4/go.mod h1:6xTRCUQhX8m+L1eqJDQIALJlRv2Bnhrmku80YU9bBXE=
github.com/projectdiscovery/yamldoc-go v1.0.7 h1:Odmjd1o4EkMff2F6QHi1dIYPIQPZUFy24YqmQCprmTk=
github.com/projectdiscovery/yamldoc-go v1.0.7/go.mod h1:nqD1iPMcofpRA1DbXavPByJhWURrshT+l8WN9XjyQag=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
@@ -1075,8 +1079,9 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/sirupsen/logrus v1.10.2 h1:G2SED73/qrAu6YwbdxOD6peLkCBI3z7L+ykJFTXJBBo=
github.com/sirupsen/logrus v1.10.2/go.mod h1:SLEg8TqYulVKKfIGHldVp2K2aYz2DKSVBq4g/H5bR7Q=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8=
github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
@@ -1182,8 +1187,9 @@ github.com/vulncheck-oss/go-exploit v1.51.0/go.mod h1:J28w0dLnA6DnCrnBm9Sbt6smX8
github.com/weppos/publicsuffix-go v0.12.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k=
github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k=
github.com/weppos/publicsuffix-go v0.30.0/go.mod h1:kBi8zwYnR0zrbm8RcuN1o9Fzgpnnn+btVN8uWPMyXAY=
github.com/weppos/publicsuffix-go v0.50.4-0.20260918082853-3c3d7dc4a0cb h1:QIW0GHVTSubm3VxkKpn1FnFAgins6N8FiGTvUQwpXzw=
github.com/weppos/publicsuffix-go v0.50.4-0.20260918082853-3c3d7dc4a0cb/go.mod h1:/rOa781xBykZhHK/I3QeHo92qdDKVmKZKF7s8qAEM/4=
github.com/weppos/publicsuffix-go v0.40.2/go.mod h1:XsLZnULC3EJ1Gvk9GVjuCTZ8QUu9ufE4TZpOizDShko=
github.com/weppos/publicsuffix-go v0.50.3 h1:eT5dcjHQcVDNc0igpFEsGHKIip30feuB2zuuI9eJxiE=
github.com/weppos/publicsuffix-go v0.50.3/go.mod h1:/rOa781xBykZhHK/I3QeHo92qdDKVmKZKF7s8qAEM/4=
github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220927085643-dc0d00c92642/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE=
github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
@@ -1253,14 +1259,15 @@ github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE=
github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 h1:Nzukz5fNOBIHOsnP+6I79kPx3QhLv8nBy2mfFhBRq30=
github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE=
github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is=
github.com/zmap/zcertificate v0.0.1/go.mod h1:q0dlN54Jm4NVSSuzisusQY0hqDWvu92C+TWveAxiVWk=
github.com/zmap/zcrypto v0.0.0-20201128221613-3719af1573cf/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ=
github.com/zmap/zcrypto v0.0.0-20201211161100-e54a5822fb7e/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ=
github.com/zmap/zcrypto v0.0.0-20230310154051-c8b263fd8300/go.mod h1:mOd4yUMgn2fe2nV9KXsa9AyQBFZGzygVPovsZR+Rl5w=
github.com/zmap/zcrypto v0.0.0-20261015050653-aa1de3126e87 h1:B9sBGvtHjiGOOemAh0cCGTzub76DXlxmOh3Qtx3TkRs=
github.com/zmap/zcrypto v0.0.0-20261015050653-aa1de3126e87/go.mod h1:9O2aSz1l9fUHTFxJMYCoFsZBy6yGlz3nrOp4dHue5qw=
github.com/zmap/zcrypto v0.0.0-20240803002437-3a861682ac77 h1:DCz0McWRVJNICkHdu2XpETqeLvPtZXs315OZyUs1BDk=
github.com/zmap/zcrypto v0.0.0-20240803002437-3a861682ac77/go.mod h1:aSvf+uTU222mUYq/KQj3oiEU7ajhCZe8RRSLHIoM4EM=
github.com/zmap/zflags v1.4.0-beta.1.0.20200204220219-9d95409821b6/go.mod h1:HXDUD+uue8yeLHr0eXx1lvY6CvMiHbTKw5nGmA9OUoo=
github.com/zmap/zgrab2 v0.1.8 h1:PFnXrIBcGjYFec1JNbxMKQuSXXzS+SbqE89luuF4ORY=
github.com/zmap/zgrab2 v0.1.8/go.mod h1:5d8HSmUwvllx4q1qG50v/KXphkg45ZzWdaQtgTFnegE=
@@ -1338,9 +1345,10 @@ golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1378,8 +1386,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c=
golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o=
golang.org/x/mod v0.39.0 h1:UF5zwQdCRRUpHfyPwr7d4UrGiVeldIsogtzWVnczL74=
golang.org/x/mod v0.39.0/go.mod h1:bvIbwjQ0HUFFf5AKukeeYQG4ZBUG9yxQbR9aEweIwYY=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1427,9 +1435,10 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1438,6 +1447,7 @@ golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4Iltr
golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -1456,8 +1466,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1504,6 +1514,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -1521,9 +1532,10 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -1535,9 +1547,10 @@ golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4=
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE=
golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1553,9 +1566,10 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1704,6 +1718,7 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
+3 -5
View File
@@ -18,11 +18,9 @@ import (
)
const (
dbName = "test"
dbRepository = "mongo"
// 8.x refuses to start on kernels 6.19 through 7.0.13 (SERVER-125742).
// The GitHub Ubuntu image is in that range. 7.0 stays up.
dbTag = "7.0"
dbName = "test"
dbRepository = "mongo"
dbTag = "8"
dbPort = "27017/tcp"
mongoDatabaseReadyTimeout = 3 * time.Minute
mongoServerSelectionDelay = time.Second
+1 -1
View File
@@ -17,7 +17,7 @@ type multiProtoDynamicExtractor struct{}
// Execute executes a test case and returns an error if occurred
func (h *multiProtoDynamicExtractor) Execute(templatePath string) error {
results, err := testutils.RunNucleiTemplateAndGetResults(templatePath, "https://docs.projectdiscovery.io/", debug)
results, err := testutils.RunNucleiTemplateAndGetResults(templatePath, "docs.projectdiscovery.io", debug)
if err != nil {
return err
}
@@ -24,6 +24,6 @@ http:
matchers:
- type: dsl
dsl:
- location == '/home' # the docs host redirects here
- contains(body,'home') # check for http string
- blogid == 'cname' # check for cname (extracted information from dns response)
condition: and
@@ -24,7 +24,7 @@ http:
matchers:
- type: dsl
dsl:
- http_location == '/home' # the docs host redirects here
- contains(http_body,'home') # check for http string
- cname_filtered == 'cname' # check for cname (extracted information from dns response)
- ssl_subject_cn == 'docs.projectdiscovery.io'
condition: and
@@ -20,7 +20,7 @@ http:
matchers:
- type: dsl
dsl:
- http_location == '/home' # the docs host redirects here
- contains(http_body,'home') # check for http string
- trim_suffix(dns_cname,'.vercel-dns.com') == 'cname' # check for cname (extracted information from dns response)
- ssl_subject_cn == 'docs.projectdiscovery.io'
condition: and
@@ -256,12 +256,6 @@ func (ctx *Context) Clone() *Context {
return newCtx
}
func (ctx *Context) CloneWithContext(parent context.Context) *Context {
cloned := ctx.Clone()
cloned.ctx = parent
return cloned
}
// resolvePortList converts a list of port strings (numeric or service names) to numeric port strings.
func resolvePortList(ports []string) []string {
resolved := make([]string, 0, len(ports))
@@ -7,25 +7,6 @@ import (
"github.com/stretchr/testify/require"
)
func TestCloneWithContextPreservesInputWithoutSharingMutableState(t *testing.T) {
parent := NewWithInput(context.Background(), "https://example.com")
parent.Set("shared", "original")
childCtx, cancel := context.WithCancel(context.Background())
child := parent.CloneWithContext(childCtx)
require.Equal(t, parent.MetaInput.Input, child.MetaInput.Input)
childValue, ok := child.Get("shared")
require.True(t, ok)
require.Equal(t, "original", childValue)
child.Set("shared", "changed")
parentValue, ok := parent.Get("shared")
require.True(t, ok)
require.Equal(t, "original", parentValue)
cancel()
require.ErrorIs(t, child.Context().Err(), context.Canceled)
require.NoError(t, parent.Context().Err())
}
// TestUseNetworkPort is the behavior matrix for UseNetworkPort.
//
// The contract (see issue #7323):
@@ -85,9 +85,6 @@ func (c *Config) applyDefaults() {
type hostState struct {
delay time.Duration
forbidden int
// reachable is set once the host has answered. A dial that fails before
// then is a closed or filtered port, which is no reason to slow down.
reachable bool
// next is the earliest start time still free. Concurrent waits reserve
// successive slots so a delay paces the host instead of holding a burst
// and releasing it together.
@@ -168,11 +165,11 @@ func (g *Governor) Observe(host string, statusCode int, retryAfter time.Duration
g.mu.Lock()
defer g.mu.Unlock()
state := g.state(host)
defer func() { _ = g.hosts.Set(host, state) }()
if statusCode > 0 {
state.reachable = true
state, lookupErr := g.hosts.GetIFPresent(host)
if lookupErr != nil || state == nil {
state = &hostState{}
}
defer func() { _ = g.hosts.Set(host, state) }()
if statusCode == http.StatusForbidden {
state.forbidden++
@@ -190,48 +187,15 @@ func (g *Governor) Observe(host string, statusCode int, retryAfter time.Duration
// recovered. A 404 or 500 is what most templates get back, and treating
// it as healthy would wipe the delay on the next probe.
if healthyStatus(statusCode) {
g.decay(state)
state.delay = time.Duration(float64(state.delay) * g.cfg.Decay)
if state.delay < g.cfg.Step {
state.delay = 0
state.next = time.Time{}
}
}
return
}
g.grow(state, retryAfter)
}
// ObserveDial feeds one connection attempt back to the governor, for
// protocols that have no response status to judge. A failed dial slows a host
// that accepted connections before, and a successful one counts as recovery.
// Errors on an established connection are the template's business, not the
// host asking for less.
func (g *Governor) ObserveDial(host string, err error) {
if g == nil || host == "" {
return
}
g.mu.Lock()
defer g.mu.Unlock()
state := g.state(host)
switch {
case err == nil:
state.reachable = true
g.decay(state)
case state.reachable:
g.grow(state, 0)
default:
// A port that never accepted is closed or filtered. Storing it would
// fill the cache and evict hosts that actually have a delay.
return
}
_ = g.hosts.Set(host, state)
}
func (g *Governor) state(host string) *hostState {
if state, err := g.hosts.GetIFPresent(host); err == nil && state != nil {
return state
}
return &hostState{}
}
func (g *Governor) grow(state *hostState, retryAfter time.Duration) {
next := state.delay
if next <= 0 {
next = g.cfg.Step
@@ -249,14 +213,6 @@ func (g *Governor) grow(state *hostState, retryAfter time.Duration) {
state.delay = next
}
func (g *Governor) decay(state *hostState) {
state.delay = time.Duration(float64(state.delay) * g.cfg.Decay)
if state.delay < g.cfg.Step {
state.delay = 0
state.next = time.Time{}
}
}
// isBlocking reports whether a response means the host wants less traffic.
func (g *Governor) isBlocking(statusCode int, err error, state *hostState) bool {
if err != nil {
@@ -210,39 +210,3 @@ func TestForbiddenStreakSurvivesAMiss(t *testing.T) {
g.Observe("acme.test", http.StatusForbidden, 0, nil)
require.Equal(t, 100*time.Millisecond, g.Delay("acme.test"))
}
func TestFailedDialBacksOffAndSuccessfulDialDecays(t *testing.T) {
g := New(Config{Step: 100 * time.Millisecond, Factor: 2, Decay: 0.5})
g.ObserveDial("acme.test:22", nil)
g.ObserveDial("acme.test:22", errors.New("connection refused"))
g.ObserveDial("acme.test:22", errors.New("connection refused"))
require.Equal(t, 200*time.Millisecond, g.Delay("acme.test:22"))
g.ObserveDial("acme.test:22", nil)
require.Equal(t, 100*time.Millisecond, g.Delay("acme.test:22"))
g.ObserveDial("acme.test:22", nil)
require.Zero(t, g.Delay("acme.test:22"))
}
// A port that never accepted is closed or filtered, not pushing back. Slowing
// it would cost every template aimed at it a growing wait for nothing.
func TestDialFailureOnNeverReachedHostIsIgnored(t *testing.T) {
g := New(Config{Step: 100 * time.Millisecond})
for i := 0; i < 3; i++ {
g.ObserveDial("acme.test:22", errors.New("connection refused"))
}
require.Zero(t, g.Delay("acme.test:22"))
_, err := g.hosts.GetIFPresent("acme.test:22")
require.Error(t, err, "a port that never accepted must not take a cache slot")
g.ObserveDial("ok.test:22", nil)
_, err = g.hosts.GetIFPresent("ok.test:22")
require.NoError(t, err, "a port that accepted is tracked so a later refusal can slow it")
}
func TestResponseMarksHostReachableForDials(t *testing.T) {
g := New(Config{Step: 100 * time.Millisecond})
g.Observe("acme.test:443", http.StatusNotFound, 0, nil)
g.ObserveDial("acme.test:443", errors.New("connection refused"))
require.Equal(t, 100*time.Millisecond, g.Delay("acme.test:443"))
}
+6 -7
View File
@@ -89,15 +89,13 @@ func newHttpClient(options *types.Options) (*http.Client, error) {
// causing "malformed HTTP response" errors. We use http2.Transport directly
// with utls dialing to properly handle h2 connections (see #6360).
if options.TlsImpersonate {
// utls conns are not *tls.Conn, so http.Transport speaks HTTP/1 on an h2 session.
// http2.Transport is deprecated in x/net 0.59, and it is still the dial that keeps h2. See #6360.
roundTripper = &http2.Transport{ //nolint:staticcheck // SA1019
DialTLSContext: func(ctx context.Context, network, addr string, _ *tls.Config) (net.Conn, error) { //nolint:staticcheck // SA1019
roundTripper = &http2.Transport{
DialTLSContext: func(ctx context.Context, network, addr string, _ *tls.Config) (net.Conn, error) {
return dialers.Fastdialer.DialTLSWithConfigImpersonate(ctx, network, addr, tlsConfig, impersonate.Random, nil)
},
TLSClientConfig: tlsConfig, //nolint:staticcheck // SA1019
AllowHTTP: false, //nolint:staticcheck // SA1019
DisableCompression: false, //nolint:staticcheck // SA1019
TLSClientConfig: tlsConfig,
AllowHTTP: false,
DisableCompression: false,
}
}
@@ -115,3 +113,4 @@ func newHttpClient(options *types.Options) (*http.Client, error) {
return httpclient, nil
}
+4 -3
View File
@@ -52,9 +52,10 @@ type HistoryData struct {
// Options contains additional configuration options for the browser instance
type Options struct {
Timeout time.Duration
DisableCookie bool
Options *types.Options
Timeout time.Duration
DisableCookie bool
Options *types.Options
OnActionTiming func(ActionType, time.Duration)
}
// Run runs a list of actions by creating a new page in the browser.
+28 -11
View File
@@ -52,6 +52,7 @@ func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action) (ou
type deferredWaitFunc struct {
actionIndex int
actionType ActionType
run func() error
}
@@ -76,6 +77,7 @@ func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action) (ou
actionIndex := len(p.ActionDurations)
timeStart := time.Now()
var deferredWaitDuration time.Duration
deferredWaitFailed := false
switch act.ActionType.ActionType {
case ActionNavigate:
err = p.NavigateURL(act, outData)
@@ -84,18 +86,24 @@ func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action) (ou
for _, waitFunc := range waitFuncs {
if waitFunc.run != nil {
waitStart := time.Now()
if err := waitFunc.run(); err != nil {
waitDuration := time.Since(waitStart)
addActionDuration(p.ActionDurations, waitFunc.actionIndex, waitDuration)
return nil, errkit.Wrap(err, "error occurred while executing waitFunc")
}
waitErr := waitFunc.run()
waitDuration := time.Since(waitStart)
addActionDuration(p.ActionDurations, waitFunc.actionIndex, waitDuration)
if p.options.OnActionTiming != nil {
p.options.OnActionTiming(waitFunc.actionType, waitDuration)
}
deferredWaitDuration += waitDuration
if waitErr != nil {
err = waitErr
deferredWaitFailed = true
break
}
}
}
p.lastActionNavigate = act
if err == nil {
p.lastActionNavigate = act
}
}
case ActionScript:
err = p.RunScript(act, outData)
@@ -141,7 +149,7 @@ func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action) (ou
var waitFunc func() error
waitFunc, err = p.WaitEvent(act, outData)
if waitFunc != nil {
waitFuncs = append(waitFuncs, deferredWaitFunc{actionIndex: actionIndex, run: waitFunc})
waitFuncs = append(waitFuncs, deferredWaitFunc{actionIndex: actionIndex, actionType: act.ActionType.ActionType, run: waitFunc})
}
case ActionWaitDialog:
err = p.HandleDialog(act, outData)
@@ -177,11 +185,20 @@ func (p *Page) ExecuteActions(input *contextargs.Context, actions []*Action) (ou
default:
executed = false
}
if err != nil {
return nil, errors.Wrap(err, "error occurred executing action")
}
if executed {
p.ActionDurations = append(p.ActionDurations, max(time.Since(timeStart)-deferredWaitDuration, 0))
duration := max(time.Since(timeStart)-deferredWaitDuration, 0)
if p.options.OnActionTiming != nil {
p.options.OnActionTiming(act.ActionType.ActionType, duration)
}
if err == nil {
p.ActionDurations = append(p.ActionDurations, duration)
}
}
if err != nil {
if deferredWaitFailed {
return nil, errkit.Wrap(err, "error occurred while executing waitFunc")
}
return nil, errors.Wrap(err, "error occurred executing action")
}
}
return outData, nil
@@ -902,6 +902,25 @@ func TestActionWaitEventDuration(t *testing.T) {
})
}
func TestActionNavigateDurationIncludesResponseHeaderWait(t *testing.T) {
actions := []*Action{
{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}},
{ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}},
}
handler := func(w http.ResponseWriter, _ *http.Request) {
time.Sleep(350 * time.Millisecond)
_, _ = fmt.Fprintln(w, `<html><body>loaded</body></html>`)
}
testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, _ ActionData) {
require.NoError(t, err)
require.Len(t, page.ActionDurations, 2)
require.GreaterOrEqual(t, page.ActionDurations[0], 300*time.Millisecond)
require.Less(t, page.ActionDurations[1], time.Second)
})
}
func TestActionWaitEventSeesEventFiredBeforeWaiting(t *testing.T) {
actions := []*Action{
{ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}},
@@ -1134,10 +1153,21 @@ func TestHeadlessRunHonorsParentCancellation(t *testing.T) {
}}
startedAt := time.Now()
_, page, err := instance.Run(input, actions, nil, &Options{Timeout: 5 * time.Second, Options: opts})
var timedAction ActionType
var timedDuration time.Duration
_, page, err := instance.Run(input, actions, nil, &Options{
Timeout: 5 * time.Second,
Options: opts,
OnActionTiming: func(action ActionType, duration time.Duration) {
timedAction = action
timedDuration = duration
},
})
require.ErrorIs(t, err, context.Canceled)
require.Nil(t, page)
require.Equal(t, ActionNavigate, timedAction)
require.Positive(t, timedDuration)
require.Less(t, time.Since(startedAt), 2*time.Second)
}
+64 -7
View File
@@ -79,6 +79,10 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
if len(request.Fuzzing) > 0 {
return request.executeFuzzingRule(input, vars, previous, wrappedCallback)
}
timing := newHeadlessTimingTrace(request.options.TemplateID, input.MetaInput.Input)
if timing != nil {
defer timing.emit(request.options.Logger)
}
if request.generator != nil {
iterator := request.generator.NewIterator()
for {
@@ -103,13 +107,13 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
return errors.Wrap(err, "could not evaluate payload helper expressions")
}
if err := request.executeRequestWithPayloads(input, renderedValue.Values, previous, renderedValue.InteractURLs, wrappedCallback); err != nil {
if err := request.executeRequestWithPayloads(input, renderedValue.Values, previous, renderedValue.InteractURLs, wrappedCallback, timing); err != nil {
return err
}
}
} else {
value := maps.Clone(vars)
if err := request.executeRequestWithPayloads(input, value, previous, interactshURLs, wrappedCallback); err != nil {
if err := request.executeRequestWithPayloads(input, value, previous, interactshURLs, wrappedCallback, timing); err != nil {
return err
}
}
@@ -132,9 +136,15 @@ func extractBaseURLFromActions(steps []*engine.Action) (string, error) {
return "", errors.New("no navigation action found")
}
func (request *Request) executeRequestWithPayloads(input *contextargs.Context, payloads map[string]interface{}, previous output.InternalEvent, interactshURLs []string, callback protocols.OutputEventCallback) error {
func (request *Request) executeRequestWithPayloads(input *contextargs.Context, payloads map[string]interface{}, previous output.InternalEvent, interactshURLs []string, callback protocols.OutputEventCallback, timing *headlessTimingTrace) error {
if timing != nil {
timing.attempts++
}
pageTimeout := time.Duration(request.options.Options.PageTimeout) * time.Second
if _, err := url.Parse(input.MetaInput.Input); err != nil {
if timing != nil {
timing.errors++
}
request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), err)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, errCouldNotGetHtmlElement)
@@ -144,35 +154,82 @@ func (request *Request) executeRequestWithPayloads(input *contextargs.Context, p
DisableCookie: request.DisableCookie,
Options: request.options.Options,
}
var actionTimings []headlessActionTiming
if timing != nil {
options.OnActionTiming = func(action engine.ActionType, duration time.Duration) {
actionTimings = append(actionTimings, headlessActionTiming{action: action, duration: duration})
}
}
if !options.DisableCookie && input.CookieJar == nil {
if timing != nil {
timing.errors++
}
return errors.New("cookie reuse enabled but cookie-jar is nil")
}
// limited as one request per run of the steps, the unit progress counts
if err := request.options.RateLimitTakeContext(input.Context(), input.MetaInput.Input); err != nil {
request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), err)
var rateWaitStarted time.Time
if timing != nil {
rateWaitStarted = time.Now()
}
rateWaitErr := request.options.RateLimitTakeContext(input.Context(), input.MetaInput.Input)
if timing != nil {
timing.rateWait += time.Since(rateWaitStarted)
}
if rateWaitErr != nil {
if timing != nil {
timing.errors++
}
request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), rateWaitErr)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could not take rate limit")
return errors.Wrap(rateWaitErr, "could not take rate limit")
}
var createStarted time.Time
if timing != nil {
createStarted = time.Now()
}
instance, err := request.options.Browser.NewInstanceWithContext(input.Context(), pageTimeout)
if timing != nil {
timing.contextCreate += time.Since(createStarted)
}
if err != nil {
if timing != nil {
timing.errors++
}
request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), err)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, errCouldNotGetHtmlElement)
}
var page *engine.Page
defer func() {
var cleanupStarted time.Time
if timing != nil {
cleanupStarted = time.Now()
}
if closeErr := instance.CloseWithPage(page); closeErr != nil {
gologger.Warning().Msgf("Could not close headless browser instance: %s", closeErr)
}
if timing != nil {
timing.cleanup += time.Since(cleanupStarted)
}
}()
instance.SetInteractsh(request.options.Interactsh)
timeStart := time.Now()
out, createdPage, err := instance.Run(input, request.Steps, payloads, options)
runDuration := time.Since(timeStart)
if timing != nil {
if err != nil {
timing.errors++
timing.recordFailedRun(actionTimings, runDuration)
} else if createdPage != nil {
timing.recordRun(request.Steps, createdPage.ActionDurations, runDuration)
} else {
timing.runOther += runDuration
}
}
if err != nil {
request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, request.Type().String(), err)
request.options.Progress.IncrementFailedRequestsBy(1)
@@ -294,7 +351,7 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, payloads
newInput := input.Clone()
newInput.MetaInput.Input = gr.Request.String()
if err := request.executeRequestWithPayloads(newInput, gr.DynamicValues, previous, gr.InteractURLs, callback); err != nil {
if err := request.executeRequestWithPayloads(newInput, gr.DynamicValues, previous, gr.InteractURLs, callback, nil); err != nil {
return false
}
+141
View File
@@ -0,0 +1,141 @@
package headless
import (
"crypto/sha256"
"encoding/binary"
"fmt"
"os"
"strings"
"time"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
)
const (
headlessTimingTemplatesEnv = "NUCLEI_HEADLESS_TIMING_TEMPLATES"
headlessTimingSampleRate = 50
headlessTimingSampleMin = 30 * time.Second
headlessTimingOutlierMin = 5 * time.Minute
)
type headlessTimingTrace struct {
templateID string
targetFingerprint string
sampled bool
started time.Time
attempts int
errors int
rateWait time.Duration
contextCreate time.Duration
navigate time.Duration
waitLoad time.Duration
waitDialog time.Duration
otherActions time.Duration
runOther time.Duration
cleanup time.Duration
}
type headlessActionTiming struct {
action engine.ActionType
duration time.Duration
}
func newHeadlessTimingTrace(templateID, target string) *headlessTimingTrace {
selectedTemplates := os.Getenv(headlessTimingTemplatesEnv)
if selectedTemplates == "" {
return nil
}
selected := false
for _, candidate := range strings.Split(selectedTemplates, ",") {
if strings.TrimSpace(candidate) == templateID {
selected = true
break
}
}
if !selected {
return nil
}
fingerprint := sha256.Sum256([]byte(target))
return &headlessTimingTrace{
templateID: templateID,
targetFingerprint: fmt.Sprintf("%x", fingerprint[:8]),
sampled: binary.BigEndian.Uint64(fingerprint[:8])%headlessTimingSampleRate == 0,
started: time.Now(),
}
}
func (t *headlessTimingTrace) recordRun(steps []*engine.Action, durations []time.Duration, elapsed time.Duration) {
actions := time.Duration(0)
if len(steps) != len(durations) {
for _, duration := range durations {
actions += duration
}
t.otherActions += actions
if elapsed > actions {
t.runOther += elapsed - actions
}
return
}
for index, duration := range durations {
actions += duration
if steps[index] == nil {
t.otherActions += duration
continue
}
switch steps[index].ActionType.ActionType {
case engine.ActionNavigate:
t.navigate += duration
case engine.ActionWaitLoad:
t.waitLoad += duration
case engine.ActionWaitDialog:
t.waitDialog += duration
default:
t.otherActions += duration
}
}
if elapsed > actions {
t.runOther += elapsed - actions
}
}
func (t *headlessTimingTrace) recordFailedRun(actions []headlessActionTiming, elapsed time.Duration) {
measured := time.Duration(0)
for _, action := range actions {
measured += action.duration
switch action.action {
case engine.ActionNavigate:
t.navigate += action.duration
case engine.ActionWaitLoad:
t.waitLoad += action.duration
case engine.ActionWaitDialog:
t.waitDialog += action.duration
default:
t.otherActions += action.duration
}
}
t.runOther += max(elapsed-measured, 0)
}
func (t *headlessTimingTrace) summary(elapsed time.Duration) (string, bool) {
if elapsed < headlessTimingOutlierMin && (!t.sampled || elapsed < headlessTimingSampleMin) {
return "", false
}
measured := t.rateWait + t.contextCreate + t.navigate + t.waitLoad + t.waitDialog + t.otherActions + t.runOther + t.cleanup
unattributed := max(elapsed-measured, 0)
return fmt.Sprintf(
"slow headless timing template_id=%q target_fingerprint=%s attempts=%d errors=%d total_ms=%d rate_wait_ms=%d context_create_ms=%d navigate_ms=%d waitload_ms=%d waitdialog_ms=%d other_actions_ms=%d run_other_ms=%d cleanup_ms=%d unattributed_ms=%d",
t.templateID, t.targetFingerprint, t.attempts, t.errors, elapsed.Milliseconds(), t.rateWait.Milliseconds(),
t.contextCreate.Milliseconds(), t.navigate.Milliseconds(), t.waitLoad.Milliseconds(), t.waitDialog.Milliseconds(),
t.otherActions.Milliseconds(), t.runOther.Milliseconds(), t.cleanup.Milliseconds(), unattributed.Milliseconds(),
), true
}
func (t *headlessTimingTrace) emit(logger *gologger.Logger) {
if message, ok := t.summary(time.Since(t.started)); ok {
if logger == nil {
logger = gologger.DefaultLogger
}
logger.Info().Msg(message)
}
}
+127
View File
@@ -0,0 +1,127 @@
package headless
import (
"bytes"
"strings"
"testing"
"time"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/gologger/formatter"
"github.com/projectdiscovery/gologger/levels"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
"github.com/stretchr/testify/require"
)
type timingCaptureWriter struct{ buffer *bytes.Buffer }
func (w timingCaptureWriter) Write(data []byte, _ levels.Level) {
w.buffer.Write(data)
}
func TestHeadlessTimingTraceUsesExecutionLogger(t *testing.T) {
previous := gologger.DefaultLogger
t.Cleanup(func() { gologger.DefaultLogger = previous })
var output bytes.Buffer
silentLogger := &gologger.Logger{}
silentLogger.SetFormatter(formatter.NewCLI(false))
silentLogger.SetWriter(timingCaptureWriter{buffer: &output})
silentLogger.SetMaxLevel(levels.LevelSilent)
gologger.DefaultLogger = silentLogger
logger := &gologger.Logger{}
logger.SetFormatter(formatter.NewCLI(false))
logger.SetWriter(timingCaptureWriter{buffer: &output})
logger.SetMaxLevel(levels.LevelInfo)
trace := &headlessTimingTrace{templateID: "headless-open-redirect", sampled: true, started: time.Now().Add(-31 * time.Second)}
trace.emit(silentLogger)
trace.emit(nil)
require.Empty(t, output.String())
trace.emit(logger)
require.Contains(t, output.String(), "slow headless timing template_id=\"headless-open-redirect\"")
}
func TestHeadlessTimingTraceIsTemplateOptIn(t *testing.T) {
t.Setenv(headlessTimingTemplatesEnv, "")
require.Nil(t, newHeadlessTimingTrace("headless-open-redirect", "https://example.com"))
t.Setenv(headlessTimingTemplatesEnv, "CVE-2025-8191, headless-open-redirect")
require.Nil(t, newHeadlessTimingTrace("other-template", "https://example.com"))
trace := newHeadlessTimingTrace("headless-open-redirect", "https://example.com")
require.NotNil(t, trace)
require.Len(t, trace.targetFingerprint, 16)
require.NotContains(t, trace.targetFingerprint, "example.com")
}
func TestHeadlessTimingTraceAccountsForActionsAndOverhead(t *testing.T) {
trace := &headlessTimingTrace{
templateID: "headless-open-redirect",
targetFingerprint: "0123456789abcdef",
sampled: true,
attempts: 2,
errors: 1,
rateWait: time.Second,
contextCreate: time.Second,
cleanup: 250 * time.Millisecond,
}
steps := []*engine.Action{
{ActionType: engine.ActionTypeHolder{ActionType: engine.ActionNavigate}},
{ActionType: engine.ActionTypeHolder{ActionType: engine.ActionWaitLoad}},
{ActionType: engine.ActionTypeHolder{ActionType: engine.ActionWaitDialog}},
}
trace.recordRun(steps, []time.Duration{2 * time.Second, 500 * time.Millisecond, 3 * time.Second}, 6*time.Second)
message, ok := trace.summary(30 * time.Second)
require.True(t, ok)
for _, field := range []string{
"attempts=2", "errors=1", "total_ms=30000", "rate_wait_ms=1000",
"context_create_ms=1000", "navigate_ms=2000", "waitload_ms=500",
"waitdialog_ms=3000", "run_other_ms=500", "cleanup_ms=250", "unattributed_ms=21750",
} {
require.True(t, strings.Contains(message, field), "missing %s in %s", field, message)
}
require.NotContains(t, message, "https://")
}
func TestHeadlessTimingTraceBoundsOutput(t *testing.T) {
trace := &headlessTimingTrace{sampled: false}
_, ok := trace.summary(4 * time.Minute)
require.False(t, ok)
_, ok = trace.summary(5 * time.Minute)
require.True(t, ok)
trace.sampled = true
_, ok = trace.summary(29 * time.Second)
require.False(t, ok)
_, ok = trace.summary(30 * time.Second)
require.True(t, ok)
}
func TestHeadlessTimingTraceDoesNotMisattributePartialActionDurations(t *testing.T) {
trace := &headlessTimingTrace{sampled: true}
steps := []*engine.Action{
{ActionType: engine.ActionTypeHolder{ActionType: engine.ActionNavigate}},
{ActionType: engine.ActionTypeHolder{ActionType: engine.ActionWaitLoad}},
}
trace.recordRun(steps, []time.Duration{2 * time.Second}, 3*time.Second)
require.Zero(t, trace.navigate)
require.Zero(t, trace.waitLoad)
require.Equal(t, 2*time.Second, trace.otherActions)
require.Equal(t, time.Second, trace.runOther)
}
func TestHeadlessTimingTraceAttributesFailedActions(t *testing.T) {
trace := &headlessTimingTrace{sampled: true}
trace.recordFailedRun([]headlessActionTiming{
{action: engine.ActionNavigate, duration: 2 * time.Second},
{action: engine.ActionWaitLoad, duration: 500 * time.Millisecond},
}, 3*time.Second)
require.Equal(t, 2*time.Second, trace.navigate)
require.Equal(t, 500*time.Millisecond, trace.waitLoad)
require.Zero(t, trace.otherActions)
require.Equal(t, 500*time.Millisecond, trace.runOther)
}
+11 -22
View File
@@ -122,8 +122,7 @@ type Request struct {
// examples:
// - name: Send requests using 10 concurrent threads
// value: 10
Threads int `yaml:"threads,omitempty" json:"threads,omitempty" jsonschema:"title=threads for sending requests,description=Threads specifies number of threads to use sending requests. This enables Connection Pooling"`
PayloadThreads int `yaml:"payload-threads,omitempty" json:"payload-threads,omitempty" jsonschema:"title=dependent payload concurrency,description=Opt-in concurrency across independent payload groups while preserving request order inside each group"`
Threads int `yaml:"threads,omitempty" json:"threads,omitempty" jsonschema:"title=threads for sending requests,description=Threads specifies number of threads to use sending requests. This enables Connection Pooling"`
// description: |
// MaxSize is the maximum size of http response body to read in bytes.
// examples:
@@ -139,15 +138,14 @@ type Request struct {
CompiledOperators *operators.Operators `yaml:"-" json:"-"`
options *protocols.ExecutorOptions
connConfiguration *httpclientpool.Configuration
totalRequests int
customHeaders map[string]string
generator *generators.PayloadGenerator // optional, only enabled when using payloads
dependentPayloadThreads int
rawhttpClient *rawhttp.Client
replayClient *http.Client // set when -replay-proxy is given
dialer *fastdialer.Dialer
options *protocols.ExecutorOptions
connConfiguration *httpclientpool.Configuration
totalRequests int
customHeaders map[string]string
generator *generators.PayloadGenerator // optional, only enabled when using payloads
rawhttpClient *rawhttp.Client
replayClient *http.Client // set when -replay-proxy is given
dialer *fastdialer.Dialer
// description: |
// SelfContained specifies if the request is self-contained.
@@ -522,7 +520,6 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
delete(request.Payloads, payload)
}
request.dependentPayloadThreads = 0
if len(request.Payloads) > 0 {
request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Catalog, request.options.Options.AttackType, request.options.Options)
if err != nil {
@@ -564,20 +561,12 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
}
}
if hasMultipleRequests && (hasNamedInternalExtractor || request.NeedsRequestCondition()) {
if !hasNamedInternalExtractor && request.NeedsRequestCondition() && request.PayloadThreads > 0 && request.canRunDependentPayloadsConcurrently() {
request.dependentPayloadThreads = request.PayloadThreads
}
if request.dependentPayloadThreads == 0 {
stats.Increment(SetThreadToCountZero)
}
stats.Increment(SetThreadToCountZero)
request.Threads = 0
} else {
request.Threads = options.GetThreadsForNPayloadRequests(request.Requests(), request.Threads)
}
}
if request.PayloadThreads > 0 && request.dependentPayloadThreads == 0 {
return errors.New("payload-threads requires multiple raw requests, indexed response conditions, and independent payload groups without iterate-all, interactsh, or @once")
}
// Avoid reusing client-side HTTP proxy connections for the legacy non-threaded spray path.
if shouldDisableKeepAliveForHTTPProxy(request, options) {
@@ -643,7 +632,7 @@ func shouldDisableKeepAliveForHTTPProxy(request *Request, options *protocols.Exe
if request == nil || options == nil || options.Options == nil || options.Options.AliveHttpProxy == "" {
return false
}
return request.Threads <= 0 && request.dependentPayloadThreads <= 0 && options.Options.ScanStrategy != scanstrategy.HostSpray.String()
return request.Threads <= 0 && options.Options.ScanStrategy != scanstrategy.HostSpray.String()
}
// AnalyzeConnectionReuse determines if a request can safely reuse connections.
@@ -13,8 +13,6 @@ func TestShouldDisableKeepAliveForHTTPProxy(t *testing.T) {
opts.AliveHttpProxy = "http://127.0.0.1:8080"
opts.ScanStrategy = scanstrategy.TemplateSpray.String()
require.True(t, shouldDisableKeepAliveForHTTPProxy(&Request{}, &protocols.ExecutorOptions{Options: opts}))
require.False(t, shouldDisableKeepAliveForHTTPProxy(&Request{Threads: 10}, &protocols.ExecutorOptions{Options: opts}))
require.False(t, shouldDisableKeepAliveForHTTPProxy(&Request{dependentPayloadThreads: 2}, &protocols.ExecutorOptions{Options: opts}))
opts.ScanStrategy = scanstrategy.HostSpray.String()
require.False(t, shouldDisableKeepAliveForHTTPProxy(&Request{}, &protocols.ExecutorOptions{Options: opts}))
+14 -178
View File
@@ -19,7 +19,6 @@ import (
"github.com/pkg/errors"
"go.uber.org/multierr"
"golang.org/x/sync/semaphore"
"moul.io/http2curl"
"github.com/projectdiscovery/fastdialer/fastdialer"
@@ -117,10 +116,10 @@ func (request *Request) observeHostBackoff(host string, resp *http.Response, err
// waitHostBackoff pauses for the delay this host has already earned.
func (request *Request) waitHostBackoff(ctx context.Context, host string) error {
if request.options == nil {
if request.options == nil || request.options.HostBackoff == nil {
return nil
}
return request.options.HostBackoffWait(ctx, host)
return request.options.HostBackoff.Wait(ctx, hostratelimit.NormalizeHostPort(host))
}
type httpRateLimitWaitObserver interface {
@@ -662,62 +661,29 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
if request.Threads > 0 && (len(request.Payloads) > 0 || request.Race) {
return request.executeParallelHTTP(input, dynamicValues, callback)
}
if request.dependentPayloadThreads > 0 {
return request.executeDependentPayloads(input, dynamicValues, previous, callback)
}
return request.executeSequentialHTTP(input, dynamicValues, previous, callback, request.newGenerator(false), nil, nil)
}
func (request *Request) executeSequentialHTTP(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback, generator *requestGenerator, accounted *atomic.Int64, stopOnMatch *atomic.Bool) error {
var recordedCurrent bool
countAccounted := func(count int64) {
if accounted != nil {
accounted.Add(count)
}
}
recordSkipped := func(count int64) {
countAccounted(count)
request.options.Progress.SetRequests(uint64(count))
}
recordSent := func() {
recordedCurrent = true
countAccounted(1)
request.options.Progress.IncrementRequests()
}
recordFailed := func() {
recordedCurrent = true
countAccounted(1)
request.options.Progress.IncrementFailedRequestsBy(1)
}
recordStopped := func() {
remaining := int64(generator.Remaining())
if !recordedCurrent {
remaining++
}
recordSkipped(remaining)
}
generator := request.newGenerator(false)
initialPrevious := cloneInternalEvent(previous)
initialContext := dynamicValueContext{
values: generators.CopyMap(dynamicValues),
previous: cloneInternalEvent(initialPrevious),
previous: cloneInternalEvent(previous),
}
dynamicValueContexts := []dynamicValueContext{initialContext}
if request.IterateAll {
dynamicValueContexts = expandDynamicValueContexts(initialContext, generators.MergeMapsMany(dynamicValues), true)
}
oncePrevious := make(output.InternalEvent)
lastRequestIndex := 0
var requestErr error
executionLoop:
for {
executeFunc := func(data string, payloads, dynamicValue map[string]interface{}, previousEvent output.InternalEvent) (skipBranch, stop bool, extracted map[string][]string, err error) {
recordedCurrent = false
hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators)
ctx := request.newContext(input)
generatedHttpRequest, err := generator.Make(ctx, input, data, payloads, dynamicValue)
ctxWithTimeout, cancel := context.WithTimeoutCause(ctx, request.options.Options.GetTimeouts().HttpTimeout, ErrHttpEngineRequestDeadline)
defer cancel()
generatedHttpRequest, err := generator.Make(ctxWithTimeout, input, data, payloads, dynamicValue)
if err != nil {
if err == types.ErrNoMoreRequests {
return false, true, nil, nil
@@ -741,12 +707,6 @@ executionLoop:
if err := request.rateLimitTake(ctx, hostBackoffKey(generatedHttpRequest, input)); err != nil {
return false, true, nil, err
}
if generatedHttpRequest.request != nil && generatedHttpRequest.customCancelFunction == nil {
// Rate admission must not consume the network request's timeout.
timedCtx, cancel := context.WithTimeoutCause(generatedHttpRequest.request.Context(), request.options.Options.GetTimeouts().HttpTimeout, ErrHttpEngineRequestDeadline)
defer cancel()
generatedHttpRequest.request.WithContext(timedCtx)
}
hasInteractMarkers := interactsh.HasMarkers(data) || len(generatedHttpRequest.interactshURLs) > 0
if input.MetaInput.Input == "" {
@@ -796,9 +756,6 @@ executionLoop:
if errors.Is(execReqErr, ErrMissingVars) {
return true, false, nil, nil
}
if stopOnMatch != nil && stopOnMatch.Load() && errors.Is(execReqErr, context.Canceled) {
return false, true, nil, nil
}
request.recordHostResult(updatedInput, execReqErr)
if execReqErr != nil {
@@ -806,9 +763,9 @@ executionLoop:
reqKitErr.Msgf("got err while executing %v", generatedHttpRequest.URL())
requestErr = reqKitErr
recordFailed()
request.options.Progress.IncrementFailedRequestsBy(1)
} else {
recordSent()
request.options.Progress.IncrementRequests()
}
// If this was a match, and we want to stop at first match, skip all further requests.
@@ -823,13 +780,6 @@ executionLoop:
if !ok {
break
}
if generator.payloadIterator != nil && generator.currentIndex <= lastRequestIndex {
for i := range dynamicValueContexts {
dynamicValueContexts[i].previous = cloneInternalEvent(initialPrevious)
maps.Copy(dynamicValueContexts[i].previous, oncePrevious)
}
}
lastRequestIndex = generator.currentIndex
select {
case <-input.Context().Done():
@@ -857,11 +807,11 @@ executionLoop:
if requestErr == nil {
requestErr = gotErr
}
recordSkipped(int64(generator.Remaining() + 1))
request.options.Progress.SetRequests(uint64(generator.Remaining() + 1))
break executionLoop
}
if stop {
recordStopped()
request.options.Progress.SetRequests(uint64(generator.Remaining() + 1))
break executionLoop
}
if skipBranch {
@@ -876,7 +826,6 @@ executionLoop:
dynamicValueContexts = nil
continue
}
maps.Copy(oncePrevious, oncePreviousDelta)
nextDynamicValueContexts := make([]dynamicValueContext, 0, len(dynamicValueContexts))
for _, branch := range dynamicValueContexts {
@@ -900,11 +849,11 @@ executionLoop:
if requestErr == nil {
requestErr = gotErr
}
recordSkipped(int64(generator.Remaining() + 1))
request.options.Progress.SetRequests(uint64(generator.Remaining() + 1))
break executionLoop
}
if stop {
recordStopped()
request.options.Progress.SetRequests(uint64(generator.Remaining() + 1))
break executionLoop
}
if skipBranch {
@@ -917,119 +866,6 @@ executionLoop:
return requestErr
}
func (request *Request) canRunDependentPayloadsConcurrently() bool {
if request.IterateAll || request.Race || request.Pipeline || len(request.Fuzzing) > 0 || len(request.Raw) < 2 || interactsh.HasMatchers(request.CompiledOperators) {
return false
}
for _, raw := range request.Raw {
if interactsh.HasMarkers(raw) {
return false
}
if mark, hasMark := parseFlowAnnotations(raw); hasMark && mark == Once {
return false
}
}
return true
}
func (request *Request) executeDependentPayloads(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
workers := min(request.dependentPayloadThreads, defaultMaxWorkers)
if request.options.Options.PayloadConcurrency > 0 {
workers = min(workers, request.options.Options.PayloadConcurrency)
}
workerCtx, cancel := context.WithCancel(input.Context())
defer cancel()
permits := semaphore.NewWeighted(int64(workers))
shouldStop := request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch || request.options.StopAtFirstMatch
var found atomic.Bool
var callbackMu sync.Mutex
wrappedCallback := func(event *output.InternalWrappedEvent) {
callbackMu.Lock()
defer callbackMu.Unlock()
if shouldStop && found.Load() {
return
}
callback(event)
if shouldStop && event.OperatorsResult != nil && event.OperatorsResult.Matched {
found.Store(true)
cancel()
}
}
tasks := make(chan map[string]interface{}, workers)
var workersWG sync.WaitGroup
var errorsMu sync.Mutex
var executionErrors []error
var accounted atomic.Int64
for i := 0; i < workers; i++ {
workersWG.Add(1)
go func() {
defer workersWG.Done()
for payloads := range tasks {
if workerCtx.Err() != nil {
return
}
weight := int64(1)
if protocolstate.IsLowOnMemory() {
allowed := max(1, protocolstate.GuardThreadsOrDefault(workers))
weight = int64((workers + allowed - 1) / allowed)
}
if err := permits.Acquire(workerCtx, weight); err != nil {
return
}
generator := &requestGenerator{
request: request,
options: request.options,
currentPayloads: payloads,
onceFlow: make(map[string]struct{}),
}
groupInput := input.CloneWithContext(workerCtx)
err := request.executeSequentialHTTP(groupInput, dynamicValues, previous, wrappedCallback, generator, &accounted, &found)
permits.Release(weight)
if err != nil {
if found.Load() && errors.Is(err, context.Canceled) {
continue
}
errorsMu.Lock()
executionErrors = append(executionErrors, err)
errorsMu.Unlock()
}
}
}()
}
iterator := request.generator.NewIterator()
enqueue:
for {
payloads, ok := iterator.Value()
if !ok {
break
}
select {
case <-workerCtx.Done():
break enqueue
case tasks <- payloads:
}
}
close(tasks)
workersWG.Wait()
missing := int64(request.Requests()) - accounted.Load()
if missing < 0 {
return fmt.Errorf("dependent payload request accounting exceeded planned requests by %d", -missing)
}
if missing > 0 {
request.options.Progress.SetRequests(uint64(missing))
}
if found.Load() {
return nil
}
if err := input.Context().Err(); err != nil {
return err
}
return multierr.Combine(executionErrors...)
}
type dynamicValueContext struct {
values map[string]interface{}
seen map[string]map[string]struct{}
@@ -2,15 +2,7 @@ package http
import (
"context"
"crypto/md5"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/require"
@@ -21,595 +13,9 @@ import (
extractorpkg "github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors"
matcherpkg "github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/progress"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/projectdiscovery/utils/memguardian"
)
func TestIndexedResponsePayloadCannotUseIndependentRequestThreads(t *testing.T) {
options := testutils.DefaultOptions.Copy()
options.ExecutionId = t.Name()
options.RestrictLocalNetworkAccess = false
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
var requests atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
requests.Add(1)
if request.URL.Path == "/first" {
writer.WriteHeader(http.StatusCreated)
return
}
writer.WriteHeader(http.StatusOK)
}))
defer server.Close()
request := &Request{
ID: "indexed-response-payload-threading",
Raw: []string{
"GET /first?probe={{probe}} HTTP/1.1\nHost: {{Hostname}}",
"GET /second?probe={{probe}} HTTP/1.1\nHost: {{Hostname}}",
},
Threads: 10,
Payloads: map[string]interface{}{"probe": []string{"one", "two"}},
Operators: operatorsWithMatchers(&matcherpkg.Matcher{
Type: matcherpkg.MatcherTypeHolder{MatcherType: matcherpkg.DSLMatcher},
DSL: []string{"status_code_1 == 201 && status_code_2 == 200"},
}),
}
executerOptions := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: request.ID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
executerOptions.RateLimiter = nil
require.NoError(t, request.Compile(executerOptions))
require.Zero(t, request.Threads)
run := func() int {
matched := 0
err := request.ExecuteWithResults(
contextargs.NewWithInput(t.Context(), server.URL),
make(output.InternalEvent),
make(output.InternalEvent),
func(event *output.InternalWrappedEvent) {
if event.OperatorsResult != nil && event.OperatorsResult.Matched {
matched++
}
},
)
require.NoError(t, err)
return matched
}
require.Equal(t, 2, run())
require.Equal(t, int32(4), requests.Load())
request.Threads = 10
requests.Store(0)
require.Zero(t, run())
require.Equal(t, int32(4), requests.Load())
}
func TestDependentPayloadThreadsRejectSharedRequestState(t *testing.T) {
newRequest := func() *Request {
return &Request{Raw: []string{
"GET /first HTTP/1.1\nHost: {{Hostname}}",
"GET /second HTTP/1.1\nHost: {{Hostname}}",
}}
}
base := newRequest()
require.True(t, base.canRunDependentPayloadsConcurrently())
iterating := newRequest()
iterating.IterateAll = true
require.False(t, iterating.canRunDependentPayloadsConcurrently())
racing := newRequest()
racing.Race = true
require.False(t, racing.canRunDependentPayloadsConcurrently())
pipelined := newRequest()
pipelined.Pipeline = true
require.False(t, pipelined.canRunDependentPayloadsConcurrently())
once := newRequest()
once.Raw[0] = "@once\n" + once.Raw[0]
require.False(t, once.canRunDependentPayloadsConcurrently())
}
func TestIndexedResponsePayloadThreadsKeepPairsOrdered(t *testing.T) {
options := testutils.DefaultOptions.Copy()
options.ExecutionId = t.Name()
options.RestrictLocalNetworkAccess = false
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
require.NotNil(t, memguardian.DefaultMemGuardian)
previousWarning := memguardian.DefaultMemGuardian.Warning.Swap(false)
t.Cleanup(func() { memguardian.DefaultMemGuardian.Warning.Store(previousWarning) })
var firstStarted atomic.Int32
var firstActive atomic.Int32
var maximumFirstActive atomic.Int32
var outOfOrder atomic.Int32
firstGate := make(chan struct{})
firstDone := make(map[string]bool)
var firstDoneMu sync.Mutex
negativeProbe := fmt.Sprintf("%x", md5.Sum([]byte("two")))
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
probe := request.URL.Query().Get("probe")
switch request.URL.Path {
case "/first":
active := firstActive.Add(1)
for old := maximumFirstActive.Load(); active > old && !maximumFirstActive.CompareAndSwap(old, active); old = maximumFirstActive.Load() {
}
if firstStarted.Add(1) == 2 {
close(firstGate)
}
select {
case <-firstGate:
case <-time.After(250 * time.Millisecond):
}
firstDoneMu.Lock()
firstDone[probe] = true
firstDoneMu.Unlock()
firstActive.Add(-1)
writer.WriteHeader(http.StatusCreated)
case "/second":
firstDoneMu.Lock()
if !firstDone[probe] {
outOfOrder.Add(1)
}
firstDoneMu.Unlock()
if probe == negativeProbe {
writer.WriteHeader(http.StatusNotFound)
return
}
writer.WriteHeader(http.StatusOK)
}
}))
defer server.Close()
request := &Request{
ID: "indexed-response-payload-pairs",
Raw: []string{
"GET /first?probe={{md5(probe)}} HTTP/1.1\nHost: {{Hostname}}",
"GET /second?probe={{md5(probe)}} HTTP/1.1\nHost: {{Hostname}}",
},
PayloadThreads: 2,
Payloads: map[string]interface{}{"probe": []string{"one", "two"}},
Operators: operatorsWithMatchers(&matcherpkg.Matcher{
Type: matcherpkg.MatcherTypeHolder{MatcherType: matcherpkg.DSLMatcher},
DSL: []string{"status_code_1 == 201 && status_code_2 == 200"},
}),
}
executerOptions := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: request.ID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
executerOptions.RateLimiter = nil
require.NoError(t, request.Compile(executerOptions))
var matches atomic.Int32
require.NoError(t, request.ExecuteWithResults(
contextargs.NewWithInput(t.Context(), server.URL),
make(output.InternalEvent),
make(output.InternalEvent),
func(event *output.InternalWrappedEvent) {
if event.OperatorsResult != nil && event.OperatorsResult.Matched {
matches.Add(1)
}
},
))
require.Equal(t, int32(1), matches.Load())
require.Zero(t, outOfOrder.Load())
require.Equal(t, int32(2), maximumFirstActive.Load())
require.Len(t, firstDone, 2)
}
func TestDependentPayloadRateLimitWaitDoesNotExpireHTTPTimeout(t *testing.T) {
options := testutils.DefaultOptions.Copy()
options.ExecutionId = t.Name()
options.RestrictLocalNetworkAccess = false
options.PerHostRateLimit = true
options.RateLimit = 1
options.RateLimitDuration = 80 * time.Millisecond
options.SetTimeouts(&types.Timeouts{HttpTimeout: 50 * time.Millisecond})
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
var sent atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) {
sent.Add(1)
writer.WriteHeader(http.StatusOK)
}))
defer server.Close()
request := &Request{
ID: "dependent-payload-rate-wait",
Raw: []string{
"GET /first?probe={{probe}} HTTP/1.1\nHost: {{Hostname}}",
"GET /second?probe={{probe}} HTTP/1.1\nHost: {{Hostname}}",
},
PayloadThreads: 2,
Payloads: map[string]interface{}{
"probe": []string{"one", "two"},
},
Operators: operatorsWithMatchers(&matcherpkg.Matcher{
Type: matcherpkg.MatcherTypeHolder{MatcherType: matcherpkg.DSLMatcher},
DSL: []string{"status_code_1 == 200 && status_code_2 == 200"},
}),
}
executerOptions := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: request.ID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
executerOptions.RateLimiter = nil
require.NoError(t, request.Compile(executerOptions))
var matches atomic.Int32
require.NoError(t, request.ExecuteWithResults(
contextargs.NewWithInput(t.Context(), server.URL),
make(output.InternalEvent),
make(output.InternalEvent),
func(event *output.InternalWrappedEvent) {
if event.OperatorsResult != nil && event.OperatorsResult.Matched {
matches.Add(1)
}
},
))
require.Equal(t, int32(4), sent.Load())
require.Equal(t, int32(2), matches.Load())
}
func TestDependentPayloadRateLimitedLastMatchAccounting(t *testing.T) {
options := testutils.DefaultOptions.Copy()
options.ExecutionId = t.Name()
options.RestrictLocalNetworkAccess = false
options.PerHostRateLimit = true
options.RateLimit = 1
options.RateLimitDuration = 20 * time.Millisecond
options.SetTimeouts(&types.Timeouts{HttpTimeout: 15 * time.Millisecond})
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
var sent atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
sent.Add(1)
if request.URL.Path == "/first" {
writer.WriteHeader(http.StatusCreated)
return
}
if request.URL.Query().Get("probe") == "key-50" {
writer.WriteHeader(http.StatusOK)
return
}
writer.WriteHeader(http.StatusNotFound)
}))
defer server.Close()
keys := make([]string, 51)
for i := range keys {
keys[i] = fmt.Sprintf("key-%d", i)
}
request := &Request{
ID: "dependent-payload-last-match",
Raw: []string{
"GET /first?probe={{probe}} HTTP/1.1\nHost: {{Hostname}}",
"GET /second?probe={{probe}} HTTP/1.1\nHost: {{Hostname}}",
},
PayloadThreads: 10,
StopAtFirstMatch: true,
Payloads: map[string]interface{}{"probe": keys},
Operators: operatorsWithMatchers(&matcherpkg.Matcher{
Type: matcherpkg.MatcherTypeHolder{MatcherType: matcherpkg.DSLMatcher},
DSL: []string{"status_code_1 == 201 && status_code_2 == 200"},
}),
}
executerOptions := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: request.ID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
executerOptions.RateLimiter = nil
progressCounter := &payloadRequestProgress{Progress: executerOptions.Progress}
executerOptions.Progress = progressCounter
require.NoError(t, request.Compile(executerOptions))
var matches atomic.Int32
require.NoError(t, request.ExecuteWithResults(
contextargs.NewWithInput(t.Context(), server.URL),
make(output.InternalEvent),
make(output.InternalEvent),
func(event *output.InternalWrappedEvent) {
if event.OperatorsResult != nil && event.OperatorsResult.Matched {
matches.Add(1)
}
},
))
require.Equal(t, int32(1), matches.Load())
require.GreaterOrEqual(t, sent.Load(), int32(2))
require.LessOrEqual(t, sent.Load(), int32(102))
require.Equal(t, int64(sent.Load()), progressCounter.sent.Load())
require.Equal(t, int64(102), progressCounter.sent.Load()+progressCounter.skipped.Load()+progressCounter.failed.Load())
require.Zero(t, progressCounter.failed.Load())
}
func TestIndexedResponsePayloadThreadsStopAfterFirstMatch(t *testing.T) {
options := testutils.DefaultOptions.Copy()
options.ExecutionId = t.Name()
options.RestrictLocalNetworkAccess = false
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
require.NotNil(t, memguardian.DefaultMemGuardian)
previousWarning := memguardian.DefaultMemGuardian.Warning.Swap(false)
t.Cleanup(func() { memguardian.DefaultMemGuardian.Warning.Store(previousWarning) })
var requests atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
requests.Add(1)
if request.URL.Path == "/first" {
writer.WriteHeader(http.StatusCreated)
return
}
if request.URL.Query().Get("probe") == "match" {
writer.WriteHeader(http.StatusOK)
return
}
writer.WriteHeader(http.StatusNotFound)
}))
defer server.Close()
request := &Request{
ID: "indexed-response-stop-first-match",
Raw: []string{
"GET /first?probe={{probe}} HTTP/1.1\nHost: {{Hostname}}",
"GET /second?probe={{probe}} HTTP/1.1\nHost: {{Hostname}}",
},
PayloadThreads: 2,
StopAtFirstMatch: true,
Payloads: map[string]interface{}{
"probe": []string{"match", "miss-a", "miss-b", "miss-c"},
},
Operators: operatorsWithMatchers(&matcherpkg.Matcher{
Type: matcherpkg.MatcherTypeHolder{MatcherType: matcherpkg.DSLMatcher},
DSL: []string{"status_code_1 == 201 && status_code_2 == 200"},
}),
}
executerOptions := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: request.ID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
executerOptions.RateLimiter = nil
progressCounter := &payloadRequestProgress{Progress: executerOptions.Progress}
executerOptions.Progress = progressCounter
require.NoError(t, request.Compile(executerOptions))
var matches atomic.Int32
require.NoError(t, request.ExecuteWithResults(
contextargs.NewWithInput(t.Context(), server.URL),
make(output.InternalEvent),
make(output.InternalEvent),
func(event *output.InternalWrappedEvent) {
if event.OperatorsResult != nil && event.OperatorsResult.Matched {
matches.Add(1)
}
},
))
require.Equal(t, int32(1), matches.Load())
require.Less(t, requests.Load(), int32(8))
require.Equal(t, int64(request.Requests()), progressCounter.sent.Load()+progressCounter.skipped.Load()+progressCounter.failed.Load())
require.Zero(t, progressCounter.failed.Load())
}
func TestIndexedResponsePayloadThreadsCoverAllFiveHundredResponses(t *testing.T) {
options := testutils.DefaultOptions.Copy()
options.ExecutionId = t.Name()
options.RestrictLocalNetworkAccess = false
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
require.NotNil(t, memguardian.DefaultMemGuardian)
previousWarning := memguardian.DefaultMemGuardian.Warning.Swap(false)
t.Cleanup(func() { memguardian.DefaultMemGuardian.Warning.Store(previousWarning) })
var active atomic.Int32
var maximum atomic.Int32
var served atomic.Int32
var keysMu sync.Mutex
keys := make(map[string]int)
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
current := active.Add(1)
for old := maximum.Load(); current > old && !maximum.CompareAndSwap(old, current); old = maximum.Load() {
}
key := strings.SplitN(request.URL.RawQuery, "=", 2)[0]
keysMu.Lock()
keys[key]++
keysMu.Unlock()
time.Sleep(5 * time.Millisecond)
active.Add(-1)
served.Add(1)
writer.WriteHeader(http.StatusInternalServerError)
}))
defer server.Close()
buckets := make([]string, 17)
for i := range buckets {
buckets[i] = fmt.Sprintf("bucket-%d", i)
}
for _, tc := range []struct {
name string
payloadThreads int
minimumActive int32
maximumActive int32
lowMemory bool
}{
{name: "serial", minimumActive: 1, maximumActive: 1},
{name: "guarded", payloadThreads: 10, minimumActive: 1, maximumActive: 2, lowMemory: true},
{name: "grouped", payloadThreads: 10, minimumActive: 5},
} {
t.Run(tc.name, func(t *testing.T) {
memguardian.DefaultMemGuardian.Warning.Store(tc.lowMemory)
maximum.Store(0)
served.Store(0)
keysMu.Lock()
clear(keys)
keysMu.Unlock()
request := &Request{
ID: "indexed-response-5xx-coverage-" + tc.name,
Raw: []string{
"GET /first?{{rand}}-{{md5(escape)}}-{{bucket}}=x HTTP/1.1\nHost: {{Hostname}}",
"GET /second?{{rand}}-{{md5(escape)}}-{{bucket}}=x HTTP/1.1\nHost: {{Hostname}}",
},
PayloadThreads: tc.payloadThreads,
AttackType: generators.AttackTypeHolder{Value: generators.ClusterBombAttack},
Payloads: map[string]interface{}{
"escape": []string{"\v", "\f", "\x1c", "\x1d", "\x1e", "\x1f"},
"bucket": buckets,
},
Operators: operatorsWithMatchers(&matcherpkg.Matcher{
Type: matcherpkg.MatcherTypeHolder{MatcherType: matcherpkg.DSLMatcher},
DSL: []string{"status_code_2 == 200"},
}),
StopAtFirstMatch: true,
}
executerOptions := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: request.ID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
executerOptions.RateLimiter = nil
require.NoError(t, executerOptions.Variables.UnmarshalJSON([]byte(`{"rand":"{{rand_base(5)}}"}`)))
progressCounter := &payloadRequestProgress{Progress: executerOptions.Progress}
executerOptions.Progress = progressCounter
require.NoError(t, request.Compile(executerOptions))
var matches atomic.Int32
require.NoError(t, request.ExecuteWithResults(
contextargs.NewWithInput(t.Context(), server.URL),
make(output.InternalEvent),
make(output.InternalEvent),
func(event *output.InternalWrappedEvent) {
if event.OperatorsResult != nil && event.OperatorsResult.Matched {
matches.Add(1)
}
},
))
require.Zero(t, matches.Load())
require.Equal(t, int32(204), served.Load())
require.Equal(t, int64(204), progressCounter.sent.Load())
require.Zero(t, progressCounter.skipped.Load())
require.Zero(t, progressCounter.failed.Load())
require.GreaterOrEqual(t, maximum.Load(), tc.minimumActive)
if tc.maximumActive > 0 {
require.LessOrEqual(t, maximum.Load(), tc.maximumActive)
}
keysMu.Lock()
require.Len(t, keys, 102)
for _, count := range keys {
require.Equal(t, 2, count)
}
keysMu.Unlock()
})
}
}
func TestIndexedResponsePayloadThreadsPreserveCachePoisoningDetection(t *testing.T) {
options := testutils.DefaultOptions.Copy()
options.ExecutionId = t.Name()
options.RestrictLocalNetworkAccess = false
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
require.NotNil(t, memguardian.DefaultMemGuardian)
previousWarning := memguardian.DefaultMemGuardian.Warning.Swap(false)
t.Cleanup(func() { memguardian.DefaultMemGuardian.Warning.Store(previousWarning) })
var mu sync.Mutex
cache := make(map[string]string)
var served atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
served.Add(1)
key := strings.SplitN(request.URL.RawQuery, "=", 2)[0]
mu.Lock()
defer mu.Unlock()
if request.URL.Path == "/first" {
if request.Header.Get("X-Bucket") == "positive" {
cache[key] = "alert(document.domain)"
}
writer.WriteHeader(http.StatusBadGateway)
return
}
if value, ok := cache[key]; ok {
writer.WriteHeader(http.StatusOK)
_, _ = writer.Write([]byte(value))
return
}
writer.WriteHeader(http.StatusNotFound)
}))
defer server.Close()
request := &Request{
ID: "indexed-response-cache-poisoning",
Raw: []string{
"GET /first?{{rand}}-{{md5(escape)}}-{{bucket}}=x HTTP/1.1\nHost: {{Hostname}}\nX-Bucket: {{bucket}}",
"GET /second?{{rand}}-{{md5(escape)}}-{{bucket}}=x HTTP/1.1\nHost: {{Hostname}}",
},
PayloadThreads: 2,
AttackType: generators.AttackTypeHolder{Value: generators.ClusterBombAttack},
Payloads: map[string]interface{}{
"escape": []string{"\v"},
"bucket": []string{"positive", "negative"},
},
Operators: operatorsWithMatchers(&matcherpkg.Matcher{
Type: matcherpkg.MatcherTypeHolder{MatcherType: matcherpkg.DSLMatcher},
DSL: []string{`contains(body_2, "alert(document.domain)") && status_code_2 == 200`},
}),
}
executerOptions := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
ID: request.ID,
Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
})
executerOptions.RateLimiter = nil
require.NoError(t, executerOptions.Variables.UnmarshalJSON([]byte(`{"rand":"{{rand_base(5)}}"}`)))
require.NoError(t, request.Compile(executerOptions))
var matches atomic.Int32
require.NoError(t, request.ExecuteWithResults(
contextargs.NewWithInput(t.Context(), server.URL),
make(output.InternalEvent),
make(output.InternalEvent),
func(event *output.InternalWrappedEvent) {
if event.OperatorsResult != nil && event.OperatorsResult.Matched {
matches.Add(1)
}
},
))
require.Equal(t, int32(4), served.Load())
require.Equal(t, int32(1), matches.Load())
require.Len(t, cache, 1)
}
type payloadRequestProgress struct {
progress.Progress
sent atomic.Int64
skipped atomic.Int64
failed atomic.Int64
}
func (p *payloadRequestProgress) IncrementRequests() {
p.sent.Add(1)
p.Progress.IncrementRequests()
}
func (p *payloadRequestProgress) SetRequests(count uint64) {
p.skipped.Add(int64(count))
p.Progress.SetRequests(count)
}
func (p *payloadRequestProgress) IncrementFailedRequestsBy(count int64) {
p.failed.Add(count)
p.Progress.IncrementFailedRequestsBy(count)
}
func TestNeedsRequestConditionDurationReferences(t *testing.T) {
tests := []struct {
name string
-29
View File
@@ -1,9 +1,7 @@
package network
import (
"crypto/tls"
"encoding/hex"
stderrors "errors"
"fmt"
"maps"
"net"
@@ -71,11 +69,7 @@ func (request *Request) getOpenPorts(target *contextargs.Context) ([]string, err
request.dialer, _ = networkclientpool.Get(request.options.Options, &networkclientpool.Configuration{})
}
if err := request.options.HostBackoffWait(target.Context(), addr); err != nil {
return nil, err
}
conn, err := request.dialer.Dial(target.Context(), "tcp", addr)
request.options.HostBackoffObserveDial(addr, err)
if err != nil {
errs = append(errs, err)
continue
@@ -332,9 +326,6 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
return nil
}
if err := request.options.HostBackoffWait(input.Context(), actualAddress); err != nil {
return err
}
if err := request.options.RateLimitTakeContext(input.Context(), actualAddress); err != nil {
request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err)
request.options.Progress.IncrementFailedRequestsBy(1)
@@ -346,9 +337,6 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
} else {
conn, err = request.dialer.Dial(input.Context(), "tcp", actualAddress)
}
// DialTLS fails the handshake after TCP has connected. Backoff follows the
// TCP result; the request still returns the handshake error.
request.options.HostBackoffObserveDial(actualAddress, tcpDialError(err))
// adds it to unresponsive address list if applicable
request.markHostError(updatedTarget, err)
if err != nil {
@@ -558,23 +546,6 @@ func displayCompactHexView(event *output.InternalWrappedEvent, response string,
}
}
// tcpDialError reports a failure to open TCP. A TLS handshake error means the
// port already accepted the connection.
func tcpDialError(err error) error {
if err == nil {
return nil
}
var record tls.RecordHeaderError
if stderrors.As(err, &record) {
return nil
}
message := err.Error()
if strings.Contains(message, "tls:") || strings.Contains(message, "tls handshake") {
return nil
}
return err
}
// getAddress returns the address of the host to make request to
func getAddress(toTest string) (string, error) {
if strings.Contains(toTest, "://") {
@@ -1,221 +0,0 @@
package network
import (
"context"
"net"
"net/http"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hostbackoff"
)
func newBackoffRequest(t *testing.T, address string, g *hostbackoff.Governor) (*Request, *protocols.ExecutorOptions) {
t.Helper()
options := testutils.DefaultOptions.Copy()
options.SetExecutionID(t.Name())
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ID: "network-host-backoff"})
executerOpts.HostBackoff = g
request := &Request{ID: "network-host-backoff", Address: []string{address}, ReadSize: 2}
require.NoError(t, request.Compile(executerOpts))
return request, executerOpts
}
func execute(request *Request, address string) error {
input := contextargs.NewWithInput(context.Background(), address)
return request.ExecuteWithResults(input, output.InternalEvent{}, output.InternalEvent{}, func(*output.InternalWrappedEvent) {})
}
func TestFailedDialBacksOffHost(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
address := listener.Addr().String()
go func() {
conn, err := listener.Accept()
if err == nil {
_, _ = conn.Write([]byte("ok"))
_ = conn.Close()
}
}()
g := hostbackoff.New(hostbackoff.Config{Step: 100 * time.Millisecond})
request, _ := newBackoffRequest(t, address, g)
require.NoError(t, execute(request, address))
require.NoError(t, listener.Close())
require.Error(t, execute(request, address))
require.Equal(t, 100*time.Millisecond, g.Delay(address), "a port that accepted and now refuses is pushing back")
}
func TestClosedPortIsNotBackedOff(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
address := listener.Addr().String()
require.NoError(t, listener.Close())
g := hostbackoff.New(hostbackoff.Config{Step: 100 * time.Millisecond})
request, _ := newBackoffRequest(t, address, g)
require.Error(t, execute(request, address))
require.Error(t, execute(request, address))
require.Zero(t, g.Delay(address), "a port that never accepted is closed, not blocking")
}
func TestSuccessfulDialWaitsThenDecays(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
t.Cleanup(func() { _ = listener.Close() })
go func() {
for {
conn, err := listener.Accept()
if err != nil {
return
}
_, _ = conn.Write([]byte("ok"))
_ = conn.Close()
}
}()
address := listener.Addr().String()
const step = 150 * time.Millisecond
g := hostbackoff.New(hostbackoff.Config{Step: step})
g.Observe(address, http.StatusTooManyRequests, 0, nil)
request, _ := newBackoffRequest(t, address, g)
start := time.Now()
require.NoError(t, execute(request, address))
require.GreaterOrEqual(t, time.Since(start), step, "a host that earned a delay is paced")
require.Zero(t, g.Delay(address), "a connection that succeeds is recovery")
}
func TestMultiPortProbeWaitsThenDecays(t *testing.T) {
openLn, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
t.Cleanup(func() { _ = openLn.Close() })
go func() {
for {
conn, err := openLn.Accept()
if err != nil {
return
}
_ = conn.Close()
}
}()
openAddr := openLn.Addr().String()
_, openPort, err := net.SplitHostPort(openAddr)
require.NoError(t, err)
closedLn, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
closedAddr := closedLn.Addr().String()
_, closedPort, err := net.SplitHostPort(closedAddr)
require.NoError(t, err)
require.NoError(t, closedLn.Close())
const step = 150 * time.Millisecond
g := hostbackoff.New(hostbackoff.Config{Step: step})
g.Observe(openAddr, http.StatusTooManyRequests, 0, nil)
options := testutils.DefaultOptions.Copy()
options.SetExecutionID(t.Name())
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ID: "network-host-backoff"})
executerOpts.HostBackoff = g
request := &Request{
ID: "network-host-backoff",
Address: []string{"{{Hostname}}"},
Port: openPort + "," + closedPort,
ReadSize: 2,
}
require.NoError(t, request.Compile(executerOpts))
start := time.Now()
require.NoError(t, execute(request, "127.0.0.1"))
require.GreaterOrEqual(t, time.Since(start), step, "a multi-port probe waits on a host:port that already has a delay")
require.Zero(t, g.Delay(openAddr), "a probe that connects is recovery")
require.Zero(t, g.Delay(closedAddr), "a port that never accepted stays unpaced")
}
func TestMultiPortProbeFailureBacksOffPreviouslyOpenPort(t *testing.T) {
closedLn, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
closedAddr := closedLn.Addr().String()
_, closedPort, err := net.SplitHostPort(closedAddr)
require.NoError(t, err)
require.NoError(t, closedLn.Close())
otherLn, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
otherAddr := otherLn.Addr().String()
_, otherPort, err := net.SplitHostPort(otherAddr)
require.NoError(t, err)
require.NoError(t, otherLn.Close())
const step = 100 * time.Millisecond
g := hostbackoff.New(hostbackoff.Config{Step: step})
g.ObserveDial(closedAddr, nil)
options := testutils.DefaultOptions.Copy()
options.SetExecutionID(t.Name())
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ID: "network-host-backoff"})
executerOpts.HostBackoff = g
request := &Request{
ID: "network-host-backoff",
Address: []string{"{{Hostname}}"},
Port: closedPort + "," + otherPort,
}
require.NoError(t, request.Compile(executerOpts))
require.Error(t, execute(request, "127.0.0.1"))
require.Equal(t, step, g.Delay(closedAddr), "a probe that fails on a port that accepted before slows it")
require.Zero(t, g.Delay(otherAddr), "a port that never accepted stays unpaced")
}
func TestTLSHandshakeFailureDoesNotBackOff(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
t.Cleanup(func() { _ = listener.Close() })
go func() {
for {
conn, err := listener.Accept()
if err != nil {
return
}
_, _ = conn.Write([]byte("not tls"))
_ = conn.Close()
}
}()
address := listener.Addr().String()
const step = 100 * time.Millisecond
g := hostbackoff.New(hostbackoff.Config{Step: step})
g.ObserveDial(address, nil)
options := testutils.DefaultOptions.Copy()
options.SetExecutionID(t.Name())
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ID: "network-host-backoff"})
executerOpts.HostBackoff = g
request := &Request{ID: "network-host-backoff", Address: []string{"tls://" + address}, ReadSize: 2}
require.NoError(t, request.Compile(executerOpts))
err = execute(request, address)
require.Error(t, err)
require.Zero(t, g.Delay(address), "a TLS handshake failure after TCP connected stays with the template: %v", err)
}
-15
View File
@@ -3,7 +3,6 @@ package protocols
import (
"context"
"encoding/base64"
"errors"
"sync/atomic"
"github.com/projectdiscovery/fastdialer/fastdialer"
@@ -241,20 +240,6 @@ func (e *ExecutorOptions) RateLimitTakeContext(ctx context.Context, host string)
return nil
}
// HostBackoffWait pauses for the delay address has earned with HostBackoff.
func (e *ExecutorOptions) HostBackoffWait(ctx context.Context, address string) error {
return e.HostBackoff.Wait(ctx, hostratelimit.NormalizeHostPort(address))
}
// HostBackoffObserveDial reports a connection attempt to address to HostBackoff.
func (e *ExecutorOptions) HostBackoffObserveDial(address string, err error) {
// a canceled dial is the scan stopping, not the host asking for less traffic
if errors.Is(err, context.Canceled) {
return
}
e.HostBackoff.ObserveDial(hostratelimit.NormalizeHostPort(address), err)
}
// GetThreadsForNPayloadRequests returns the number of threads to use as default for
// given max-request of payloads
func (e *ExecutorOptions) GetThreadsForNPayloadRequests(totalRequests int, currentThreads int) int {
-4
View File
@@ -256,9 +256,6 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
hostIp = host
}
if err := request.options.HostBackoffWait(input.Context(), addressToDial); err != nil {
return err
}
if err := request.options.RateLimitTakeContext(input.Context(), addressToDial); err != nil {
requestOptions.Output.Request(requestOptions.TemplateID, input.MetaInput.Input, request.Type().String(), err)
requestOptions.Progress.IncrementFailedRequestsBy(1)
@@ -268,7 +265,6 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
timeStart := time.Now()
response, err := request.tlsx.Connect(host, hostIp, port)
duration := time.Since(timeStart)
request.options.HostBackoffObserveDial(addressToDial, err)
if err != nil {
requestOptions.Output.Request(requestOptions.TemplateID, input.MetaInput.Input, request.Type().String(), err)
requestOptions.Progress.IncrementFailedRequestsBy(1)
+3 -20
View File
@@ -1,7 +1,6 @@
package websocket
import (
"context"
"crypto/tls"
"fmt"
"io"
@@ -221,20 +220,10 @@ func (request *Request) executeRequestWithPayloads(target *contextargs.Context,
tlsConfig.ServerName = requestOptions.Options.SNI
}
// Dial reports both a refused TCP connection and a rejected HTTP upgrade.
// Only the TCP result belongs to host backoff: an upgrade failure means
// the port already accepted the connection.
var dialed bool
var dialErr error
websocketDialer := ws.Dialer{
Header: ws.HandshakeHeaderHTTP(header),
Timeout: time.Duration(requestOptions.Options.Timeout) * time.Second,
NetDial: func(ctx context.Context, network, address string) (net.Conn, error) {
dialed = true
conn, err := request.dialer.Dial(ctx, network, address)
dialErr = err
return conn, err
},
Header: ws.HandshakeHeaderHTTP(header),
Timeout: time.Duration(requestOptions.Options.Timeout) * time.Second,
NetDial: request.dialer.Dial,
TLSConfig: tlsConfig,
}
@@ -266,9 +255,6 @@ func (request *Request) executeRequestWithPayloads(target *contextargs.Context,
addressToDial = parsedAddress.String()
if err := requestOptions.HostBackoffWait(target.Context(), addressToDial); err != nil {
return err
}
if err := requestOptions.RateLimitTakeContext(target.Context(), addressToDial); err != nil {
requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err)
requestOptions.Progress.IncrementFailedRequestsBy(1)
@@ -279,9 +265,6 @@ func (request *Request) executeRequestWithPayloads(target *contextargs.Context,
timeStart := time.Now()
conn, readBuffer, _, err := websocketDialer.Dial(target.Context(), addressToDial)
handshakeDuration := time.Since(timeStart)
if dialed {
requestOptions.HostBackoffObserveDial(addressToDial, dialErr)
}
if err != nil {
requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err)
requestOptions.Progress.IncrementFailedRequestsBy(1)
@@ -1,71 +0,0 @@
package websocket
import (
"context"
"net"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hostbackoff"
)
func newBackoffWebsocket(t *testing.T, address string, g *hostbackoff.Governor) *Request {
t.Helper()
options := testutils.DefaultOptions.Copy()
options.SetExecutionID(t.Name())
testutils.Init(options)
t.Cleanup(func() { testutils.Cleanup(options) })
executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ID: "websocket-host-backoff"})
executerOpts.HostBackoff = g
request := &Request{ID: "websocket-host-backoff", Address: address}
require.NoError(t, request.Compile(executerOpts))
return request
}
func executeWebsocket(request *Request, address string) error {
input := contextargs.NewWithInput(context.Background(), address)
return request.ExecuteWithResults(input, output.InternalEvent{}, output.InternalEvent{}, func(*output.InternalWrappedEvent) {})
}
func TestWebsocketHandshakeFailureDoesNotBackOff(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.NotFound(w, r)
}))
t.Cleanup(server.Close)
address := strings.Replace(server.URL, "http://", "ws://", 1)
host := server.Listener.Addr().String()
const step = 100 * time.Millisecond
g := hostbackoff.New(hostbackoff.Config{Step: step})
g.ObserveDial(host, nil)
request := newBackoffWebsocket(t, address, g)
require.Error(t, executeWebsocket(request, address))
require.Zero(t, g.Delay(host), "a rejected upgrade means the port accepted the connection")
}
func TestWebsocketDialFailureBacksOffPreviouslyOpenPort(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
host := listener.Addr().String()
require.NoError(t, listener.Close())
address := "ws://" + host
const step = 100 * time.Millisecond
g := hostbackoff.New(hostbackoff.Config{Step: step})
g.ObserveDial(host, nil)
request := newBackoffWebsocket(t, address, g)
require.Error(t, executeWebsocket(request, address))
require.Equal(t, step, g.Delay(host), "a refused TCP dial on a port that accepted before slows it")
}
+18 -123
View File
@@ -7,8 +7,6 @@ import (
"fmt"
"io"
"net/http"
"sort"
"sync"
"github.com/shurcooL/graphql"
@@ -28,9 +26,6 @@ type Integration struct {
url string
httpclient *http.Client
options *Options
closedStateMu sync.Mutex
closedStateID string
}
// Options contains the configuration options for linear issue tracker client
@@ -52,8 +47,6 @@ type Options struct {
// OpenStateID is the id of the open state for the project
OpenStateID string `yaml:"open-state-id"`
// ClosedStateID is the id of the closed state for the project, discovered from the team's completed workflow states when empty
ClosedStateID string `yaml:"closed-state-id"`
HttpClient *retryablehttp.Client `yaml:"-"`
OmitRaw bool `yaml:"-"`
@@ -122,8 +115,7 @@ func (i *Integration) CreateIssue(event *output.ResultEvent) (*filters.CreateIss
}
var resp struct {
IssueUpdate struct {
LastSyncID int `json:"lastSyncId"`
Success bool `json:"success"`
LastSyncID int `json:"lastSyncId"`
}
}
err := i.doGraphqlRequest(ctx, existingIssueUpdateStateMutation, &resp, variables, "IssueUpdate")
@@ -176,8 +168,10 @@ type createIssueMutation struct {
ID graphql.ID
Title graphql.String
Identifier graphql.String
State linearState
URL graphql.String
State struct {
Name graphql.String
}
URL graphql.String
}
}
}
@@ -191,7 +185,6 @@ const (
identifier
state {
name
type
}
url
}
@@ -210,28 +203,16 @@ const (
identifier
state {
name
type
}
url
}
}
}
`
completedWorkflowStatesQuery = `query ($teamID: ID) {
workflowStates(filter: { team: { id: { eq: $teamID } }, type: { eq: "completed" } }) {
nodes {
id
position
}
}
}
`
existingIssueUpdateStateMutation = `mutation IssueUpdate($issueUpdateInput: IssueUpdateInput!, $issueID: String!) {
issueUpdate(input: $issueUpdateInput, id: $issueID) {
lastSyncId
success
}
}
`
@@ -271,8 +252,12 @@ func (i *Integration) createIssueLinear(ctx context.Context, title, description
ID: mutation.IssueCreate.Issue.ID,
Title: mutation.IssueCreate.Issue.Title,
Identifier: mutation.IssueCreate.Issue.Identifier,
State: mutation.IssueCreate.Issue.State,
URL: mutation.IssueCreate.Issue.URL,
State: struct {
Name graphql.String
}{
Name: mutation.IssueCreate.Issue.State.Name,
},
URL: mutation.IssueCreate.Issue.URL,
}, nil
}
@@ -282,7 +267,7 @@ func (i *Integration) findIssueByTitle(ctx context.Context, title string) (*line
"title": graphql.String(title),
}
if i.options.TeamID != "" {
variables["teamID"] = graphql.ID(i.options.TeamID)
variables["teamId"] = graphql.ID(i.options.TeamID)
}
if i.options.ProjectID != "" {
variables["projectID"] = graphql.ID(i.options.ProjectID)
@@ -304,84 +289,12 @@ func (i *Integration) Name() string {
}
func (i *Integration) CloseIssue(event *output.ResultEvent) error {
ctx := context.Background()
existingIssue, err := i.findIssueByTitle(ctx, format.Summary(event))
if errors.Is(err, io.EOF) {
return nil
}
if err != nil {
return fmt.Errorf("could not find linear issue: %w", err)
}
if isClosedStateType(string(existingIssue.State.Type)) {
return nil
}
stateID, err := i.resolveClosedStateID(ctx)
if err != nil {
return fmt.Errorf("could not resolve linear closed state: %w", err)
}
variables := map[string]interface{}{
"issueUpdateInput": map[string]interface{}{"stateId": stateID},
"issueID": types.ToString(existingIssue.ID),
}
var resp struct {
IssueUpdate struct {
LastSyncID int `json:"lastSyncId"`
Success bool `json:"success"`
}
}
if err := i.doGraphqlRequest(ctx, existingIssueUpdateStateMutation, &resp, variables, "IssueUpdate"); err != nil {
return fmt.Errorf("error closing issue %s: %w", existingIssue.ID, err)
}
if !resp.IssueUpdate.Success {
return fmt.Errorf("error closing issue %s: Linear rejected the update", existingIssue.ID)
}
// TODO: Unimplemented for now as not used in many places
// and overhead of maintaining our own API for this.
// This is too much code as it is :(
return nil
}
func isClosedStateType(stateType string) bool {
switch stateType {
case linearStateTypeCompleted, linearStateTypeCanceled, linearStateTypeDuplicate:
return true
}
return false
}
func (i *Integration) resolveClosedStateID(ctx context.Context) (string, error) {
if i.options.ClosedStateID != "" {
return i.options.ClosedStateID, nil
}
i.closedStateMu.Lock()
cached := i.closedStateID
i.closedStateMu.Unlock()
if cached != "" {
return cached, nil
}
if i.options.TeamID == "" {
return "", errors.New("team-id or closed-state-id is required to close issues")
}
var result completedWorkflowStatesQueryResult
variables := map[string]interface{}{"teamID": graphql.ID(i.options.TeamID)}
if err := i.doGraphqlRequest(ctx, completedWorkflowStatesQuery, &result, variables, ""); err != nil {
return "", err
}
nodes := result.WorkflowStates.Nodes
if len(nodes) == 0 {
return "", fmt.Errorf("no completed workflow state found for team %s", i.options.TeamID)
}
sort.SliceStable(nodes, func(a, b int) bool { return nodes[a].Position < nodes[b].Position })
stateID := types.ToString(nodes[0].ID)
i.closedStateMu.Lock()
i.closedStateID = stateID
i.closedStateMu.Unlock()
return stateID, nil
}
// ShouldFilter determines if an issue should be logged to this tracker
func (i *Integration) ShouldFilter(event *output.ResultEvent) bool {
if i.options.AllowList != nil && !i.options.AllowList.GetMatch(event) {
@@ -395,26 +308,14 @@ func (i *Integration) ShouldFilter(event *output.ResultEvent) bool {
return true
}
type linearState struct {
Name graphql.String
Type graphql.String
}
type linearIssue struct {
ID graphql.ID
Title graphql.String
Identifier graphql.String
State linearState
URL graphql.String
}
type completedWorkflowStatesQueryResult struct {
WorkflowStates struct {
Nodes []struct {
ID graphql.ID
Position graphql.Float
}
State struct {
Name graphql.String
}
URL graphql.String
}
type findExistingIssuesSearch struct {
@@ -434,12 +335,6 @@ func (adt *addHeaderTransport) RoundTrip(req *http.Request) (*http.Response, err
return adt.T.RoundTrip(req)
}
const (
linearStateTypeCompleted = "completed"
linearStateTypeCanceled = "canceled"
linearStateTypeDuplicate = "duplicate"
)
const (
linearPriorityNone = float64(0)
linearPriorityCritical = float64(1)
@@ -1,140 +0,0 @@
package linear
import (
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v3/pkg/model"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
)
type fakeLinear struct {
mu sync.Mutex
stateType string
rejectUpdates bool
holdStates chan struct{}
mutations []map[string]any
searches []map[string]any
stateQuery int
}
func (f *fakeLinear) ServeHTTP(w http.ResponseWriter, r *http.Request) {
var body struct {
Query string `json:"query"`
Variables map[string]any `json:"variables"`
}
_ = json.NewDecoder(r.Body).Decode(&body)
f.mu.Lock()
defer f.mu.Unlock()
w.Header().Set("Content-Type", "application/json")
switch {
case strings.Contains(body.Query, "workflowStates"):
f.stateQuery++
if f.holdStates != nil {
f.mu.Unlock()
<-f.holdStates
f.mu.Lock()
}
_, _ = w.Write([]byte(`{"data":{"workflowStates":{"nodes":[{"id":"state-late","position":9},{"id":"state-done","position":2}]}}}`))
case strings.Contains(body.Query, "issueUpdate"):
f.mutations = append(f.mutations, body.Variables)
success := "true"
if f.rejectUpdates {
success = "false"
}
_, _ = w.Write([]byte(`{"data":{"issueUpdate":{"lastSyncId":1,"success":` + success + `}}}`))
default:
f.searches = append(f.searches, body.Variables)
_, _ = w.Write([]byte(`{"data":{"issues":{"nodes":[{"id":"issue-1","title":"t","identifier":"ENG-1","state":{"name":"Todo","type":"` + f.stateType + `"},"url":"u"}]}}}`))
}
}
func newTestIntegration(t *testing.T, fake *fakeLinear) *Integration {
t.Helper()
server := httptest.NewServer(fake)
t.Cleanup(server.Close)
integration, err := New(&Options{APIKey: "key", TeamID: "team-1"})
require.NoError(t, err)
integration.url = server.URL
return integration
}
func testEvent() *output.ResultEvent {
return &output.ResultEvent{TemplateID: "tpl", Host: "example.com", Info: model.Info{Name: "finding"}}
}
func TestCloseIssueMovesToDiscoveredCompletedState(t *testing.T) {
fake := &fakeLinear{stateType: "unstarted"}
integration := newTestIntegration(t, fake)
require.NoError(t, integration.CloseIssue(testEvent()))
require.NoError(t, integration.CloseIssue(testEvent()))
require.Len(t, fake.mutations, 2)
require.Equal(t, 1, fake.stateQuery)
require.Equal(t, "team-1", fake.searches[0]["teamID"])
require.Equal(t, "issue-1", fake.mutations[0]["issueID"])
require.Equal(t, map[string]any{"stateId": "state-done"}, fake.mutations[0]["issueUpdateInput"])
}
func TestCloseIssueDoesNotHoldTheLockDuringTheStateLookup(t *testing.T) {
fake := &fakeLinear{stateType: "unstarted", holdStates: make(chan struct{})}
integration := newTestIntegration(t, fake)
release := sync.OnceFunc(func() { close(fake.holdStates) })
t.Cleanup(release)
errs := make(chan error, 2)
for range 2 {
go func() { errs <- integration.CloseIssue(testEvent()) }()
}
require.Eventually(t, func() bool {
fake.mu.Lock()
defer fake.mu.Unlock()
return fake.stateQuery == 2
}, 2*time.Second, 10*time.Millisecond, "a second close waits on the first one's Linear request")
release()
require.NoError(t, <-errs)
require.NoError(t, <-errs)
require.Len(t, fake.mutations, 2)
require.Equal(t, map[string]any{"stateId": "state-done"}, fake.mutations[1]["issueUpdateInput"])
}
func TestCloseIssueUsesConfiguredClosedStateID(t *testing.T) {
fake := &fakeLinear{stateType: "started"}
integration := newTestIntegration(t, fake)
integration.options.ClosedStateID = "configured"
require.NoError(t, integration.CloseIssue(testEvent()))
require.Len(t, fake.mutations, 1)
require.Zero(t, fake.stateQuery)
require.Equal(t, map[string]any{"stateId": "configured"}, fake.mutations[0]["issueUpdateInput"])
}
func TestCloseIssueSkipsAlreadyClosedIssue(t *testing.T) {
for _, stateType := range []string{"completed", "canceled", "duplicate"} {
fake := &fakeLinear{stateType: stateType}
integration := newTestIntegration(t, fake)
require.NoError(t, integration.CloseIssue(testEvent()))
require.Empty(t, fake.mutations, stateType)
}
}
func TestCloseIssueFailsWhenLinearRejectsTheUpdate(t *testing.T) {
fake := &fakeLinear{stateType: "started", rejectUpdates: true}
integration := newTestIntegration(t, fake)
require.ErrorContains(t, integration.CloseIssue(testEvent()), "Linear rejected the update")
}