Compare commits

..
Author SHA1 Message Date
Mzack9999 fe24f8638b review fixes 2026-11-04 05:09:15 +01:00
Mzack9999 8b7cbf44e6 all trackers 2026-11-04 03:26:07 +01:00
Mzack9999 59ae4cac04 tls dialer 2026-11-04 03:00:58 +01:00
Mzack9999 64965ba776 tls verify 2026-11-04 00:54:48 +01:00
Mzack9999 15271926ad tracker client 2026-11-03 23:28:48 +01:00
Mzack9999 46fbc92a76 Merge pull request #8023 from projectdiscovery/fix/dns-resolver-client
Give each scan its own DNS resolvers
2026-11-03 21:42:45 +01:00
Mzack9999 36d0adba18 Merge pull request #8024 from projectdiscovery/fix/headless-exclude-lock
Read headless excludes under the dialer lock
2026-11-03 20:47:14 +01:00
Mzack9999 62dae92b3d refresh resolvers 2026-11-03 17:30:04 +01:00
Mzack9999 e44df6e947 lock excludes 2026-11-03 17:30:04 +01:00
15 changed files with 615 additions and 104 deletions

No files matched your search

+26 -25
View File
@@ -77,37 +77,37 @@ require (
github.com/ory/dockertest/v3 v3.12.0
github.com/pkg/errors v0.9.1
github.com/praetorian-inc/fingerprintx v1.1.15
github.com/projectdiscovery/clistats v0.1.9
github.com/projectdiscovery/dsl v0.9.1
github.com/projectdiscovery/fastdialer v0.6.1
github.com/projectdiscovery/clistats v0.1.7
github.com/projectdiscovery/dsl v0.9.0
github.com/projectdiscovery/fastdialer v0.6.0
github.com/projectdiscovery/fasttemplate v0.0.2
github.com/projectdiscovery/gcache v0.0.0-20241015120333-12546c6e3f4c
github.com/projectdiscovery/goflags v0.2.1
github.com/projectdiscovery/goimpacket v0.1.1-0.20261007205806-6042a8213eeb
github.com/projectdiscovery/goja v0.0.0-20261012075649-9b78f2e05b9c
github.com/projectdiscovery/goja_nodejs v0.0.0-20260618132410-8519f75f703d
github.com/projectdiscovery/gologger v1.1.75
github.com/projectdiscovery/gologger v1.1.73
github.com/projectdiscovery/gostruct v0.0.2
github.com/projectdiscovery/govaluate v0.0.0-20261102102118-9294133b15a2
github.com/projectdiscovery/gozero v0.1.1-0.20260530071156-fa1dad563d76
github.com/projectdiscovery/hmap v0.0.104
github.com/projectdiscovery/hmap v0.0.102
github.com/projectdiscovery/httpx v1.12.0
github.com/projectdiscovery/interactsh v1.4.1
github.com/projectdiscovery/mapcidr v1.1.97
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5
github.com/projectdiscovery/networkpolicy v0.1.57
github.com/projectdiscovery/networkpolicy v0.1.56
github.com/projectdiscovery/ratelimit v0.0.94
github.com/projectdiscovery/rawhttp v0.2.2
github.com/projectdiscovery/rawhttp v0.2.0
github.com/projectdiscovery/rdap v0.9.0
github.com/projectdiscovery/retryabledns v1.0.118
github.com/projectdiscovery/retryablehttp-go v1.3.33
github.com/projectdiscovery/retryabledns v1.0.116
github.com/projectdiscovery/retryablehttp-go v1.3.31
github.com/projectdiscovery/sarif v0.1.0
github.com/projectdiscovery/tlsx v1.4.4
github.com/projectdiscovery/tlsx v1.4.3-0.20261020052910-acb80c2c7c75
github.com/projectdiscovery/uncover v1.2.1
github.com/projectdiscovery/useragent v0.0.110
github.com/projectdiscovery/useragent v0.0.109
github.com/projectdiscovery/utils v0.11.13-0.20261103094816-5c0b20e35350
github.com/projectdiscovery/wappalyzergo v0.3.7
github.com/projectdiscovery/yamldoc-go v1.0.9
github.com/projectdiscovery/wappalyzergo v0.3.6
github.com/projectdiscovery/yamldoc-go v1.0.7
github.com/redis/go-redis/v9 v9.19.0
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rs/xid v1.6.0
@@ -128,14 +128,14 @@ require (
go.mongodb.org/mongo-driver v1.17.9
go.uber.org/multierr v1.11.0
golang.org/x/exp v0.0.0-20260527015227-08cc5374adb3
golang.org/x/net v0.61.0
golang.org/x/net v0.59.0
golang.org/x/oauth2 v0.36.0
golang.org/x/sync v0.24.0
golang.org/x/sys v0.49.0
golang.org/x/term v0.47.0
golang.org/x/text v0.43.0
golang.org/x/sync v0.23.0
golang.org/x/sys v0.48.0
golang.org/x/term v0.46.0
golang.org/x/text v0.42.0
golang.org/x/time v0.15.0
golang.org/x/tools v0.52.0
golang.org/x/tools v0.49.0
google.golang.org/grpc v1.83.2
google.golang.org/protobuf v1.36.11
gopkg.in/yaml.v3 v3.0.1
@@ -167,6 +167,7 @@ require (
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 // indirect
github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect
github.com/Mzack9999/goimpacket v0.0.0-20260420131935-a9fe473cda7d // indirect
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
github.com/ProtonMail/go-crypto v1.1.6 // indirect
@@ -373,10 +374,10 @@ require (
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
github.com/projectdiscovery/asnmap v1.2.0 // indirect
github.com/projectdiscovery/asnmap v1.1.1 // indirect
github.com/projectdiscovery/blackrock v0.0.2 // indirect
github.com/projectdiscovery/cdncheck v1.3.3 // indirect
github.com/projectdiscovery/freeport v0.0.8 // indirect
github.com/projectdiscovery/cdncheck v1.3.2 // indirect
github.com/projectdiscovery/freeport v0.0.7 // indirect
github.com/projectdiscovery/jarm-go v0.0.0-20260910160638-430e2bae86f6 // indirect
github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb // indirect
github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582 // indirect
@@ -433,7 +434,7 @@ require (
github.com/ysmood/got v0.40.0 // indirect
github.com/ysmood/gson v0.7.3 // indirect
github.com/ysmood/leakless v0.9.0 // indirect
github.com/yuin/goldmark v1.8.6 // indirect
github.com/yuin/goldmark v1.7.17 // indirect
github.com/yuin/goldmark-emoji v1.0.6 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
github.com/zcalusic/sysinfo v1.1.3 // indirect
@@ -456,8 +457,8 @@ require (
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
goftp.io/server/v2 v2.0.1 // indirect
golang.org/x/arch v0.3.0 // indirect
golang.org/x/crypto v0.58.0 // indirect
golang.org/x/mod v0.42.0 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/mod v0.41.0 // indirect
google.golang.org/api v0.264.0 // indirect
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect
+52 -50
View File
@@ -120,6 +120,8 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 h1:KFac3SiGbId8ub47e7kd2PLZeACxc1LkiiNoDOFRClE=
github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057/go.mod h1:iLB2pivrPICvLOuROKmlqURtFIEsoJZaMidQfCG1+D4=
github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8=
github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4=
github.com/Mzack9999/go-rsync v0.0.0-20250821180103-81ffa574ef4d h1:DofPB5AcjTnOU538A/YD86/dfqSNTvQsAXgwagxmpu4=
github.com/Mzack9999/go-rsync v0.0.0-20250821180103-81ffa574ef4d/go.mod h1:uzdh/m6XQJI7qRvufeBPDa+lj5SVCJO8B9eLxTbtI5U=
github.com/Mzack9999/goimpacket v0.0.0-20260420131935-a9fe473cda7d h1:xX93NZHxxrHcmN4nn1owvJdr1E/l1kp1Rv7rMr9ly+U=
@@ -924,22 +926,22 @@ github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
github.com/praetorian-inc/fingerprintx v1.1.15 h1:CVIxrIQARbmdk5h8E9tIJZbvFoY2sGLLG9rpFVfQqpA=
github.com/praetorian-inc/fingerprintx v1.1.15/go.mod h1:hqRroITBwKpP8BOGF+n/A+qv9wSF7OSVinmu5NCyOUI=
github.com/projectdiscovery/asnmap v1.2.0 h1:xxZxVYJYPyBNbwF7jL/rtpThKBMdjomQBmlAZRqhq7E=
github.com/projectdiscovery/asnmap v1.2.0/go.mod h1:0tNdUGHUE7TFWDaHZtuXcVpQfRTZZ1BKmFfl8gtQbiU=
github.com/projectdiscovery/asnmap v1.1.1 h1:ImJiKIaACOT7HPx4Pabb5dksolzaFYsD1kID2iwsDqI=
github.com/projectdiscovery/asnmap v1.1.1/go.mod h1:QT7jt9nQanj+Ucjr9BqGr1Q2veCCKSAVyUzLXfEcQ60=
github.com/projectdiscovery/blackrock v0.0.2 h1:mxXdu0uM8P2L2Qi210COlU8QiICPFW/Rxk5QUhlPO2k=
github.com/projectdiscovery/blackrock v0.0.2/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss=
github.com/projectdiscovery/cdncheck v1.3.3 h1:ueG4ePG+E1Gt3MPsiSPKEMdJR/Ll1tdY//1FMD5LMSI=
github.com/projectdiscovery/cdncheck v1.3.3/go.mod h1:27FjISwBKtdQXsC31q16xspX5xAu0gbYfF0rs3yGNxs=
github.com/projectdiscovery/clistats v0.1.9 h1:qIxxxjWYuxoGIG02jOeztvz/8metvay2pG2P+Xld154=
github.com/projectdiscovery/clistats v0.1.9/go.mod h1:4TcGnCtsmyy9/pkKaRyqQmKQljwiOPTCc0nnbluhYX0=
github.com/projectdiscovery/dsl v0.9.1 h1:SRInxq1pgA473Zn4sO+6Rtv58J/UUOcr0+1Js4ln5LE=
github.com/projectdiscovery/dsl v0.9.1/go.mod h1:lD4o4Fqgql7YxZYYb4XYR9rHhMG5UZM3iB9MQx3/zKM=
github.com/projectdiscovery/fastdialer v0.6.1 h1:mRwzskHwyFD9fv8iYt85lRvX190Ocys1aDW8IJ7yuEk=
github.com/projectdiscovery/fastdialer v0.6.1/go.mod h1:fZqTJhL43+skKCNeXXuxPA585dFuleIuFW02ZSudWZ8=
github.com/projectdiscovery/cdncheck v1.3.2 h1:I453ZWNBeG0WWIy3wi9412l4PTzorLgebEnqmbcpSfo=
github.com/projectdiscovery/cdncheck v1.3.2/go.mod h1:5IkUMMlcGaeg1DWW5oJVd3r9X3EpJ88j5HxBeF4Me9M=
github.com/projectdiscovery/clistats v0.1.7 h1:NXf4lUzx1QcbJd6QycNwwYHTv2FsW9MchQ2LsgSL6C0=
github.com/projectdiscovery/clistats v0.1.7/go.mod h1:RxRGxhMzkT344VTTvZWIrojurUXdQQRN1yfwgg/88I8=
github.com/projectdiscovery/dsl v0.9.0 h1:i1KyCbKJyKfHkgdbfh0JkWdd14APUjpEYf+GaYANlCg=
github.com/projectdiscovery/dsl v0.9.0/go.mod h1:ntNFkkiYCTEosGFTbApA61RMrvJTCjnakPxnWj56MyY=
github.com/projectdiscovery/fastdialer v0.6.0 h1:4ZeFQ6LyRH4D8e3qO/yzqIP4Cf9Fr0DduaVpJD6ENnE=
github.com/projectdiscovery/fastdialer v0.6.0/go.mod h1:FpHwa9S5PUgfPbh+irOm+cdU6AO631XidxkhPwPv7yY=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
github.com/projectdiscovery/freeport v0.0.8 h1:7EE+m7DY7Yc2afpB/F45V+zH2DL+M0skCSUc3gb5KUs=
github.com/projectdiscovery/freeport v0.0.8/go.mod h1:dtIhwfdXOiA0gyr2PWno1X8RTqtzHbQDNfJGpSvptD4=
github.com/projectdiscovery/freeport v0.0.7 h1:Q6uXo/j8SaV/GlAHkEYQi8WQoPXyJWxyspx+aFmz9Qk=
github.com/projectdiscovery/freeport v0.0.7/go.mod h1:cOhWKvNBe9xM6dFJ3RrrLvJ5vXx2NQ36SecuwjenV2k=
github.com/projectdiscovery/gcache v0.0.0-20241015120333-12546c6e3f4c h1:s+lLAlrOrgwlPZQ9DFqNw+kia2nteKnJZ2Ek313yoUc=
github.com/projectdiscovery/gcache v0.0.0-20241015120333-12546c6e3f4c/go.mod h1:rN35/D3lVx2YDeENFFz06uj8j3XIqK1Ym9XcISF5fzg=
github.com/projectdiscovery/goflags v0.2.1 h1:XnSYvsCjUwHbZaaJUZ3YznCJPoXRk/IhTFP9Gecmx6M=
@@ -950,16 +952,16 @@ github.com/projectdiscovery/goja v0.0.0-20261012075649-9b78f2e05b9c h1:4H/CfVNlu
github.com/projectdiscovery/goja v0.0.0-20261012075649-9b78f2e05b9c/go.mod h1:SO0AP+uKfYeeoR6jyVH/PKRYJE/f5gJrPMAM00iGEMc=
github.com/projectdiscovery/goja_nodejs v0.0.0-20260618132410-8519f75f703d h1:fqqH9LHpN2WDz9QuxFrhKNxXSRtzk+Sa6jAhbB7tXcQ=
github.com/projectdiscovery/goja_nodejs v0.0.0-20260618132410-8519f75f703d/go.mod h1:Ezmbgdaw4EunGGBU4MQViLoGMJc37LA3ip55YV3KeRI=
github.com/projectdiscovery/gologger v1.1.75 h1:w+8DgCOCFpUegE2Ww8LQBUWPACkJUOvJ4Q1C0f2N9qI=
github.com/projectdiscovery/gologger v1.1.75/go.mod h1:LDeWHizV+0QVuvDP4uJHM/M+eQlp0XLWlQ1z8PgOE1k=
github.com/projectdiscovery/gologger v1.1.73 h1:WX2nzq29cL7HdvOHqIttacoid/5lMwapyM7ukGU1jFY=
github.com/projectdiscovery/gologger v1.1.73/go.mod h1:mJwODZcFDg70ihINpOvZevmBtgvpP8H9/l8Y+OPhZPY=
github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
github.com/projectdiscovery/govaluate v0.0.0-20261102102118-9294133b15a2 h1:aSLbDvrJQPlp4HvojU+bTU5XV8xdAVCjoc5gX1Wfbw4=
github.com/projectdiscovery/govaluate v0.0.0-20261102102118-9294133b15a2/go.mod h1:Z3VjpUJV1pnXDwvAddA8aZ4e839SoeftB7CSdZntAVI=
github.com/projectdiscovery/gozero v0.1.1-0.20260530071156-fa1dad563d76 h1:AN70bbi6BBs7KpIM9w0LxygUN7uzT/oH+owDIQ+Fz/k=
github.com/projectdiscovery/gozero v0.1.1-0.20260530071156-fa1dad563d76/go.mod h1:cWHYnRXoYWHtTpOYyAp5laGYX8GH8ITUhgQaP8G/8FA=
github.com/projectdiscovery/hmap v0.0.104 h1:38j4A2gttI48tI5MrdRV2ZTH3SWEysoSYeDxqYqW3rA=
github.com/projectdiscovery/hmap v0.0.104/go.mod h1:koGp3dcpH+BNj/9JuoZnXhbRfAiZrgIiCzsiBCtSIFs=
github.com/projectdiscovery/hmap v0.0.102 h1:ybPuf7UQvuOLiHLOUbVMvd+PSlItQU5iQpP1Erl8kaU=
github.com/projectdiscovery/hmap v0.0.102/go.mod h1:IZ92wXFmxYpdn0euZ7VFmypzOBNDwq8c3yRyIIol6XE=
github.com/projectdiscovery/httpx v1.12.0 h1:NN5iDrw7NjPBDlACdjac4lePTQmZ5aOmgOzptMpP44c=
github.com/projectdiscovery/httpx v1.12.0/go.mod h1:OgtyibNGdu9BrJxRdNKm280Davw1Z4s2FAhfPNNtLB4=
github.com/projectdiscovery/interactsh v1.4.1 h1:EGHPJ1rnuxvILWfcoaRgb/Ykz67WiDXABRCSJN666GA=
@@ -974,34 +976,34 @@ github.com/projectdiscovery/mapcidr v1.1.97 h1:7FkxNNVXp+m1rIu5Nv/2SrF9k4+LwP8Qu
github.com/projectdiscovery/mapcidr v1.1.97/go.mod h1:9dgTJh1SP02gYZdpzMjm6vtYFkEHQHoTyaVNvaeJ7lA=
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw1pfT6bg35NiN7yd1XKtJap5Nk6lMwQ0RNi8=
github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc=
github.com/projectdiscovery/networkpolicy v0.1.57 h1:3P7esW8KE3yoZSE/GvswbSZ1bh8Gk+dURtvNlFzOid0=
github.com/projectdiscovery/networkpolicy v0.1.57/go.mod h1:Ky0GOIqAL666JBHyHeIsQ5rjkET902I6XjGEpvCVx9s=
github.com/projectdiscovery/networkpolicy v0.1.56 h1:M/L421HFuN707oX0tLTzKZY46uRyaKwxAeyi0q8PlxI=
github.com/projectdiscovery/networkpolicy v0.1.56/go.mod h1:N5bXm8L3UsDw51CeA1nhTsGEbQMbgL5lXTiIU013xd0=
github.com/projectdiscovery/ratelimit v0.0.94 h1:2qciSuUxarn73YGICZeprZtjrwyAH+0iVpdKzKCXoXE=
github.com/projectdiscovery/ratelimit v0.0.94/go.mod h1:GAerX6s+eWRuACedHRv2/xmMtlW+VahZmbRstQh0sS0=
github.com/projectdiscovery/rawhttp v0.2.2 h1:l1UvmSWrpd1wpTIlYKlP4VnmV3jhqR7P05wyDwKBml8=
github.com/projectdiscovery/rawhttp v0.2.2/go.mod h1:Wg1bJ3DGXdVcU2ieU1NqT09rGolvp+FlVeo1QdTnRmg=
github.com/projectdiscovery/rawhttp v0.2.0 h1:TGiHC5hydL97/zoi0WjEuNrfO6Gjtg9LeXndUf1Cq3M=
github.com/projectdiscovery/rawhttp v0.2.0/go.mod h1:Waz+K5y6r44ElpqAoOfYtY03zrGGZ12nx0yDKq7Ot1E=
github.com/projectdiscovery/rdap v0.9.0 h1:wPhHx5pQ2QI+WGhyNb2PjhTl0NtB39Nk7YFZ9cp8ZGA=
github.com/projectdiscovery/rdap v0.9.0/go.mod h1:zk4yrJFQ2Hy36Aqk+DvotYQxYAeALaCJ5ORySkff36Q=
github.com/projectdiscovery/retryabledns v1.0.118 h1:vrWzUgjRyD5fg2QiPF0boDuSL/QYuoxPftgpYQv2MiQ=
github.com/projectdiscovery/retryabledns v1.0.118/go.mod h1:qHuxflnN8BWWqQ+N+TIe9OTqmaMpE/G2+36chAcFQhg=
github.com/projectdiscovery/retryablehttp-go v1.3.33 h1:4lxqAmMsYmUBj9Vzr/ShVCVNv4l7oHZzuky06jQMzSs=
github.com/projectdiscovery/retryablehttp-go v1.3.33/go.mod h1:5f8006QmBK+6ik+r4mQNQrDqIl2qJ3zXrEMzhHZNWLA=
github.com/projectdiscovery/retryabledns v1.0.116 h1:tK28PJ4tLO7bgM89RpIFjkfTPMSbRegwvg9mN/q6z78=
github.com/projectdiscovery/retryabledns v1.0.116/go.mod h1:r3vH0AKYtJ9fculcqRFrdVYe2ABcHwZeR4C6j3rqCbY=
github.com/projectdiscovery/retryablehttp-go v1.3.31 h1:0hSEZ5+jj6Oo1IL8qrytD36TK/qCxfe543jEaRqSTjs=
github.com/projectdiscovery/retryablehttp-go v1.3.31/go.mod h1:/WEa7YejxTCXEMLCGfGYaC3JsgXgzuqw8rv8cb4buEc=
github.com/projectdiscovery/sarif v0.1.0 h1:O541T+a448nSJsmIMnXXSOeDQEzpnCAYvRfe0eG5h74=
github.com/projectdiscovery/sarif v0.1.0/go.mod h1:LBC+reM3bkI3qIIhE0rZaINaYX6VG+En6u2hHa5mA7E=
github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0=
github.com/projectdiscovery/tlsx v1.4.4 h1:XBoy/wwJ3pPwD+8vEtiIrJVJnC3QrKtVNRRZj3A0vH4=
github.com/projectdiscovery/tlsx v1.4.4/go.mod h1:T26MvekV2O9keTvCKvij5pLEBb1OtC5LKYjAs/xgs1A=
github.com/projectdiscovery/tlsx v1.4.3-0.20261020052910-acb80c2c7c75 h1:3qJgfWzzwtFXc/vdUIurSzPzyjBH9NQe0wATkZMoQGI=
github.com/projectdiscovery/tlsx v1.4.3-0.20261020052910-acb80c2c7c75/go.mod h1:8jqB9aHT6U8NkUsENvaSjD/iBEGB4Ta+CbJBMcvzP9c=
github.com/projectdiscovery/uncover v1.2.1 h1:8U46T/96CLT7BPoXBgkTvWqB06lOyeTSLvh5+UjzATE=
github.com/projectdiscovery/uncover v1.2.1/go.mod h1:0p8onrWxfpXQEYs90ZDzTSpu1107gWmodX1NWqu/+z4=
github.com/projectdiscovery/useragent v0.0.110 h1:hoKpQssbFLSSmK/Vxf/fG/a+PToEYdzLafG7HHbvCAc=
github.com/projectdiscovery/useragent v0.0.110/go.mod h1:fOeQ2+nlOJHYuPpnWWrBz5q5dUo7efe/AmvWQdNsLUM=
github.com/projectdiscovery/useragent v0.0.109 h1:b86BPdZvmgbJousa04jAugDRY4Y+rAvEamE0TFCPmeE=
github.com/projectdiscovery/useragent v0.0.109/go.mod h1:sRyBqDAcD31RKYrO8fyXGowLd36fNwY1g0MNW9mp7zk=
github.com/projectdiscovery/utils v0.11.13-0.20261103094816-5c0b20e35350 h1:uXqpYa6XKV9dCvoti9PbSgIWIsBKPu5YKMywB9G4q0Q=
github.com/projectdiscovery/utils v0.11.13-0.20261103094816-5c0b20e35350/go.mod h1:TJgsyVjB02CavwZVFzUzPg6YOSJXQxLhL808JCiQKlU=
github.com/projectdiscovery/wappalyzergo v0.3.7 h1:EITzdN07LR+opmKWT0iEaki366CVB57EBbjq9llsbf0=
github.com/projectdiscovery/wappalyzergo v0.3.7/go.mod h1:7gTNIjQSXsVmFDJJ3PT8+FhYE+dv6hZLGZSsRAEjVSw=
github.com/projectdiscovery/yamldoc-go v1.0.9 h1:pms4u/qWaMYTMnI1R+MUKu/qRV3k/nruHG0wDj2IJmo=
github.com/projectdiscovery/yamldoc-go v1.0.9/go.mod h1:9PyUvPQYG/JuO8ejBDwCcpCWk12/1R/i+7WUbsZ+KNY=
github.com/projectdiscovery/wappalyzergo v0.3.6 h1:1FXCMeCa1xVu2zaW+eeBcbVxtlMStqGOlKcMLyh2pzI=
github.com/projectdiscovery/wappalyzergo v0.3.6/go.mod h1:4E6+fh/ckaY3wo5tlM7lNYSkhuTYRHzu2YmGIRraXP0=
github.com/projectdiscovery/yamldoc-go v1.0.7 h1:Odmjd1o4EkMff2F6QHi1dIYPIQPZUFy24YqmQCprmTk=
github.com/projectdiscovery/yamldoc-go v1.0.7/go.mod h1:nqD1iPMcofpRA1DbXavPByJhWURrshT+l8WN9XjyQag=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
@@ -1231,8 +1233,8 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yuin/goldmark v1.8.6 h1:d0VcaP1sx9GkFVkoW+KtggpGi2KZ965i14b0+bDQST4=
github.com/yuin/goldmark v1.8.6/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
github.com/yuin/goldmark v1.7.17 h1:p36OVWwRb246iHxA/U4p8OPEpOTESm4n+g+8t0EE5uA=
github.com/yuin/goldmark v1.7.17/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
github.com/yuin/goldmark-emoji v1.0.6 h1:QWfF2FYaXwL74tfGOW5izeiZepUDroDJfWubQI9HTHs=
github.com/yuin/goldmark-emoji v1.0.6/go.mod h1:ukxJDKFpdFb5x0a5HqbdlcKtebh086iJpI31LTKmWuA=
github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
@@ -1336,8 +1338,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1375,8 +1377,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg=
golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs=
golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c=
golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1425,8 +1427,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/net v0.61.0 h1:RR+6j/BTBCrggOLugxa+66R8FvoN6MHlzDlZIbN9YFw=
golang.org/x/net v0.61.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI=
golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1453,8 +1455,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA=
golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1519,8 +1521,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -1533,8 +1535,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM=
golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY=
golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE=
golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1551,8 +1553,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc=
golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s=
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1604,8 +1606,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/tools v0.52.0 h1:ryxg4U63oaUYk7uOLTF37n43f4qgtGvqV4Ioit1BaEs=
golang.org/x/tools v0.52.0/go.mod h1:ItcP5UNaVGG+Yc4eIYmaFLNJ4soEopkx5Y2d4lcEurs=
golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -0,0 +1,46 @@
package protocolstate
import (
"sync"
"testing"
"github.com/projectdiscovery/networkpolicy"
"github.com/stretchr/testify/require"
)
func TestIsHostAllowedRacesExcludeUpdate(t *testing.T) {
np, err := networkpolicy.New(networkpolicy.Options{DenyList: []string{"127.0.0.0/8"}})
require.NoError(t, err)
const executionID = "exclude-race"
entry := &Dialers{NetworkPolicy: np, ExcludeTargets: []string{"192.0.2.1"}}
err = dialers.Set(executionID, entry)
require.NoError(t, err)
t.Cleanup(func() { dialers.Delete(executionID) })
stop := make(chan struct{})
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
lists := [][]string{{"192.0.2.1"}, {"192.0.2.2", "192.0.2.3"}}
i := 0
for {
select {
case <-stop:
return
default:
}
entry.Lock()
entry.ExcludeTargets = lists[i%2]
entry.Unlock()
i++
}
}()
for i := 0; i < 2000; i++ {
_ = IsHostAllowed(executionID, "http://192.0.2.1")
}
close(stop)
wg.Wait()
}
@@ -138,11 +138,14 @@ func CanonicalizeHost(host string) string {
// IsHostAllowed checks if the host is allowed by network policy
func IsHostAllowed(executionId string, targetUrl string) bool {
dialers, ok := dialers.Get(executionId)
if !ok {
if !ok || dialers == nil {
return true
}
dialers.Lock()
np := dialers.NetworkPolicy
excludedTargets := append([]string(nil), dialers.ExcludeTargets...)
dialers.Unlock()
if np == nil {
// no policy present: deny rather than allow.
return false
@@ -174,7 +177,7 @@ func IsHostAllowed(executionId string, targetUrl string) bool {
canonHost := CanonicalizeHost(host)
// Check explicit ExcludeTargets against raw, host, and canonical host
for _, excluded := range dialers.ExcludeTargets {
for _, excluded := range excludedTargets {
ex := CanonicalizeHost(strings.TrimSpace(excluded))
if ex != "" {
if strings.EqualFold(canonHost, ex) || strings.EqualFold(host, ex) || strings.EqualFold(rawTarget, ex) {
+28 -19
View File
@@ -26,33 +26,42 @@ var defaultResolvers = []string{
"8.8.4.4:53", // Google
}
// Init initializes the client pool implementation
// Init initializes the client pool implementation.
// Each resolver list gets its own client, so a later scan is not stuck with the first scan's resolvers.
func Init(options *types.Options) error {
m.Lock()
defer m.Unlock()
// Don't create clients if already created in the past.
if normalClient != nil {
return nil
}
clientPool = mapsutil.NewSyncLockMap[string, *retryabledns.Client]()
resolvers := defaultResolvers
if len(options.InternalResolversList) > 0 {
resolvers = options.InternalResolversList
}
var err error
normalClient, err = retryabledns.New(resolvers, 1)
client, err := cachedClient(resolversFor(options))
if err != nil {
return errors.Wrap(err, "could not create dns client")
}
m.Lock()
normalClient = client
m.Unlock()
return nil
}
func getNormalClient() *retryabledns.Client {
func resolversFor(options *types.Options) []string {
if options != nil && len(options.InternalResolversList) > 0 {
return options.InternalResolversList
}
return defaultResolvers
}
func cachedClient(resolvers []string) (*retryabledns.Client, error) {
m.Lock()
defer m.Unlock()
return normalClient
if clientPool == nil {
clientPool = mapsutil.NewSyncLockMap[string, *retryabledns.Client]()
}
key := strings.Join(resolvers, ",")
if client, ok := clientPool.Get(key); ok {
return client, nil
}
client, err := retryabledns.New(resolvers, 1)
if err != nil {
return nil, err
}
_ = clientPool.Set(key, client)
return client, nil
}
// Configuration contains the custom configuration options for a client
@@ -81,7 +90,7 @@ func (c *Configuration) Hash() string {
// Get creates or gets a client for the protocol based on custom configuration
func Get(options *types.Options, configuration *Configuration) (*retryabledns.Client, error) {
if (configuration.Retries <= 1) && len(configuration.Resolvers) == 0 {
return getNormalClient(), nil
return cachedClient(resolversFor(options))
}
hash := configuration.Hash()
if client, ok := clientPool.Get(hash); ok {
@@ -0,0 +1,35 @@
package dnsclientpool
import (
"testing"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/stretchr/testify/require"
)
func TestClientsFollowResolverList(t *testing.T) {
previousClient := normalClient
previousPool := clientPool
t.Cleanup(func() {
normalClient = previousClient
clientPool = previousPool
})
normalClient = nil
clientPool = nil
firstOptions := &types.Options{InternalResolversList: []string{"192.0.2.1:53"}}
secondOptions := &types.Options{InternalResolversList: []string{"192.0.2.2:53"}}
require.NoError(t, Init(firstOptions))
require.NoError(t, Init(secondOptions))
first, err := Get(firstOptions, &Configuration{})
require.NoError(t, err)
second, err := Get(secondOptions, &Configuration{})
require.NoError(t, err)
require.NotSame(t, first, second)
again, err := Get(firstOptions, &Configuration{})
require.NoError(t, err)
require.Same(t, first, again)
}
+12 -1
View File
@@ -94,10 +94,21 @@ func NewTransportWrapper() func(http.RoundTripper) http.RoundTripper {
},
DisableWarningHeader: true,
}
return transport
return &unwrappingTransport{Transport: transport}
}
}
// unwrappingTransport lets code that must not use the cache, such as reporting
// clients that need TLS verification, reach the wrapped transport.
type unwrappingTransport struct {
*httpcache.Transport
}
// Unwrap returns the transport the cache wraps.
func (t *unwrappingTransport) Unwrap() http.RoundTripper {
return t.Transport.Transport
}
func openDiskCache(dir string, maxBytes int64) (httpcache.Cache, *leveldb.DB, error) {
if err := os.MkdirAll(dir, 0o700); err != nil {
return nil, nil, fmt.Errorf("create cache dir: %w", err)
@@ -1,6 +1,7 @@
package httpcache
import (
"crypto/tls"
"fmt"
"io"
"net/http"
@@ -10,6 +11,7 @@ import (
"testing"
"time"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/tlsverify"
libcache "github.com/sandrolain/httpcache"
)
@@ -178,3 +180,29 @@ func TestDiskCacheRespectsSizeCap(t *testing.T) {
t.Fatal("stored a response over a 1-byte directory cap")
}
}
// Reporting clients send tokens, so tlsverify must reach the scan transport
// under the cache wrapper and turn certificate verification on.
func TestTransportWrapperUnwrapsForTLSVerification(t *testing.T) {
cacheMu.Lock()
saved := cache
cache = libcache.NewMemoryCache()
cacheMu.Unlock()
t.Cleanup(func() {
cacheMu.Lock()
cache = saved
cacheMu.Unlock()
})
base := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
wrapped := NewTransportWrapper()(base)
u, ok := wrapped.(interface{ Unwrap() http.RoundTripper })
if !ok || u.Unwrap() != base {
t.Fatal("the cache wrapper does not unwrap to the scan transport")
}
got, ok := tlsverify.Client(&http.Client{Transport: wrapped}).Transport.(*http.Transport)
if !ok || got.TLSClientConfig.InsecureSkipVerify {
t.Fatal("tlsverify did not turn verification on under the cache wrapper")
}
}
@@ -166,6 +166,11 @@ func (t *connTrackingTransport) RoundTrip(req *http.Request) (*http.Response, er
return resp, err
}
// Unwrap returns the underlying transport.
func (t *connTrackingTransport) Unwrap() http.RoundTripper {
return t.base
}
func (t *connTrackingTransport) CloseIdleConnections() {
type closeIdler interface{ CloseIdleConnections() }
if ci, ok := t.base.(closeIdler); ok {
+13
View File
@@ -20,6 +20,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/pdf"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/sarif"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/splunk"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/tlsverify"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/gitea"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/github"
@@ -27,6 +28,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/jira"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/linear"
"github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml"
"github.com/projectdiscovery/retryablehttp-go"
"github.com/projectdiscovery/utils/errkit"
fileutil "github.com/projectdiscovery/utils/file"
"go.uber.org/multierr"
@@ -76,6 +78,17 @@ type IssueTrackerStats struct {
func New(options *Options, db string, doNotDedupe bool) (Client, error) {
client := &ReportingClient{options: options}
// The caller can pass the scan client, which skips TLS verification;
// trackers and exporters send tokens and passwords, so they get a copy
// that verifies certificates.
if options.HttpClient != nil && options.HttpClient.HTTPClient != nil {
if hc := tlsverify.Client(options.HttpClient.HTTPClient); hc != options.HttpClient.HTTPClient {
opts := retryablehttp.DefaultOptionsSingle
opts.Timeout = hc.Timeout // keep the caller's timeout, also when it is 0
options.HttpClient = retryablehttp.NewWithHTTPClient(hc, opts)
}
}
if options.GitHub != nil {
options.GitHub.HttpClient = options.HttpClient
options.GitHub.OmitRaw = options.OmitRaw
+98
View File
@@ -0,0 +1,98 @@
package reporting_test
import (
"io"
"net"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"github.com/projectdiscovery/nuclei/v3/pkg/model"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/es"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/splunk"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/gitea"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/github"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/gitlab"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/jira"
"github.com/projectdiscovery/nuclei/v3/pkg/types"
"github.com/stretchr/testify/require"
)
// connectProxy tunnels every CONNECT to target, whatever host the client asked
// for, so no name is resolved and nothing leaves 127.0.0.1.
func connectProxy(t *testing.T, target string) *httptest.Server {
p := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodConnect {
http.Error(w, "connect only", http.StatusMethodNotAllowed)
return
}
up, err := net.Dial("tcp", target)
if err != nil {
http.Error(w, err.Error(), http.StatusBadGateway)
return
}
c, _, err := w.(http.Hijacker).Hijack()
if err != nil {
_ = up.Close()
return
}
_, _ = c.Write([]byte("HTTP/1.1 200 Connection established\r\n\r\n"))
go func() { _, _ = io.Copy(up, c); _ = up.Close() }()
go func() { _, _ = io.Copy(c, up); _ = c.Close() }()
}))
t.Cleanup(p.Close)
return p
}
// With -pi and a proxy, the runner gives reporting the scan client, which skips
// TLS verification. No tracker or exporter may send its secret to a server with
// an untrusted certificate.
func TestNewVerifiesTLSWithScanClient(t *testing.T) {
var hits atomic.Int32
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
w.WriteHeader(http.StatusUnauthorized)
}))
t.Cleanup(srv.Close)
proxy := connectProxy(t, srv.Listener.Addr().String())
opts := types.DefaultOptions()
opts.SetExecutionID("reporting-tls-test")
opts.ProxyInternal = true
opts.AliveHttpProxy = proxy.URL
require.NoError(t, protocolstate.Init(opts))
t.Cleanup(func() { protocolstate.Close(opts.ExecutionId) })
scanClient, err := httpclientpool.Get(opts, &httpclientpool.Configuration{}, "")
require.NoError(t, err)
event := &output.ResultEvent{
TemplateID: "test-template",
Host: "https://target.example",
Matched: "https://target.example/",
Info: model.Info{Name: "test", SeverityHolder: severity.Holder{Severity: severity.High}},
}
configs := map[string]*reporting.Options{
"github": {GitHub: &github.Options{BaseURL: "https://github.victim.example/", Username: "u", Owner: "o", ProjectName: "r", Token: "test-token"}},
"gitlab": {GitLab: &gitlab.Options{BaseURL: "https://gitlab.victim.example/", Username: "u", Token: "test-token", ProjectName: "1"}},
"gitea": {Gitea: &gitea.Options{BaseURL: "https://gitea.victim.example/", Token: "test-token", ProjectOwner: "o", ProjectName: "r"}},
"jira": {Jira: &jira.Options{Cloud: true, URL: "https://jira.victim.example", AccountID: "a", Email: "[email protected]", Token: "test-token", ProjectName: "P", IssueType: "Bug"}},
"elasticsearch": {ElasticsearchExporter: &es.Options{Host: "es.victim.example", Port: 443, SSL: true, Username: "u", Password: "test-password", IndexName: "n"}},
"splunk": {SplunkExporter: &splunk.Options{Host: "splunk.victim.example", Port: 443, SSL: true, Token: "test-token", IndexName: "n"}},
}
for name, ro := range configs {
ro.HttpClient = scanClient
ro.ExecutionId = opts.ExecutionId
client, err := reporting.New(ro, "", true)
if err == nil {
_ = client.CreateIssue(event)
client.Close()
}
require.Zero(t, hits.Load(), "%s sent a request to a server with an untrusted certificate", name)
}
}
+68
View File
@@ -0,0 +1,68 @@
// Package tlsverify gives trackers and exporters an HTTP client that verifies
// TLS certificates, since they send tokens and passwords.
package tlsverify
import (
"crypto/tls"
"errors"
"net/http"
)
// errTLSDialerOnly is returned when a transport dials HTTPS only through a
// custom TLS dialer: dropping that dialer would also drop its route or address
// policy, so the copy refuses requests instead.
var errTLSDialerOnly = errors.New("tlsverify: the transport has a custom TLS dialer and no DialContext, so certificate verification cannot be enforced")
type refusingTransport struct{}
func (refusingTransport) RoundTrip(*http.Request) (*http.Response, error) {
return nil, errTLSDialerOnly
}
// Client returns c, or a copy of c whose transport verifies TLS certificates
// when c can skip verification (the scan client can). The copy keeps the proxy
// and the dialer, so the address policy still applies, but it drops a custom
// TLS dialer, because that dialer can use the scan TLS config. So the token
// never goes over an unverified connection.
//
// A transport with an Unwrap() http.RoundTripper method declares that its
// RoundTrip only observes or caches requests and enforces no policy; the copy
// uses the unwrapped transport. Other transports are returned unchanged.
func Client(c *http.Client) *http.Client {
rt := c.Transport
for {
u, ok := rt.(interface{ Unwrap() http.RoundTripper })
if !ok {
break
}
rt = u.Unwrap()
}
t, ok := rt.(*http.Transport)
if !ok {
return c
}
insecure := t.TLSClientConfig != nil && t.TLSClientConfig.InsecureSkipVerify
tlsDialer := t.DialTLSContext != nil || t.DialTLS != nil //nolint:staticcheck // DialTLS is deprecated but still used
if !insecure && !tlsDialer {
return c
}
copied := *c
if tlsDialer && t.DialContext == nil && t.Dial == nil { //nolint:staticcheck // Dial is deprecated but still used
copied.Transport = refusingTransport{}
return &copied
}
verifying := t.Clone()
verifying.DialTLSContext = nil
verifying.DialTLS = nil //nolint:staticcheck // DialTLS is deprecated but still used
if insecure {
// Keep the caller's roots, client certificates and checks; the server
// name of the scan config belongs to a scan target, not to this host.
verifying.TLSClientConfig.InsecureSkipVerify = false
verifying.TLSClientConfig.ServerName = ""
if verifying.TLSClientConfig.MinVersion < tls.VersionTLS12 {
verifying.TLSClientConfig.MinVersion = tls.VersionTLS12
}
}
copied.Transport = verifying
return &copied
}
+67
View File
@@ -0,0 +1,67 @@
package tlsverify
import (
"context"
"crypto/tls"
"crypto/x509"
"errors"
"net"
"net/http"
"testing"
"github.com/stretchr/testify/require"
)
// The scan client can skip TLS verification; the tracker must not send the
// token over such a connection, and must not change the caller's client.
func TestClientTurnsOnTLSVerification(t *testing.T) {
insecure := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
caller := &http.Client{Transport: insecure}
got := Client(caller)
gotTransport, ok := got.Transport.(*http.Transport)
require.True(t, ok)
require.False(t, gotTransport.TLSClientConfig.InsecureSkipVerify)
require.True(t, insecure.TLSClientConfig.InsecureSkipVerify, "the caller's transport changed")
secure := &http.Client{Transport: &http.Transport{}}
require.Same(t, secure, Client(secure))
}
func TestClientKeepsCallerTLSSettings(t *testing.T) {
pinned := func(tls.ConnectionState) error { return nil }
insecure := &http.Transport{
DialTLSContext: func(context.Context, string, string) (net.Conn, error) { return nil, errors.New("scan dialer") },
DialContext: (&net.Dialer{}).DialContext,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
ServerName: "scan-target.example",
MinVersion: tls.VersionTLS10,
RootCAs: x509.NewCertPool(),
VerifyConnection: pinned,
},
}
got := Client(&http.Client{Transport: insecure}).Transport.(*http.Transport)
require.Nil(t, got.DialTLSContext)
require.NotNil(t, got.DialContext)
cfg := got.TLSClientConfig
require.False(t, cfg.InsecureSkipVerify)
require.Empty(t, cfg.ServerName)
require.Equal(t, uint16(tls.VersionTLS12), cfg.MinVersion)
require.Same(t, insecure.TLSClientConfig.RootCAs, cfg.RootCAs)
require.NotNil(t, cfg.VerifyConnection)
require.True(t, insecure.TLSClientConfig.InsecureSkipVerify, "the caller's config changed")
}
// A custom TLS dialer may carry the only route or address policy, so a
// transport without DialContext cannot be rebuilt and must refuse requests.
func TestClientRefusesTLSDialerWithoutDialContext(t *testing.T) {
var dialed bool
tlsOnly := &http.Transport{DialTLSContext: func(context.Context, string, string) (net.Conn, error) {
dialed = true
return nil, errors.New("policy dialer")
}}
_, err := Client(&http.Client{Transport: tlsOnly}).Get("https://tracker.example/")
require.ErrorIs(t, err, errTLSDialerOnly)
require.False(t, dialed)
}
+6 -7
View File
@@ -4,7 +4,6 @@ import (
"context"
"fmt"
"io"
"net/http"
"strconv"
"github.com/google/go-github/v92/github"
@@ -12,6 +11,7 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown/util"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/format"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/tlsverify"
"github.com/projectdiscovery/nuclei/v3/pkg/reporting/trackers/filters"
"github.com/projectdiscovery/retryablehttp-go"
"golang.org/x/oauth2"
@@ -55,17 +55,16 @@ type Options struct {
// New creates a new issue tracker integration client based on options.
func New(options *Options) (*Integration, error) {
ctx := context.Background()
// Build the oauth2 client on top of the caller's client, so its transport
// (proxy, address policy) and redirect rules apply to every request.
if options.HttpClient != nil && options.HttpClient.HTTPClient != nil {
ctx = context.WithValue(ctx, oauth2.HTTPClient, tlsverify.Client(options.HttpClient.HTTPClient))
}
ts := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: options.Token},
)
tc := oauth2.NewClient(ctx, ts)
if options.HttpClient != nil && options.HttpClient.HTTPClient != nil {
if tcTransport, ok := tc.Transport.(*http.Transport); ok {
tcTransport.Proxy = options.HttpClient.HTTPClient.Transport.(*http.Transport).Proxy
}
}
clientOptions := []github.ClientOptionsFunc{github.WithHTTPClient(tc)}
if options.BaseURL != "" {
clientOptions = append(clientOptions, github.WithURLs(&options.BaseURL, nil))
@@ -0,0 +1,126 @@
package github
import (
"context"
"crypto/tls"
"errors"
"net"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"github.com/projectdiscovery/nuclei/v3/pkg/model"
"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
"github.com/projectdiscovery/retryablehttp-go"
"github.com/stretchr/testify/require"
)
type roundTripFunc func(*http.Request) (*http.Response, error)
func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
func testEvent() *output.ResultEvent {
return &output.ResultEvent{
TemplateID: "test-template",
Host: "https://target.example",
Info: model.Info{Name: "test", SeverityHolder: severity.Holder{Severity: severity.High}},
}
}
func clientWithTransport(rt http.RoundTripper) *retryablehttp.Client {
c := retryablehttp.NewClient(retryablehttp.DefaultOptionsSingle)
c.HTTPClient = &http.Client{Transport: rt}
return c
}
// The tracker must send every request through the caller's client, so a
// caller's address policy also covers the GitHub API base URL.
func TestNewUsesCallerHTTPClient(t *testing.T) {
var serverHits atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
serverHits.Add(1)
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusCreated)
_, _ = w.Write([]byte(`{"number":1,"html_url":"https://github.example/o/r/issues/1"}`))
}))
defer srv.Close()
t.Run("requests go through the caller's transport", func(t *testing.T) {
var seen atomic.Int32
rt := roundTripFunc(func(r *http.Request) (*http.Response, error) {
seen.Add(1)
require.Equal(t, "Bearer test-token", r.Header.Get("Authorization"))
return http.DefaultTransport.RoundTrip(r)
})
tracker, err := New(&Options{BaseURL: srv.URL + "/", Username: "u", Owner: "o", ProjectName: "r", Token: "test-token", HttpClient: clientWithTransport(rt)})
require.NoError(t, err)
_, err = tracker.CreateIssue(testEvent())
require.NoError(t, err)
require.Positive(t, seen.Load())
})
t.Run("a refusing transport stops the request", func(t *testing.T) {
before := serverHits.Load()
denied := errors.New("denied by address policy")
rt := roundTripFunc(func(*http.Request) (*http.Response, error) { return nil, denied })
tracker, err := New(&Options{BaseURL: srv.URL + "/", Username: "u", Owner: "o", ProjectName: "r", Token: "test-token", HttpClient: clientWithTransport(rt)})
require.NoError(t, err)
_, err = tracker.CreateIssue(testEvent())
require.ErrorContains(t, err, "denied by address policy")
require.Equal(t, before, serverHits.Load(), "the request reached the server")
})
}
func TestNewRefusesUntrustedCertificateFromInsecureCallerClient(t *testing.T) {
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusCreated)
_, _ = w.Write([]byte(`{"number":1}`))
}))
defer srv.Close()
insecure := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
tracker, err := New(&Options{BaseURL: srv.URL + "/", Username: "u", Owner: "o", ProjectName: "r", Token: "test-token", HttpClient: clientWithTransport(insecure)})
require.NoError(t, err)
_, err = tracker.CreateIssue(testEvent())
require.ErrorContains(t, err, "certificate")
}
type wrappingTransport struct{ base http.RoundTripper }
func (w *wrappingTransport) RoundTrip(r *http.Request) (*http.Response, error) {
return w.base.RoundTrip(r)
}
func (w *wrappingTransport) Unwrap() http.RoundTripper { return w.base }
// The scan client wraps its transport and, with a SOCKS proxy, does TLS in its
// own dialer with the scan TLS config, which skips verification.
func TestNewRefusesUntrustedCertificateFromInsecureTLSDialer(t *testing.T) {
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusCreated)
_, _ = w.Write([]byte(`{"number":1}`))
}))
defer srv.Close()
var dialed atomic.Int32
var dialer net.Dialer
base := &http.Transport{
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
dialed.Add(1)
return dialer.DialContext(ctx, network, addr)
},
DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
conn, err := dialer.DialContext(ctx, network, addr)
if err != nil {
return nil, err
}
return tls.Client(conn, &tls.Config{InsecureSkipVerify: true}), nil
},
}
tracker, err := New(&Options{BaseURL: srv.URL + "/", Username: "u", Owner: "o", ProjectName: "r", Token: "test-token", HttpClient: clientWithTransport(&wrappingTransport{base: base})})
require.NoError(t, err)
_, err = tracker.CreateIssue(testEvent())
require.ErrorContains(t, err, "certificate")
require.Positive(t, dialed.Load(), "the caller's dialer was not used")
}