init proxy after pool init

* Clustering performance improvements

* IsClusterable filters out beforehand, update test to mirror that

* inverse IsClusterable
This makes much more sense

* HashMap based clustering

* furthur improvements to clustering

---------

Co-authored-by: Tarun Koyalwar <[email protected]>

.github/workflows/release-binary.yml

@@ -21,7 +21,7 @@ jobs:
 
       - uses: goreleaser/goreleaser-action@v4
         with: 
-          args: "release --rm-dist"
+          args: "release --clean"
           version: latest
         env: 
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"

go.mod

@@ -20,12 +20,12 @@ require (
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.0.20
-	github.com/projectdiscovery/fastdialer v0.1.4-0.20240611222741-427bffb2ea13
-	github.com/projectdiscovery/hmap v0.0.46
+	github.com/projectdiscovery/fastdialer v0.1.5
+	github.com/projectdiscovery/hmap v0.0.47
 	github.com/projectdiscovery/interactsh v1.2.0
 	github.com/projectdiscovery/rawhttp v0.1.51
-	github.com/projectdiscovery/retryabledns v1.0.63
-	github.com/projectdiscovery/retryablehttp-go v1.0.64
+	github.com/projectdiscovery/retryabledns v1.0.65
+	github.com/projectdiscovery/retryablehttp-go v1.0.66
 	github.com/projectdiscovery/yamldoc-go v1.0.4
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.5.0
@@ -78,10 +78,10 @@ require (
 	github.com/mholt/archiver v3.1.1+incompatible
 	github.com/ory/dockertest/v3 v3.10.0
 	github.com/praetorian-inc/fingerprintx v1.1.9
-	github.com/projectdiscovery/dsl v0.1.2
+	github.com/projectdiscovery/dsl v0.1.3
 	github.com/projectdiscovery/fasttemplate v0.0.2
 	github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
-	github.com/projectdiscovery/goflags v0.1.54
+	github.com/projectdiscovery/goflags v0.1.56
 	github.com/projectdiscovery/gologger v1.1.12
 	github.com/projectdiscovery/gostruct v0.0.2
 	github.com/projectdiscovery/gozero v0.0.2
@@ -93,9 +93,9 @@ require (
 	github.com/projectdiscovery/sarif v0.0.1
 	github.com/projectdiscovery/tlsx v1.1.6
 	github.com/projectdiscovery/uncover v1.0.8
-	github.com/projectdiscovery/useragent v0.0.54
-	github.com/projectdiscovery/utils v0.1.4-0.20240611113448-0e2f2d33fe1c
-	github.com/projectdiscovery/wappalyzergo v0.1.4
+	github.com/projectdiscovery/useragent v0.0.56
+	github.com/projectdiscovery/utils v0.1.4
+	github.com/projectdiscovery/wappalyzergo v0.1.6
 	github.com/redis/go-redis/v9 v9.1.0
 	github.com/seh-msft/burpxml v1.0.1
 	github.com/stretchr/testify v1.9.0
@@ -124,7 +124,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.4 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/bahlo/generic-list-go v0.2.0 // indirect
-	github.com/bits-and-blooms/bitset v1.8.0 // indirect
+	github.com/bits-and-blooms/bitset v1.13.0 // indirect
 	github.com/bits-and-blooms/bloom/v3 v3.5.0 // indirect
 	github.com/buger/jsonparser v1.1.1 // indirect
 	github.com/bytedance/sonic v1.9.1 // indirect
@@ -143,9 +143,10 @@ require (
 	github.com/docker/cli v24.0.5+incompatible // indirect
 	github.com/docker/docker v24.0.9+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
-	github.com/fatih/color v1.15.0 // indirect
+	github.com/fatih/color v1.16.0 // indirect
 	github.com/free5gc/util v1.0.5-0.20230511064842-2e120956883b // indirect
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
+	github.com/gaissmai/bart v0.9.5 // indirect
 	github.com/geoffgarside/ber v1.1.0 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
 	github.com/gin-gonic/gin v1.9.1 // indirect
@@ -267,7 +268,7 @@ require (
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/gorilla/css v1.0.1 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
-	github.com/hashicorp/go-retryablehttp v0.7.2 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
 	github.com/hdm/jarm-go v0.0.7 // indirect
 	github.com/itchyny/timefmt-go v0.1.5 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
@@ -284,7 +285,7 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/projectdiscovery/blackrock v0.0.1 // indirect
-	github.com/projectdiscovery/networkpolicy v0.0.8
+	github.com/projectdiscovery/networkpolicy v0.0.9
 	github.com/rivo/uniseg v0.4.6 // indirect
 	github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect

go.sum

@@ -186,8 +186,9 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
-github.com/bits-and-blooms/bitset v1.8.0 h1:FD+XqgOZDUxxZ8hzoBFuV9+cGWY9CslN6d5MS5JVb4c=
 github.com/bits-and-blooms/bitset v1.8.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
+github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE=
+github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
 github.com/bits-and-blooms/bloom/v3 v3.5.0 h1:AKDvi1V3xJCmSR6QhcBfHbCN4Vf8FfxeWkMNQfmAGhY=
 github.com/bits-and-blooms/bloom/v3 v3.5.0/go.mod h1:Y8vrn7nk1tPIlmLtW2ZPV+W7StdVMor6bC1xgpjMZFs=
 github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw=
@@ -309,8 +310,8 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
-github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
+github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
+github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
@@ -326,6 +327,8 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
 github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
 github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
+github.com/gaissmai/bart v0.9.5 h1:vy+r4Px6bjZ+v2QYXAsg63vpz9IfzdW146A8Cn4GPIo=
+github.com/gaissmai/bart v0.9.5/go.mod h1:KHeYECXQiBjTzQz/om2tqn3sZF1J7hw9m6z41ftj3fg=
 github.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9w=
 github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
 github.com/getkin/kin-openapi v0.123.0 h1:zIik0mRwFNLyvtXK274Q6ut+dPh6nlxBp0x7mNrPhs8=
@@ -528,13 +531,13 @@ github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
 github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
-github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
-github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
+github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
+github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
-github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0=
-github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
+github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=
+github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
 github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
@@ -831,26 +834,26 @@ github.com/projectdiscovery/cdncheck v1.0.9 h1:BS15gzj9gb5AVSKqTDzPamfSgStu7nJQO
 github.com/projectdiscovery/cdncheck v1.0.9/go.mod h1:18SSl1w7rMj53CGeRIZTbDoa286a6xZIxGbaiEo4Fxs=
 github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPooH+DGMgoWq4=
 github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
-github.com/projectdiscovery/dsl v0.1.2 h1:RdWCEsUqyWsLIQ1k16NX5FiIVbPFvzx9BOlvzK7eeGw=
-github.com/projectdiscovery/dsl v0.1.2/go.mod h1:3zwogxGPnPkVUk5OfZQ9Y51sAWdazfQoVZFSPqeEUYA=
-github.com/projectdiscovery/fastdialer v0.1.4-0.20240611222741-427bffb2ea13 h1:mAehxpgxmvFrKcQ8taA3xQskp3gg6GayAgatTAn37bg=
-github.com/projectdiscovery/fastdialer v0.1.4-0.20240611222741-427bffb2ea13/go.mod h1:PcNscfCNrCnrkmvob90yfd+rNDMh6ZOIvRIl6HFPPXE=
+github.com/projectdiscovery/dsl v0.1.3 h1:Lxlgpx8lwq/CaBjVUmwF6KmkMDnYf9IVKlMsWCF72OE=
+github.com/projectdiscovery/dsl v0.1.3/go.mod h1:WAuQLD4KQ/94RASotE7cvk7X6kY7JQG9Fcqc3lRQfpI=
+github.com/projectdiscovery/fastdialer v0.1.5 h1:5rNJ8I+GVzDsUqRSVLUqGTm7LsshfNYmCh20lLn1aaI=
+github.com/projectdiscovery/fastdialer v0.1.5/go.mod h1:oVoMl9Y6da2timhszy0okyBvhbKDSk+zqFazGV4y6pU=
 github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
 github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
 github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
 github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE=
 github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG906Drtbpz4DwU5mhGIeOhRcktDH4cGQitGUMAsg=
 github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY=
-github.com/projectdiscovery/goflags v0.1.54 h1:xbaEBNbKqXyRu4154UlhsMAFmpaDYma4jJ9zUZGvXW8=
-github.com/projectdiscovery/goflags v0.1.54/go.mod h1:cfLzWWxgl/ft5cSHVJZjnvIzd4wX3A9Kz+W1pjeBZvc=
+github.com/projectdiscovery/goflags v0.1.56 h1:tJYiZN7s9Jk9DxfYOUiqOoybaIDlXyX4ZgT4B/06SyU=
+github.com/projectdiscovery/goflags v0.1.56/go.mod h1:DsGF0NPpM5hGg75N3MTSvWJ4MIT7HFEAOEeWZ074+Fg=
 github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A=
 github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw=
 github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
 github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
 github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E=
 github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8=
-github.com/projectdiscovery/hmap v0.0.46 h1:1fdZ0A8Zkg30SLl0gAbEqOYb4vB3gtokbQuYI+oXUNk=
-github.com/projectdiscovery/hmap v0.0.46/go.mod h1:QleSTt2HrokGNPpymUqb7PQ/2EEHEnaU5qKrrm51Va8=
+github.com/projectdiscovery/hmap v0.0.47 h1:NdakfvlFruevnOvehdllofg/hc0CQn2WQaLHaRcsGAk=
+github.com/projectdiscovery/hmap v0.0.47/go.mod h1:/9V6EnyTY33hAy71Quox6Ggu9ZkKp36tgepBh/NzNSg=
 github.com/projectdiscovery/httpx v1.6.3 h1:TSu3zEKOHGFfMsGeB0lZEFRl6Tq4nPuIgKDXi9NnYpg=
 github.com/projectdiscovery/httpx v1.6.3/go.mod h1:9FmsmkT71aTa4WnOyBIPzB8I9c0QAJ9PSVUqs2NkHRQ=
 github.com/projectdiscovery/interactsh v1.2.0 h1:Al6jHiR+Usl9egYJDLJaWNHOcH8Rugk8gWMasc8Cmw8=
@@ -863,18 +866,18 @@ github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQt
 github.com/projectdiscovery/mapcidr v1.1.34/go.mod h1:1+1R6OkKSAKtWDXE9RvxXtXPoajXTYX0eiEdkqlhQqQ=
 github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw1pfT6bg35NiN7yd1XKtJap5Nk6lMwQ0RNi8=
 github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc=
-github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7x7gHm028TJ5XwI8=
-github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI=
+github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+nek7TdtvEBwCxqk=
+github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M=
 github.com/projectdiscovery/ratelimit v0.0.45 h1:h28oF+hJ0CHcdBZozT1Go7ppWmzTxSXDKNNh2G1Ot9Q=
 github.com/projectdiscovery/ratelimit v0.0.45/go.mod h1:1vSJUseDS7SjNwIBi9wNRcgsMKNTLxy/GfdlLFVbgI4=
 github.com/projectdiscovery/rawhttp v0.1.51 h1:T4sskU2fsX9zNR7nyae0tD/0fliIs73KwBzyoJwBs10=
 github.com/projectdiscovery/rawhttp v0.1.51/go.mod h1:ejBaw3dCfJSfbtFv0ib6YfFSjQzt7ZdRVpQWJASxIUs=
 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
-github.com/projectdiscovery/retryabledns v1.0.63 h1:Ijt47ybwf+iIwul606NlPKPN+ouhOYPeA9sLHVgqLG4=
-github.com/projectdiscovery/retryabledns v1.0.63/go.mod h1:lTs48OYJnMFuuBzT+3z3PrZ58K0OUBgP7Y4o3ttBwb0=
-github.com/projectdiscovery/retryablehttp-go v1.0.64 h1:90VkUuWKm/JW5Tym0u+N1qseKe4gC5Rp048VJBLcxv8=
-github.com/projectdiscovery/retryablehttp-go v1.0.64/go.mod h1:fXVLBcwKwBrCVEgi3FrbLPtSTIczyJEKIlp1WD6VTI0=
+github.com/projectdiscovery/retryabledns v1.0.65 h1:wOtMuVkrcrGkzaT8Jixb3BtUcJODXLRz+z9EjftmFsQ=
+github.com/projectdiscovery/retryabledns v1.0.65/go.mod h1:lf3xRdFhGh55fWVlzK1oAaDPrsKliqH3gGvsFNAOkCk=
+github.com/projectdiscovery/retryablehttp-go v1.0.66 h1:dY+sGVf/O8CvKcSdTx0rxJoKG+HyOScz5T89TijZohU=
+github.com/projectdiscovery/retryablehttp-go v1.0.66/go.mod h1:3QJt06j4hEZVpJOr31u49I3mDnqbAzhcjJX8Yvn/xnY=
 github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
 github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
 github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
@@ -883,12 +886,12 @@ github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1Uzc
 github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo=
 github.com/projectdiscovery/uncover v1.0.8 h1:nE8bYJuwhqk0BEMRQRhNbjPGMy40A9gkiMk5xq6U4u0=
 github.com/projectdiscovery/uncover v1.0.8/go.mod h1:1SwsNqjwMkJEzJQ7lQr5AHIdYd+BQlyqhO5IwIYmIAM=
-github.com/projectdiscovery/useragent v0.0.54 h1:3gM4TnhLSspSM/XkI97YcUazXA3KOkQXEU7TIBly2Fc=
-github.com/projectdiscovery/useragent v0.0.54/go.mod h1:dKFkqUb44qx7MpwQDAbhL1t2IrFhcrgI5MlXUZ9E8dM=
-github.com/projectdiscovery/utils v0.1.4-0.20240611113448-0e2f2d33fe1c h1:0I/iRtu5nPYle1v8/R33pCLOrH5bziP5Bi0eZURxTQY=
-github.com/projectdiscovery/utils v0.1.4-0.20240611113448-0e2f2d33fe1c/go.mod h1:mXs6OOeG9l/dVchjB2PGvQO3+wuMiE14Y/kmHeKogoM=
-github.com/projectdiscovery/wappalyzergo v0.1.4 h1:Q0j5t/c74DlPKRvGYLZFv8npTA7MBW9uwdlsMBn/Yh4=
-github.com/projectdiscovery/wappalyzergo v0.1.4/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM=
+github.com/projectdiscovery/useragent v0.0.56 h1:KpUtOgq1vUerPrgRFZAIr1hF0lDxA2RF6YkT2PLwSoQ=
+github.com/projectdiscovery/useragent v0.0.56/go.mod h1:LAh9vudxXPA5JfNN3+1g/Unqvulq0BDkjxK7IrE+e/s=
+github.com/projectdiscovery/utils v0.1.4 h1:VSULN/RBEXbkgWuhrDey84ZDnSXtYMSev9p+QHXXDeI=
+github.com/projectdiscovery/utils v0.1.4/go.mod h1:mXs6OOeG9l/dVchjB2PGvQO3+wuMiE14Y/kmHeKogoM=
+github.com/projectdiscovery/wappalyzergo v0.1.6 h1:4iZDEnytsnAqjQgD5K6lmAzVx++KSQeMCDfomDtQM1M=
+github.com/projectdiscovery/wappalyzergo v0.1.6/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM=
 github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
 github.com/projectdiscovery/yamldoc-go v1.0.4/go.mod h1:8PIPRcUD55UbtQdcfFR1hpIGRWG0P7alClXNGt1TBik=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=

lib/sdk_private.go

@@ -108,14 +108,6 @@ func (e *NucleiEngine) init(ctx context.Context) error {
 		return err
 	}
 
-	if e.opts.ProxyInternal && types.ProxyURL != "" || types.ProxySocksURL != "" {
-		httpclient, err := httpclientpool.Get(e.opts, &httpclientpool.Configuration{})
-		if err != nil {
-			return err
-		}
-		e.httpClient = httpclient
-	}
-
 	e.parser = templates.NewParser()
 
 	if sharedInit == nil || protocolstate.ShouldInit() {
@@ -129,6 +121,14 @@ func (e *NucleiEngine) init(ctx context.Context) error {
 	e.applyRequiredDefaults(ctx)
 	var err error
 
+	if e.opts.ProxyInternal && types.ProxyURL != "" || types.ProxySocksURL != "" {
+		httpclient, err := httpclientpool.Get(e.opts, &httpclientpool.Configuration{})
+		if err != nil {
+			return err
+		}
+		e.httpClient = httpclient
+	}
+
 	// setup progressbar
 	if e.enableStats {
 		progressInstance, progressErr := progress.NewStatsTicker(e.opts.StatsInterval, e.enableStats, e.opts.StatsJSON, false, e.opts.MetricsPort)
@@ -231,7 +231,10 @@ func (e *NucleiEngine) init(ctx context.Context) error {
 	// and also upgrade templates to latest version if available
 	installer.NucleiSDKVersionCheck()
 
-	return e.processUpdateCheckResults()
+	if DefaultConfig.CanCheckForUpdates() {
+		return e.processUpdateCheckResults()
+	}
+	return nil
 }
 
 type syncOnce struct {

pkg/catalog/disk/catalog.go

@@ -11,17 +11,25 @@ import (
 // DiskCatalog is a template catalog helper implementation based on disk
 type DiskCatalog struct {
 	templatesDirectory string
-	templatesFS        fs.FS // TODO: Refactor to use this
+	templatesFS        fs.FS // Due to issues with how Go has implemented fs.FS, we'll have to also implement normal os operations, as well.  See: https://github.com/golang/go/issues/44279
 }
 
 // NewCatalog creates a new Catalog structure using provided input items
 // using disk based items
 func NewCatalog(directory string) *DiskCatalog {
 	catalog := &DiskCatalog{templatesDirectory: directory}
-	if directory != "" {
-		catalog.templatesFS = os.DirFS(directory)
-	} else {
-		catalog.templatesFS = os.DirFS(config.DefaultConfig.GetTemplateDir())
+	if directory == "" {
+		catalog.templatesDirectory = config.DefaultConfig.GetTemplateDir()
+	}
+	return catalog
+}
+
+// NewFSCatalog creates a new Catalog structure using provided input items
+// using the fs.FS as its filesystem.
+func NewFSCatalog(fs fs.FS, directory string) *DiskCatalog {
+	catalog := &DiskCatalog{
+		templatesDirectory: directory,
+		templatesFS:        fs,
 	}
 	return catalog
 }
@@ -29,11 +37,15 @@ func NewCatalog(directory string) *DiskCatalog {
 // OpenFile opens a file and returns an io.ReadCloser to the file.
 // It is used to read template and payload files based on catalog responses.
 func (d *DiskCatalog) OpenFile(filename string) (io.ReadCloser, error) {
-	file, err := os.Open(filename)
-	if err != nil {
-		if file, errx := os.Open(BackwardsCompatiblePaths(d.templatesDirectory, filename)); errx == nil {
-			return file, nil
+	if d.templatesFS == nil {
+		file, err := os.Open(filename)
+		if err != nil {
+			if file, errx := os.Open(BackwardsCompatiblePaths(d.templatesDirectory, filename)); errx == nil {
+				return file, nil
+			}
 		}
+		return file, err
 	}
-	return file, err
+
+	return d.templatesFS.Open(filename)
 }

pkg/catalog/disk/find.go

@@ -79,17 +79,21 @@ func (c *DiskCatalog) GetTemplatePath(target string) ([]string, error) {
 	}
 
 	// try to handle deprecated template paths
-	absPath := BackwardsCompatiblePaths(c.templatesDirectory, target)
-	if absPath != target && strings.TrimPrefix(absPath, c.templatesDirectory+string(filepath.Separator)) != target {
-		if config.DefaultConfig.LogAllEvents {
-			gologger.DefaultLogger.Print().Msgf("[%v] requested Template path %s is deprecated, please update to %s\n", aurora.Yellow("WRN").String(), target, absPath)
+	absPath := target
+	if c.templatesFS == nil {
+		absPath = BackwardsCompatiblePaths(c.templatesDirectory, target)
+		if absPath != target && strings.TrimPrefix(absPath, c.templatesDirectory+string(filepath.Separator)) != target {
+			if config.DefaultConfig.LogAllEvents {
+				gologger.DefaultLogger.Print().Msgf("[%v] requested Template path %s is deprecated, please update to %s\n", aurora.Yellow("WRN").String(), target, absPath)
+			}
+			deprecatedPathsCounter++
 		}
-		deprecatedPathsCounter++
-	}
 
-	absPath, err := c.convertPathToAbsolute(absPath)
-	if err != nil {
-		return nil, errors.Wrapf(err, "could not find template file")
+		var err error
+		absPath, err = c.convertPathToAbsolute(absPath)
+		if err != nil {
+			return nil, errors.Wrapf(err, "could not find template file")
+		}
 	}
 
 	// Template input is either a file or a directory
@@ -143,24 +147,60 @@ func (c *DiskCatalog) findGlobPathMatches(absPath string, processed map[string]s
 		if c.templatesDirectory == "" {
 			templateDir = "./"
 		}
-		matches, _ := fs.Glob(os.DirFS(filepath.Join(templateDir, "http")), inputGlob)
-		if len(matches) != 0 {
+
+		if c.templatesFS == nil {
+			matches, _ := fs.Glob(os.DirFS(filepath.Join(templateDir, "http")), inputGlob)
+			if len(matches) != 0 {
+				return matches
+			}
+
+			// condition to support network cve related globs
+			matches, _ = fs.Glob(os.DirFS(filepath.Join(templateDir, "network")), inputGlob)
+			return matches
+		} else {
+			sub, err := fs.Sub(c.templatesFS, filepath.Join(templateDir, "http"))
+			if err != nil {
+				return nil
+			}
+			matches, _ := fs.Glob(sub, inputGlob)
+			if len(matches) != 0 {
+				return matches
+			}
+
+			// condition to support network cve related globs
+			sub, err = fs.Sub(c.templatesFS, filepath.Join(templateDir, "network"))
+			if err != nil {
+				return nil
+			}
+			matches, _ = fs.Glob(sub, inputGlob)
 			return matches
 		}
-		// condition to support network cve related globs
-		matches, _ = fs.Glob(os.DirFS(filepath.Join(templateDir, "network")), inputGlob)
-		return matches
 	}
 
 	var matched []string
-	matches, err := fs.Glob(c.templatesFS, relPath)
-	if len(matches) != 0 {
-		matched = append(matched, matches...)
+	var matches []string
+	if c.templatesFS == nil {
+		var err error
+		matches, err = filepath.Glob(relPath)
+		if len(matches) != 0 {
+			matched = append(matched, matches...)
+		} else {
+			matched = append(matched, OldPathsResolver(relPath)...)
+		}
+		if err != nil && len(matched) == 0 {
+			return nil, errors.Errorf("wildcard found, but unable to glob: %s\n", err)
+		}
 	} else {
-		matched = append(matched, OldPathsResolver(relPath)...)
-	}
-	if err != nil && len(matched) == 0 {
-		return nil, errors.Errorf("wildcard found, but unable to glob: %s\n", err)
+		var err error
+		matches, err = fs.Glob(c.templatesFS, relPath)
+		if len(matches) != 0 {
+			matched = append(matched, matches...)
+		} else {
+			matched = append(matched, OldPathsResolver(relPath)...)
+		}
+		if err != nil && len(matched) == 0 {
+			return nil, errors.Errorf("wildcard found, but unable to glob: %s\n", err)
+		}
 	}
 	results := make([]string, 0, len(matches))
 	for _, match := range matches {
@@ -175,11 +215,23 @@ func (c *DiskCatalog) findGlobPathMatches(absPath string, processed map[string]s
 // findFileMatches finds if a path is an absolute file. If the path
 // is a file, it returns true otherwise false with no errors.
 func (c *DiskCatalog) findFileMatches(absPath string, processed map[string]struct{}) (match string, matched bool, err error) {
-	info, err := os.Stat(absPath)
+	if c.templatesFS != nil {
+		absPath = strings.TrimPrefix(absPath, "/")
+	}
+	var info fs.File
+	if c.templatesFS == nil {
+		info, err = os.Open(absPath)
+	} else {
+		info, err = c.templatesFS.Open(absPath)
+	}
 	if err != nil {
 		return "", false, err
 	}
-	if !info.Mode().IsRegular() {
+	stat, err := info.Stat()
+	if err != nil {
+		return "", false, err
+	}
+	if !stat.Mode().IsRegular() {
 		return "", false, nil
 	}
 	if _, ok := processed[absPath]; !ok {
@@ -192,22 +244,43 @@ func (c *DiskCatalog) findFileMatches(absPath string, processed map[string]struc
 // findDirectoryMatches finds matches for templates from a directory
 func (c *DiskCatalog) findDirectoryMatches(absPath string, processed map[string]struct{}) ([]string, error) {
 	var results []string
-	err := filepath.WalkDir(
-		absPath,
-		func(path string, d fs.DirEntry, err error) error {
-			// continue on errors
-			if err != nil {
-				return nil
-			}
-			if !d.IsDir() && config.GetTemplateFormatFromExt(path) != config.Unknown {
-				if _, ok := processed[path]; !ok {
-					results = append(results, path)
-					processed[path] = struct{}{}
+	var err error
+	if c.templatesFS == nil {
+		err = filepath.WalkDir(
+			absPath,
+			func(path string, d fs.DirEntry, err error) error {
+				// continue on errors
+				if err != nil {
+					return nil
 				}
-			}
-			return nil
-		},
-	)
+				if !d.IsDir() && config.GetTemplateFormatFromExt(path) != config.Unknown {
+					if _, ok := processed[path]; !ok {
+						results = append(results, path)
+						processed[path] = struct{}{}
+					}
+				}
+				return nil
+			},
+		)
+	} else {
+		err = fs.WalkDir(
+			c.templatesFS,
+			absPath,
+			func(path string, d fs.DirEntry, err error) error {
+				// continue on errors
+				if err != nil {
+					return nil
+				}
+				if !d.IsDir() && config.GetTemplateFormatFromExt(path) != config.Unknown {
+					if _, ok := processed[path]; !ok {
+						results = append(results, path)
+						processed[path] = struct{}{}
+					}
+				}
+				return nil
+			},
+		)
+	}
 	return results, err
 }
 

pkg/protocols/dns/cluster.go

@@ -1,27 +1,24 @@
 package dns
 
-// CanCluster returns true if the request can be clustered.
-//
-// This used by the clustering engine to decide whether two requests
-// are similar enough to be considered one and can be checked by
-// just adding the matcher/extractors for the request and the correct IDs.
-func (request *Request) CanCluster(other *Request) bool {
-	if len(request.Resolvers) > 0 || request.Trace || request.ID != "" {
-		return false
-	}
-	if request.Name != other.Name ||
-		request.class != other.class ||
-		request.Retries != other.Retries ||
-		request.question != other.question {
-		return false
-	}
+import (
+	"fmt"
+
+	"github.com/cespare/xxhash"
+)
+
+
+// TmplClusterKey generates a unique key for the request
+// to be used in the clustering process.
+func (request *Request) TmplClusterKey() uint64 {
+	recursion := ""
 	if request.Recursion != nil {
-		if other.Recursion == nil {
-			return false
-		}
-		if *request.Recursion != *other.Recursion {
-			return false
-		}
+		recursion = fmt.Sprintf("%t", *request.Recursion)
 	}
-	return true
+	inp := fmt.Sprintf("%s-%d-%d-%d-%s", request.Name, request.class, request.Retries, request.question, recursion)
+	return xxhash.Sum64String(inp)
+}
+
+// IsClusterable returns true if the request is eligible to be clustered.
+func (request *Request) IsClusterable() bool {
+	return !(len(request.Resolvers) > 0 || request.Trace || request.ID != "")
 }

pkg/protocols/http/cluster.go

@@ -1,30 +1,21 @@
 package http
 
 import (
-	sliceutil "github.com/projectdiscovery/utils/slice"
-	"golang.org/x/exp/maps"
+	"fmt"
+	"strings"
+
+	"github.com/cespare/xxhash"
+	"github.com/projectdiscovery/nuclei/v3/pkg/utils"
 )
 
-// CanCluster returns true if the request can be clustered.
-//
-// This used by the clustering engine to decide whether two requests
-// are similar enough to be considered one and can be checked by
-// just adding the matcher/extractors for the request and the correct IDs.
-func (request *Request) CanCluster(other *Request) bool {
-	if len(request.Payloads) > 0 || len(request.Fuzzing) > 0 || len(request.Raw) > 0 || len(request.Body) > 0 || request.Unsafe || request.NeedsRequestCondition() || request.Name != "" {
-		return false
-	}
-	if request.Method != other.Method ||
-		request.MaxRedirects != other.MaxRedirects ||
-		request.DisableCookie != other.DisableCookie ||
-		request.Redirects != other.Redirects {
-		return false
-	}
-	if !sliceutil.Equal(request.Path, other.Path) {
-		return false
-	}
-	if !maps.Equal(request.Headers, other.Headers) {
-		return false
-	}
-	return true
+// TmplClusterKey generates a unique key for the request
+// to be used in the clustering process.
+func (request *Request) TmplClusterKey() uint64 {
+	inp := fmt.Sprintf("%s-%d-%t-%t-%s-%d", request.Method.String(), request.MaxRedirects, request.DisableCookie, request.Redirects, strings.Join(request.Path, "-"), utils.MapHash(request.Headers))
+	return xxhash.Sum64String(inp)
+}
+
+// IsClusterable returns true if the request is eligible to be clustered.
+func (request *Request) IsClusterable() bool {
+	return !(len(request.Payloads) > 0 || len(request.Fuzzing) > 0 || len(request.Raw) > 0 || len(request.Body) > 0 || request.Unsafe || request.NeedsRequestCondition() || request.Name != "")
 }

pkg/protocols/http/cluster_test.go

@@ -8,8 +8,11 @@ import (
 
 func TestCanCluster(t *testing.T) {
 	req := &Request{Unsafe: true}
-	require.False(t, req.CanCluster(&Request{}), "could cluster unsafe request")
+	require.False(t, req.IsClusterable(), "could cluster unsafe request")
 
 	req = &Request{Path: []string{"{{BaseURL}}"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}}
-	require.True(t, req.CanCluster(&Request{Path: []string{"{{BaseURL}}"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}}), "could not cluster GET request")
+	newReq := &Request{Path: []string{"{{BaseURL}}"}, Method: HTTPMethodTypeHolder{MethodType: HTTPGet}}
+	require.True(t, req.IsClusterable(), "could not cluster GET request")
+	require.True(t, req.IsClusterable(), "could not cluster GET request")
+	require.Equal(t, req.TmplClusterKey(), newReq.TmplClusterKey(), "cluster keys should be equal")
 }

pkg/protocols/ssl/ssl.go

@@ -3,8 +3,10 @@ package ssl
 import (
 	"fmt"
 	"net"
+	"strings"
 	"time"
 
+	"github.com/cespare/xxhash"
 	"github.com/fatih/structs"
 	jsoniter "github.com/json-iterator/go"
 	"github.com/pkg/errors"
@@ -99,15 +101,15 @@ type Request struct {
 	options *protocols.ExecutorOptions
 }
 
-// CanCluster returns true if the request can be clustered.
-func (request *Request) CanCluster(other *Request) bool {
-	if len(request.CipherSuites) > 0 || request.MinVersion != "" || request.MaxVersion != "" {
-		return false
-	}
-	if request.Address != other.Address || request.ScanMode != other.ScanMode {
-		return false
-	}
-	return true
+// TmplClusterKey generates a unique key for the request
+// to be used in the clustering process.
+func (request *Request) TmplClusterKey() uint64 {
+	inp := fmt.Sprintf("%s-%s-%t-%t-%s", request.Address, request.ScanMode, request.TLSCiphersEnum, request.TLSVersionsEnum, strings.Join(request.TLSCipherTypes, ","))
+	return xxhash.Sum64String(inp)
+}
+
+func (request *Request) IsClusterable() bool {
+	return !(len(request.CipherSuites) > 0 || request.MinVersion != "" || request.MaxVersion != "")
 }
 
 // Compile compiles the request generators preparing any requests possible.

pkg/templates/cluster.go

@@ -14,7 +14,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/scan"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	cryptoutil "github.com/projectdiscovery/utils/crypto"
-	mapsutil "github.com/projectdiscovery/utils/maps"
 )
 
 // Cluster clusters a list of templates into a lesser number if possible based
@@ -42,91 +41,68 @@ import (
 // Finally, the engine creates a single executer with a clusteredexecuter for all templates
 // in a cluster.
 func Cluster(list []*Template) [][]*Template {
+	http := make(map[uint64][]*Template)
+	dns := make(map[uint64][]*Template)
+	ssl := make(map[uint64][]*Template)
+
 	final := [][]*Template{}
-	skip := mapsutil.NewSyncLockMap[string, struct{}]()
 
+	// Split up templates that might be clusterable
 	for _, template := range list {
-		key := template.Path
-
-		if skip.Has(key) {
-			continue
-		}
-
-		// We only cluster http, dns and ssl requests as of now.
-		// Take care of requests that can't be clustered first.
-		if len(template.RequestsHTTP) == 0 && len(template.RequestsDNS) == 0 && len(template.RequestsSSL) == 0 {
-			_ = skip.Set(key, struct{}{})
-			final = append(final, []*Template{template})
-			continue
-		}
-
 		// it is not possible to cluster flow and multiprotocol due to dependent execution
 		if template.Flow != "" || template.Options.IsMultiProtocol {
-			_ = skip.Set(key, struct{}{})
 			final = append(final, []*Template{template})
 			continue
 		}
 
-		_ = skip.Set(key, struct{}{})
-
-		var templateType types.ProtocolType
 		switch {
 		case len(template.RequestsDNS) == 1:
-			templateType = types.DNSProtocol
+			if template.RequestsDNS[0].IsClusterable() {
+				hash := template.RequestsDNS[0].TmplClusterKey()
+				if dns[hash] == nil {
+					dns[hash] = []*Template{}
+				}
+				dns[hash] = append(dns[hash], template)
+			} else {
+				final = append(final, []*Template{template})
+			}
+
 		case len(template.RequestsHTTP) == 1:
-			templateType = types.HTTPProtocol
+			if template.RequestsHTTP[0].IsClusterable() {
+				hash := template.RequestsHTTP[0].TmplClusterKey()
+				if http[hash] == nil {
+					http[hash] = []*Template{}
+				}
+				http[hash] = append(http[hash], template)
+			} else {
+				final = append(final, []*Template{template})
+			}
 		case len(template.RequestsSSL) == 1:
-			templateType = types.SSLProtocol
-		}
-
-		// Find any/all similar matching request that is identical to
-		// this one and cluster them together for http protocol only.
-		cluster := []*Template{}
-		for _, other := range list {
-			otherKey := other.Path
-
-			if skip.Has(otherKey) {
-				continue
-			}
-
-			// it is not possible to cluster flow and multiprotocol due to dependent execution
-			if other.Flow != "" || other.Options.IsMultiProtocol {
-				_ = skip.Set(otherKey, struct{}{})
-				final = append(final, []*Template{other})
-				continue
-			}
-
-			switch templateType {
-			case types.DNSProtocol:
-				if len(other.RequestsDNS) != 1 {
-					continue
-				} else if template.RequestsDNS[0].CanCluster(other.RequestsDNS[0]) {
-					_ = skip.Set(otherKey, struct{}{})
-					cluster = append(cluster, other)
-				}
-			case types.HTTPProtocol:
-				if len(other.RequestsHTTP) != 1 {
-					continue
-				} else if template.RequestsHTTP[0].CanCluster(other.RequestsHTTP[0]) {
-					_ = skip.Set(otherKey, struct{}{})
-					cluster = append(cluster, other)
-				}
-			case types.SSLProtocol:
-				if len(other.RequestsSSL) != 1 {
-					continue
-				} else if template.RequestsSSL[0].CanCluster(other.RequestsSSL[0]) {
-					_ = skip.Set(otherKey, struct{}{})
-					cluster = append(cluster, other)
+			if template.RequestsSSL[0].IsClusterable() {
+				hash := template.RequestsSSL[0].TmplClusterKey()
+				if ssl[hash] == nil {
+					ssl[hash] = []*Template{}
 				}
+				ssl[hash] = append(ssl[hash], template)
+			} else {
+				final = append(final, []*Template{template})
 			}
-		}
-		if len(cluster) > 0 {
-			cluster = append(cluster, template)
-			final = append(final, cluster)
-		} else {
+		default:
 			final = append(final, []*Template{template})
 		}
 	}
+
+	// add all clusterd templates
+	for _, templates := range http {
+		final = append(final, templates)
+	}
+	for _, templates := range dns {
+		final = append(final, templates)
+	}
+	for _, templates := range ssl {
+		final = append(final, templates)
+	}
+
 	return final
 }
 

pkg/templates/compile.go

@@ -342,6 +342,11 @@ func parseTemplate(data []byte, options protocols.ExecutorOptions) (*Template, e
 		return nil, errors.New("no template author field provided")
 	}
 
+	numberOfWorkflows := len(template.Workflows)
+	if numberOfWorkflows > 0 && numberOfWorkflows != template.Requests() {
+		return nil, errors.New("workflows cannot have other protocols")
+	}
+
 	// use default unknown severity
 	if len(template.Workflows) == 0 {
 		if template.Info.SeverityHolder.Severity == severity.Undefined {

pkg/templates/compile_test.go

@@ -197,3 +197,12 @@ func Test_WrongTemplate(t *testing.T) {
 	require.Nil(t, got, "could not parse template")
 	require.ErrorContains(t, err, "no requests defined ")
 }
+
+func TestWrongWorkflow(t *testing.T) {
+	setup()
+
+	filePath := "tests/workflow-invalid.yaml"
+	got, err := templates.Parse(filePath, nil, executerOpts)
+	require.Nil(t, got, "could not parse template")
+	require.ErrorContains(t, err, "workflows cannot have other protocols")
+}

pkg/templates/templates.go

@@ -177,8 +177,6 @@ func (template *Template) Type() types.ProtocolType {
 		return types.HeadlessProtocol
 	case len(template.RequestsNetwork) > 0:
 		return types.NetworkProtocol
-	case len(template.Workflow.Workflows) > 0:
-		return types.WorkflowProtocol
 	case len(template.RequestsSSL) > 0:
 		return types.SSLProtocol
 	case len(template.RequestsWebsocket) > 0:
@@ -189,6 +187,8 @@ func (template *Template) Type() types.ProtocolType {
 		return types.CodeProtocol
 	case len(template.RequestsJavascript) > 0:
 		return types.JavascriptProtocol
+	case len(template.Workflow.Workflows) > 0:
+		return types.WorkflowProtocol
 	default:
 		return types.InvalidProtocol
 	}

pkg/templates/tests/workflow-invalid.yaml

@@ -0,0 +1,18 @@
+id: workflow-example
+
+info:
+  name: Test Invalid Workflow Template
+  author: pdteam
+  severity: info
+
+http:
+  - raw:
+      - |
+        POST /re HTTP/1.1
+        Host: {{Hostname}}
+
+        {{code_response}}
+
+workflows:
+  - template: tests/match-1.yaml
+  - template: tests/match-1.yaml

pkg/utils/utils.go

@@ -2,12 +2,16 @@ package utils
 
 import (
 	"errors"
+	"fmt"
 	"io"
 	"net/url"
 	"strings"
 
+	"github.com/cespare/xxhash"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog"
 	"github.com/projectdiscovery/retryablehttp-go"
+	mapsutil "github.com/projectdiscovery/utils/maps"
+	"golang.org/x/exp/constraints"
 )
 
 func IsBlank(value string) bool {
@@ -57,3 +61,13 @@ func StringSliceContains(slice []string, item string) bool {
 	}
 	return false
 }
+
+// MapHash generates a hash for any give map
+func MapHash[K constraints.Ordered, V any](m map[K]V) uint64 {
+	keys := mapsutil.GetSortedKeys(m)
+	var sb strings.Builder
+	for _, k := range keys {
+		sb.WriteString(fmt.Sprintf("%v:%v\n", k, m[k]))
+	}
+	return xxhash.Sum64([]byte(sb.String()))
+}