Merge remote-tracking branch 'origin'

Bumps [github.com/projectdiscovery/dsl](https://github.com/projectdiscovery/dsl) from 0.0.56 to 0.0.57.
- [Release notes](https://github.com/projectdiscovery/dsl/releases)
- [Commits](https://github.com/projectdiscovery/dsl/compare/v0.0.56...v0.0.57)

---
updated-dependencies:
- dependency-name: github.com/projectdiscovery/dsl
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <[email protected]>

.github/workflows/build-test.yml

@@ -42,6 +42,7 @@ jobs:
       - name: Test
         env:
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          PDCP_API_KEY: "${{ secrets.PDCP_API_KEY }}"
         run: go test ./...
 
       - name: Integration Tests
@@ -49,6 +50,7 @@ jobs:
         env:
           GH_ACTION: true
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          PDCP_API_KEY: "${{ secrets.PDCP_API_KEY }}"
         run: |
           chmod +x run.sh
           bash run.sh ${{ matrix.os }}

.github/workflows/lint-test.yml

@@ -21,7 +21,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v3.6.0
+        uses: golangci/golangci-lint-action@v4.0.0
         with:
           version: latest
           args: --timeout 5m

.gitignore

@@ -39,5 +39,5 @@ integration_tests/fuzzplayground
 /nuclei-stats
 /nuclei-stats-*
 /scan-charts
-
+/**/debug-*
 

README.md

@@ -137,6 +137,7 @@ TEMPLATES:
    -nss, -no-strict-syntax                disable strict syntax check on templates
    -td, -template-display                 displays the templates content
    -tl                                    list all available templates
+   -tgl                                   list all available tags
    -sign                                  signs the templates with the private key defined in NUCLEI_SIGNATURE_PRIVATE_KEY env variable
    -code                                  enable loading code protocol-based templates
    -dut, -disable-unsigned-templates      disable running unsigned templates or templates with mismatched signature

cmd/integration-test/fuzz.go

@@ -18,6 +18,7 @@ const (
 
 var fuzzingTestCases = []TestCaseInfo{
 	{Path: "fuzz/fuzz-mode.yaml", TestCase: &fuzzModeOverride{}},
+	{Path: "fuzz/fuzz-multi-mode.yaml", TestCase: &fuzzMultipleMode{}},
 	{Path: "fuzz/fuzz-type.yaml", TestCase: &fuzzTypeOverride{}},
 	{Path: "fuzz/fuzz-query.yaml", TestCase: &httpFuzzQuery{}},
 	{Path: "fuzz/fuzz-headless.yaml", TestCase: &HeadlessFuzzingQuery{}},
@@ -174,3 +175,29 @@ func (h *HeadlessFuzzingQuery) Execute(filePath string) error {
 	}
 	return expectResultsCount(got, 2)
 }
+
+type fuzzMultipleMode struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *fuzzMultipleMode) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		xClientId := r.Header.Get("X-Client-Id")
+		xSecretId := r.Header.Get("X-Secret-Id")
+		if xClientId != "nuclei-v3" || xSecretId != "nuclei-v3" {
+			w.WriteHeader(http.StatusUnauthorized)
+			return
+		}
+		w.Header().Set("Content-Type", "text/html")
+		resp := fmt.Sprintf("<html><body><h1>This is multi-mode fuzzing test: %v <h1></body></html>", xClientId)
+		fmt.Fprint(w, resp)
+	})
+	ts := httptest.NewTLSServer(router)
+	defer ts.Close()
+
+	got, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL+"?url=https://scanme.sh", debug, "-jsonl", "-fuzz")
+	if err != nil {
+		return err
+	}
+	return expectResultsCount(got, 1)
+}

cmd/integration-test/integration-test.go

@@ -37,6 +37,7 @@ var (
 		"dns":             dnsTestCases,
 		"workflow":        workflowTestcases,
 		"loader":          loaderTestcases,
+		"profile-loader":  profileLoaderTestcases,
 		"websocket":       websocketTestCases,
 		"headless":        headlessTestcases,
 		"whois":           whoisTestCases,

cmd/integration-test/library.go

@@ -128,7 +128,7 @@ func executeNucleiAsLibrary(templatePath, templateURL string) ([]string, error)
 	}
 	store.Load()
 
-	_ = engine.Execute(store.Templates(), provider.NewSimpleInputProviderWithUrls(templateURL))
+	_ = engine.Execute(context.Background(), store.Templates(), provider.NewSimpleInputProviderWithUrls(templateURL))
 	engine.WorkPool().Wait() // Wait for the scan to finish
 
 	return results, nil

cmd/integration-test/profile-loader.go

@@ -0,0 +1,54 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
+	errorutil "github.com/projectdiscovery/utils/errors"
+)
+
+var profileLoaderTestcases = []TestCaseInfo{
+	{Path: "profile-loader/load-with-filename", TestCase: &profileLoaderByRelFile{}},
+	{Path: "profile-loader/load-with-id", TestCase: &profileLoaderById{}},
+	{Path: "profile-loader/basic.yml", TestCase: &customProfileLoader{}},
+}
+
+type profileLoaderByRelFile struct{}
+
+func (h *profileLoaderByRelFile) Execute(testName string) error {
+	results, err := testutils.RunNucleiWithArgsAndGetResults(false, "-tl", "-tp", "cloud.yml")
+	if err != nil {
+		return errorutil.NewWithErr(err).Msgf("failed to load template with id")
+	}
+	if len(results) <= 10 {
+		return fmt.Errorf("incorrect result: expected more results than %d, got %v", 10, len(results))
+	}
+	return nil
+}
+
+type profileLoaderById struct{}
+
+func (h *profileLoaderById) Execute(testName string) error {
+	results, err := testutils.RunNucleiWithArgsAndGetResults(false, "-tl", "-tp", "cloud")
+	if err != nil {
+		return errorutil.NewWithErr(err).Msgf("failed to load template with id")
+	}
+	if len(results) <= 10 {
+		return fmt.Errorf("incorrect result: expected more results than %d, got %v", 10, len(results))
+	}
+	return nil
+}
+
+// this profile with load kevs
+type customProfileLoader struct{}
+
+func (h *customProfileLoader) Execute(filepath string) error {
+	results, err := testutils.RunNucleiWithArgsAndGetResults(false, "-tl", "-tp", filepath)
+	if err != nil {
+		return errorutil.NewWithErr(err).Msgf("failed to load template with id")
+	}
+	if len(results) < 1 {
+		return fmt.Errorf("incorrect result: expected more results than %d, got %v", 1, len(results))
+	}
+	return nil
+}

cmd/integration-test/ssl.go

@@ -13,6 +13,7 @@ var sslTestcases = []TestCaseInfo{
 	{Path: "protocols/ssl/custom-cipher.yaml", TestCase: &sslCustomCipher{}},
 	{Path: "protocols/ssl/custom-version.yaml", TestCase: &sslCustomVersion{}},
 	{Path: "protocols/ssl/ssl-with-vars.yaml", TestCase: &sslWithVars{}},
+	{Path: "protocols/ssl/multi-req.yaml", TestCase: &sslMultiReq{}},
 }
 
 type sslBasic struct{}
@@ -118,3 +119,23 @@ func (h *sslWithVars) Execute(filePath string) error {
 
 	return expectResultsCount(results, 1)
 }
+
+type sslMultiReq struct{}
+
+func (h *sslMultiReq) Execute(filePath string) error {
+	ts := testutils.NewTCPServer(&tls.Config{}, defaultStaticPort, func(conn net.Conn) {
+		defer conn.Close()
+		data := make([]byte, 4)
+		if _, err := conn.Read(data); err != nil {
+			return
+		}
+	})
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-V")
+	if err != nil {
+		return err
+	}
+
+	return expectResultsCount(results, 2)
+}

cmd/nuclei/main.go

@@ -42,9 +42,10 @@ import (
 )
 
 var (
-	cfgFile    string
-	memProfile string // optional profile file path
-	options    = &types.Options{}
+	cfgFile         string
+	templateProfile string
+	memProfile      string // optional profile file path
+	options         = &types.Options{}
 )
 
 func main() {
@@ -140,7 +141,10 @@ func main() {
 			nucleiRunner.Close()
 			gologger.Info().Msgf("Creating resume file: %s\n", resumeFileName)
 			err := nucleiRunner.SaveResumeConfig(resumeFileName)
-			return errorutil.NewWithErr(err).Msgf("couldn't create crash resume file")
+			if err != nil {
+				return errorutil.NewWithErr(err).Msgf("couldn't create crash resume file")
+			}
+			return nil
 		})
 	}
 
@@ -199,7 +203,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 	*/
 
 	flagSet.CreateGroup("input", "Target",
-		flagSet.StringSliceVarP(&options.Targets, "target", "u", nil, "target URLs/hosts to scan", goflags.StringSliceOptions),
+		flagSet.StringSliceVarP(&options.Targets, "target", "u", nil, "target URLs/hosts to scan", goflags.CommaSeparatedStringSliceOptions),
 		flagSet.StringVarP(&options.TargetsFilePath, "list", "l", "", "path to file containing a list of target URLs/hosts to scan (one per line)"),
 		flagSet.StringSliceVarP(&options.ExcludeTargets, "exclude-hosts", "eh", nil, "hosts to exclude to scan from the input list (ip, cidr, hostname)", goflags.FileCommaSeparatedStringSliceOptions),
 		flagSet.StringVar(&options.Resume, "resume", "", "resume scan using resume.cfg (clustering will be disabled)"),
@@ -225,6 +229,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.BoolVarP(&options.NoStrictSyntax, "no-strict-syntax", "nss", false, "disable strict syntax check on templates"),
 		flagSet.BoolVarP(&options.TemplateDisplay, "template-display", "td", false, "displays the templates content"),
 		flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"),
+		flagSet.BoolVar(&options.TagList, "tgl", false, "list all available tags"),
 		flagSet.StringSliceVarConfigOnly(&options.RemoteTemplateDomainList, "remote-template-domain", []string{"cloud.projectdiscovery.io"}, "allowed domain list to load remote templates from"),
 		flagSet.BoolVar(&options.SignTemplates, "sign", false, "signs the templates with the private key defined in NUCLEI_SIGNATURE_PRIVATE_KEY env variable"),
 		flagSet.BoolVar(&options.EnableCodeTemplates, "code", false, "enable loading code protocol-based templates"),
@@ -270,6 +275,8 @@ on extensive configurability, massive extensibility and ease of use.`)
 
 	flagSet.CreateGroup("configs", "Configurations",
 		flagSet.StringVar(&cfgFile, "config", "", "path to the nuclei configuration file"),
+		flagSet.StringVarP(&templateProfile, "profile", "tp", "", "template profile config file to run"),
+		flagSet.BoolVarP(&options.ListTemplateProfiles, "profile-list", "tpl", false, "list community template profiles"),
 		flagSet.BoolVarP(&options.FollowRedirects, "follow-redirects", "fr", false, "enable following redirects for http templates"),
 		flagSet.BoolVarP(&options.FollowHostRedirects, "follow-host-redirects", "fhr", false, "follow redirects on the same host"),
 		flagSet.IntVarP(&options.MaxRedirects, "max-redirects", "mr", 10, "max number of redirects to follow for http templates"),
@@ -296,10 +303,12 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.StringVarP(&options.Interface, "interface", "i", "", "network interface to use for network scan"),
 		flagSet.StringVarP(&options.AttackType, "attack-type", "at", "", "type of payload combinations to perform (batteringram,pitchfork,clusterbomb)"),
 		flagSet.StringVarP(&options.SourceIP, "source-ip", "sip", "", "source ip address to use for network scan"),
-		flagSet.IntVarP(&options.ResponseReadSize, "response-size-read", "rsr", 10*1024*1024, "max response size to read in bytes"),
+		flagSet.IntVarP(&options.ResponseReadSize, "response-size-read", "rsr", 0, "max response size to read in bytes"),
 		flagSet.IntVarP(&options.ResponseSaveSize, "response-size-save", "rss", 1*1024*1024, "max response size to read in bytes"),
+		flagSet.DurationVarP(&options.ResponseReadTimeout, "response-read-timeout", "rrt", time.Duration(5*time.Second), "response read timeout in seconds"),
 		flagSet.CallbackVar(resetCallback, "reset", "reset removes all nuclei configuration and data files (including nuclei-templates)"),
 		flagSet.BoolVarP(&options.TlsImpersonate, "tls-impersonate", "tlsi", false, "enable experimental client hello (ja3) tls randomization"),
+		flagSet.StringVarP(&options.HttpApiEndpoint, "http-api-endpoint", "hae", "", "experimental http api endpoint"),
 	)
 
 	flagSet.CreateGroup("interactsh", "interactsh",
@@ -407,7 +416,8 @@ on extensive configurability, massive extensibility and ease of use.`)
 	flagSet.CreateGroup("cloud", "Cloud",
 		flagSet.DynamicVar(&pdcpauth, "auth", "true", "configure projectdiscovery cloud (pdcp) api key"),
 		flagSet.BoolVarP(&options.EnableCloudUpload, "cloud-upload", "cup", false, "upload scan results to pdcp dashboard"),
-		flagSet.StringVarP(&options.ScanID, "scan-id", "sid", "", "upload scan results to given scan id"),
+		flagSet.StringVarP(&options.ScanID, "scan-id", "sid", "", "upload scan results to existing scan id (optional)"),
+		flagSet.StringVarP(&options.ScanName, "scan-name", "sname", "", "scan name to set (optional)"),
 	)
 
 	flagSet.CreateGroup("Authentication", "Authentication",
@@ -495,6 +505,34 @@ Additional documentation is available at: https://docs.nuclei.sh/getting-started
 		config.DefaultConfig.SetTemplatesDir(options.NewTemplatesDirectory)
 	}
 
+	defaultProfilesPath := filepath.Join(config.DefaultConfig.GetTemplateDir(), "profiles")
+	if templateProfile != "" {
+		if filepath.Ext(templateProfile) == "" {
+			if tp := findProfilePathById(templateProfile, defaultProfilesPath); tp != "" {
+				templateProfile = tp
+			} else {
+				gologger.Fatal().Msgf("'%s' is not a profile-id or profile path", templateProfile)
+			}
+		}
+		if !filepath.IsAbs(templateProfile) {
+			if filepath.Dir(templateProfile) == "profiles" {
+				defaultProfilesPath = filepath.Join(config.DefaultConfig.GetTemplateDir())
+			}
+			currentDir, err := os.Getwd()
+			if err == nil && fileutil.FileExists(filepath.Join(currentDir, templateProfile)) {
+				templateProfile = filepath.Join(currentDir, templateProfile)
+			} else {
+				templateProfile = filepath.Join(defaultProfilesPath, templateProfile)
+			}
+		}
+		if !fileutil.FileExists(templateProfile) {
+			gologger.Fatal().Msgf("given template profile file '%s' does not exist", templateProfile)
+		}
+		if err := flagSet.MergeConfigFile(templateProfile); err != nil {
+			gologger.Fatal().Msgf("Could not read template profile: %s\n", err)
+		}
+	}
+
 	if len(options.SecretsFile) > 0 {
 		for _, secretFile := range options.SecretsFile {
 			if !fileutil.FileExists(secretFile) {
@@ -620,6 +658,27 @@ Note: Make sure you have backup of your custom nuclei-templates before proceedin
 	os.Exit(0)
 }
 
+func findProfilePathById(profileId, templatesDir string) string {
+	var profilePath string
+	err := filepath.WalkDir(templatesDir, func(iterItem string, d fs.DirEntry, err error) error {
+		ext := filepath.Ext(iterItem)
+		isYaml := ext == extensions.YAML || ext == extensions.YML
+		if err != nil || d.IsDir() || !isYaml {
+			// skip non yaml files
+			return nil
+		}
+		if strings.TrimSuffix(filepath.Base(iterItem), ext) == profileId {
+			profilePath = iterItem
+			return fmt.Errorf("FOUND")
+		}
+		return nil
+	})
+	if err != nil && err.Error() != "FOUND" {
+		gologger.Error().Msgf("%s\n", err)
+	}
+	return profilePath
+}
+
 func init() {
 	// print stacktrace of errors in debug mode
 	if strings.EqualFold(os.Getenv("DEBUG"), "true") {

go.mod

@@ -16,16 +16,16 @@ require (
 	github.com/json-iterator/go v1.1.12
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/logrusorgru/aurora v2.0.3+incompatible
-	github.com/miekg/dns v1.1.57
+	github.com/miekg/dns v1.1.59
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.0.20
-	github.com/projectdiscovery/fastdialer v0.0.67
-	github.com/projectdiscovery/hmap v0.0.41
+	github.com/projectdiscovery/fastdialer v0.1.0
+	github.com/projectdiscovery/hmap v0.0.43
 	github.com/projectdiscovery/interactsh v1.1.9
-	github.com/projectdiscovery/rawhttp v0.1.45
-	github.com/projectdiscovery/retryabledns v1.0.58
-	github.com/projectdiscovery/retryablehttp-go v1.0.55
+	github.com/projectdiscovery/rawhttp v0.1.49
+	github.com/projectdiscovery/retryabledns v1.0.60
+	github.com/projectdiscovery/retryablehttp-go v1.0.62
 	github.com/projectdiscovery/yamldoc-go v1.0.4
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.5.0
@@ -35,12 +35,12 @@ require (
 	github.com/spf13/cast v1.5.1
 	github.com/syndtr/goleveldb v1.0.0
 	github.com/valyala/fasttemplate v1.2.2
-	github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222
+	github.com/weppos/publicsuffix-go v0.30.2
 	github.com/xanzy/go-gitlab v0.84.0
 	go.uber.org/multierr v1.11.0
-	golang.org/x/net v0.24.0
-	golang.org/x/oauth2 v0.11.0
-	golang.org/x/text v0.14.0
+	golang.org/x/net v0.25.0
+	golang.org/x/oauth2 v0.18.0
+	golang.org/x/text v0.15.0
 	gopkg.in/yaml.v2 v2.4.0
 )
 
@@ -73,34 +73,35 @@ require (
 	github.com/h2non/filetype v1.1.3
 	github.com/labstack/echo/v4 v4.10.2
 	github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa
-	github.com/lib/pq v1.10.1
+	github.com/lib/pq v1.10.9
 	github.com/mattn/go-sqlite3 v1.14.22
 	github.com/mholt/archiver v3.1.1+incompatible
 	github.com/ory/dockertest/v3 v3.10.0
 	github.com/praetorian-inc/fingerprintx v1.1.9
-	github.com/projectdiscovery/dsl v0.0.52
+	github.com/projectdiscovery/dsl v0.0.57
 	github.com/projectdiscovery/fasttemplate v0.0.2
 	github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
-	github.com/projectdiscovery/goflags v0.1.48
+	github.com/projectdiscovery/goflags v0.1.53
 	github.com/projectdiscovery/gologger v1.1.12
 	github.com/projectdiscovery/gostruct v0.0.2
 	github.com/projectdiscovery/gozero v0.0.2
-	github.com/projectdiscovery/httpx v1.6.0
-	github.com/projectdiscovery/mapcidr v1.1.16
+	github.com/projectdiscovery/httpx v1.6.1
+	github.com/projectdiscovery/mapcidr v1.1.34
 	github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5
-	github.com/projectdiscovery/ratelimit v0.0.35
+	github.com/projectdiscovery/ratelimit v0.0.42
 	github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917
 	github.com/projectdiscovery/sarif v0.0.1
 	github.com/projectdiscovery/tlsx v1.1.6
-	github.com/projectdiscovery/uncover v1.0.7
-	github.com/projectdiscovery/useragent v0.0.40
-	github.com/projectdiscovery/utils v0.0.88
-	github.com/projectdiscovery/wappalyzergo v0.0.116
+	github.com/projectdiscovery/uncover v1.0.8
+	github.com/projectdiscovery/useragent v0.0.52
+	github.com/projectdiscovery/utils v0.1.0
+	github.com/projectdiscovery/wappalyzergo v0.1.0
 	github.com/redis/go-redis/v9 v9.1.0
 	github.com/seh-msft/burpxml v1.0.1
 	github.com/stretchr/testify v1.9.0
+	github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9
 	github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706
-	golang.org/x/term v0.19.0
+	golang.org/x/term v0.20.0
 	gopkg.in/yaml.v3 v3.0.1
 	moul.io/http2curl v1.0.0
 )
@@ -133,7 +134,7 @@ require (
 	github.com/cheggaaa/pb/v3 v3.1.4 // indirect
 	github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
 	github.com/cloudflare/cfssl v1.6.4 // indirect
-	github.com/cloudflare/circl v1.3.7 // indirect
+	github.com/cloudflare/circl v1.3.8 // indirect
 	github.com/containerd/continuity v0.4.2 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
 	github.com/davidmz/go-pageant v1.0.2 // indirect
@@ -142,8 +143,6 @@ require (
 	github.com/docker/cli v24.0.5+incompatible // indirect
 	github.com/docker/docker v24.0.9+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
-	github.com/eapache/channels v1.1.0 // indirect
-	github.com/eapache/queue v1.1.0 // indirect
 	github.com/fatih/color v1.15.0 // indirect
 	github.com/free5gc/util v1.0.5-0.20230511064842-2e120956883b // indirect
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
@@ -176,9 +175,10 @@ require (
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/kataras/jwt v0.1.10 // indirect
-	github.com/klauspost/compress v1.17.6 // indirect
+	github.com/klauspost/compress v1.17.8 // indirect
 	github.com/klauspost/pgzip v1.2.6 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/logrusorgru/aurora/v4 v4.0.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/mackerelio/go-osstat v0.2.4 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
@@ -203,17 +203,15 @@ require (
 	github.com/projectdiscovery/freeport v0.0.5 // indirect
 	github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb // indirect
 	github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect
-	github.com/projectdiscovery/stringsutil v0.0.2 // indirect
-	github.com/quic-go/quic-go v0.42.0 // indirect
-	github.com/refraction-networking/utls v1.6.1 // indirect
+	github.com/refraction-networking/utls v1.6.6 // indirect
 	github.com/sashabaranov/go-openai v1.15.3 // indirect
 	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/skeema/knownhosts v1.2.1 // indirect
 	github.com/tidwall/btree v1.7.0 // indirect
-	github.com/tidwall/buntdb v1.3.0 // indirect
-	github.com/tidwall/gjson v1.17.0 // indirect
+	github.com/tidwall/buntdb v1.3.1 // indirect
+	github.com/tidwall/gjson v1.17.1 // indirect
 	github.com/tidwall/grect v0.1.4 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
@@ -232,8 +230,9 @@ require (
 	github.com/yuin/goldmark-emoji v1.0.1 // indirect
 	github.com/zcalusic/sysinfo v1.0.2 // indirect
 	github.com/zeebo/blake3 v0.2.3 // indirect
+	go.uber.org/goleak v1.3.0 // indirect
 	golang.org/x/arch v0.3.0 // indirect
-	golang.org/x/sync v0.6.0 // indirect
+	golang.org/x/sync v0.7.0 // indirect
 	gopkg.in/djherbis/times.v1 v1.3.0 // indirect
 	mellium.im/sasl v0.3.1 // indirect
 )
@@ -261,7 +260,7 @@ require (
 	github.com/gobwas/pool v0.2.1 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/uuid v1.3.1 // indirect
@@ -290,7 +289,7 @@ require (
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/trivago/tgo v1.0.7
-	github.com/ulikunitz/xz v0.5.11 // indirect
+	github.com/ulikunitz/xz v0.5.12 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/yl2chen/cidranger v1.0.2 // indirect
 	github.com/ysmood/goob v0.4.0 // indirect
@@ -298,17 +297,17 @@ require (
 	github.com/ysmood/leakless v0.8.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect
-	github.com/zmap/zcrypto v0.0.0-20231219022726-a1f61fb1661c // indirect
-	go.etcd.io/bbolt v1.3.8 // indirect
+	github.com/zmap/zcrypto v0.0.0-20240512203510-0fef58d9a9db // indirect
+	go.etcd.io/bbolt v1.3.10 // indirect
 	go.uber.org/zap v1.25.0 // indirect
 	goftp.io/server/v2 v2.0.1 // indirect
-	golang.org/x/crypto v0.22.0 // indirect
-	golang.org/x/exp v0.0.0-20240119083558-1b970713d09a
-	golang.org/x/mod v0.14.0 // indirect
-	golang.org/x/sys v0.19.0 // indirect
+	golang.org/x/crypto v0.23.0 // indirect
+	golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842
+	golang.org/x/mod v0.17.0 // indirect
+	golang.org/x/sys v0.20.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
-	golang.org/x/tools v0.17.0
-	google.golang.org/appengine v1.6.7 // indirect
+	golang.org/x/tools v0.21.0
+	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect
 	gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect
@@ -349,5 +348,3 @@ require (
 
 // https://go.dev/ref/mod#go-mod-file-retract
 retract v3.2.0 // retract due to broken js protocol issue
-
-replace github.com/go-echarts/go-echarts/v2 => github.com/tarunKoyalwar/go-echarts/v2 v2.1.1

go.sum

@@ -235,8 +235,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
 github.com/cloudflare/cfssl v1.6.4 h1:NMOvfrEjFfC63K3SGXgAnFdsgkmiq4kATme5BfcqrO8=
 github.com/cloudflare/cfssl v1.6.4/go.mod h1:8b3CQMxfWPAeom3zBnGJ6sd+G1NkL5TXqmDXacb+1J0=
 github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I=
-github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU=
-github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA=
+github.com/cloudflare/circl v1.3.8 h1:j+V8jJt09PoeMFIu2uh5JUyEaIHTXVOHslFoLNAKqwI=
+github.com/cloudflare/circl v1.3.8/go.mod h1:PDRU+oXvdD7KCtgKxW95M5Z8BpSCJXQORiZFnBQS5QU=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ=
 github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4=
@@ -296,11 +296,8 @@ github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj6
 github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
-github.com/eapache/channels v1.1.0 h1:F1taHcn7/F0i8DYqKXJnyhJcVpp2kgFcNePxXtnyu4k=
-github.com/eapache/channels v1.1.0/go.mod h1:jMm2qB5Ubtg9zLd+inMZd2/NUvXgzmWXsDaLyQIGfH0=
 github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
-github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
 github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
@@ -317,6 +314,8 @@ github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
 github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
 github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
+github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
 github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
 github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
@@ -368,8 +367,6 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
-github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
-github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q=
@@ -395,8 +392,6 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG
 github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
 github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
-github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
 github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM=
 github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
 github.com/goburrow/cache v0.1.4 h1:As4KzO3hgmzPlnaMniZU9+VmoNYseUhuELbxy9mRBfw=
@@ -455,8 +450,9 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
 github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
@@ -631,8 +627,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI=
-github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
+github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
@@ -663,14 +659,16 @@ github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
 github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
 github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa h1:KQKuQDgA3DZX6C396lt3WDYB9Um1gLITLbvficVbqXk=
 github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa/go.mod h1:HbwNE4XGwjgtUELkvQaAOjWrpianHYZdQVNqSdYW3UM=
-github.com/lib/pq v1.10.1 h1:6VXZrLU0jHBYyAqrSPa+MgPfnSvTPuMgK+k0o5kVFWo=
-github.com/lib/pq v1.10.1/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
+github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/libdns/libdns v0.2.1 h1:Wu59T7wSHRgtA0cfxC+n1c/e+O3upJGWytknkmFEDis=
 github.com/libdns/libdns v0.2.1/go.mod h1:yQCXzk1lEZmmCPa857bnk4TsOiqYasqpyOEeSObbb40=
 github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
 github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
 github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
 github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
+github.com/logrusorgru/aurora/v4 v4.0.0 h1:sRjfPpun/63iADiSvGGjgA1cAYegEWMPCJdUpJYn9JA=
+github.com/logrusorgru/aurora/v4 v4.0.0/go.mod h1:lP0iIa2nrnT/qoFXcOZSrZQpJ1o6n2CUf/hyHi2Q4ZQ=
 github.com/lor00x/goldap v0.0.0-20180618054307-a546dffdd1a3 h1:wIONC+HMNRqmWBjuMxhatuSzHaljStc4gjDeKycxy0A=
 github.com/lor00x/goldap v0.0.0-20180618054307-a546dffdd1a3/go.mod h1:37YR9jabpiIxsb8X9VCIx8qFOjTDIIrIHHODa8C4gz0=
 github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
@@ -713,8 +711,8 @@ github.com/microcosm-cc/bluemonday v1.0.26 h1:xbqSvqzQMeEHCqMi64VAs4d8uy6Mequs3r
 github.com/microcosm-cc/bluemonday v1.0.26/go.mod h1:JyzOCs9gkyQyjs+6h10UEVSe02CGwkhd72Xdqh78TWs=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
-github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM=
-github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk=
+github.com/miekg/dns v1.1.59 h1:C9EXc/UToRwKLhK5wKU/I4QVsBUc8kE6MkHBkeypWZs=
+github.com/miekg/dns v1.1.59/go.mod h1:nZpewl5p6IvctfgrckopVx2OlSEHPRO/U4SYkRklrEk=
 github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg=
 github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 h1:yRZGarbxsRytL6EGgbqK2mCY+Lk5MWKQYKJT2gEglhc=
 github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM=
@@ -772,8 +770,6 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
 github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
-github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q=
-github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
 github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M=
@@ -834,64 +830,64 @@ github.com/projectdiscovery/cdncheck v1.0.9 h1:BS15gzj9gb5AVSKqTDzPamfSgStu7nJQO
 github.com/projectdiscovery/cdncheck v1.0.9/go.mod h1:18SSl1w7rMj53CGeRIZTbDoa286a6xZIxGbaiEo4Fxs=
 github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPooH+DGMgoWq4=
 github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
-github.com/projectdiscovery/dsl v0.0.52 h1:jvIvF+qN8+MbI1MHtWJJKfWqAZQlCExL3ob7SddQbZE=
-github.com/projectdiscovery/dsl v0.0.52/go.mod h1:xfcHwhy2HSaeGgh+1wqzOoCGm2XTdh5JzjBRBVHEMvI=
-github.com/projectdiscovery/fastdialer v0.0.67 h1:NvBpZUiLr9Ne9N+Lvi6FFiNNLWuhk5Bc1H+oE9J8C1E=
-github.com/projectdiscovery/fastdialer v0.0.67/go.mod h1:GhSAKnojJN8N9K0JNjLmwLCmEDsQ5cBAStqSCm/tm84=
+github.com/projectdiscovery/dsl v0.0.57 h1:BFjkhtJHzvxSNO6bJJ0UkA7tIwysgsyqWbuhn3FVQfI=
+github.com/projectdiscovery/dsl v0.0.57/go.mod h1:w9XL7AT/RDyg7yiFQN8eWWefXiYQt3mojPDdjyTdQ0k=
+github.com/projectdiscovery/fastdialer v0.1.0 h1:NiId7eXvYZjiBo83AnIZ6K8ZjzLJU8wGXdRLzuKcRrg=
+github.com/projectdiscovery/fastdialer v0.1.0/go.mod h1:jKOWzyHx+Q2sMiYst5HP6tWLQLDFPVgWBR72szn2h8w=
 github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
 github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
 github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
 github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE=
 github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG906Drtbpz4DwU5mhGIeOhRcktDH4cGQitGUMAsg=
 github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY=
-github.com/projectdiscovery/goflags v0.1.48 h1:iJgq+YuQOEm0V7BqlV/0wx7GaI2rmGiFzPkvL/4sCj0=
-github.com/projectdiscovery/goflags v0.1.48/go.mod h1:C9Qqyivehk5iYCf3VAC3+85y60Qbde1ZBMqe5wHhcVw=
+github.com/projectdiscovery/goflags v0.1.53 h1:wXNIMjvjBo0Hvo1183DQE2RyqlEbrhink/jYrFwRMwE=
+github.com/projectdiscovery/goflags v0.1.53/go.mod h1:yw4HnwPGKeCHx48tcl9ZqLrKeSArtUv2LK1dJP1i380=
 github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A=
 github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw=
 github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
 github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE=
 github.com/projectdiscovery/gozero v0.0.2 h1:8fJeaCjxL9tpm33uG/RsCQs6HGM/NE6eA3cjkilRQ+E=
 github.com/projectdiscovery/gozero v0.0.2/go.mod h1:d8bZvDWW07LWNYWrwjZ4OO1I0cpkfqaysyDfSs9ibK8=
-github.com/projectdiscovery/hmap v0.0.41 h1:8IgTyDce3/2JzcfPVA4H+XpBRFfETULx8td3BMdSYVE=
-github.com/projectdiscovery/hmap v0.0.41/go.mod h1:bCrai6x5Eijqm2U+jtcH0wZX5ZcaZhcvzoMGTZgLAf0=
-github.com/projectdiscovery/httpx v1.6.0 h1:6g4UoSQpsOyZgaK+SMLLnZIAU0eYyTxBUwVl+jtm0JQ=
-github.com/projectdiscovery/httpx v1.6.0/go.mod h1:dzMzOWKfeofaXcXzac3O+YmuY24P0CRnviKGxvol6MM=
+github.com/projectdiscovery/hmap v0.0.43 h1:9A/lGm/0uP9x2hP90dmK5vkc6KfGD0XYxDBAQ7lx/zg=
+github.com/projectdiscovery/hmap v0.0.43/go.mod h1:cMLXGlkLcPK0Y/aJVa+IZM9wdtkQ6HX4mHTXVC05Jbc=
+github.com/projectdiscovery/httpx v1.6.1 h1:aqKzoSydzKAJBV3Sxbqt44nTah4fNS9GkSD4Dy6Fzlc=
+github.com/projectdiscovery/httpx v1.6.1/go.mod h1:RvibGQbm2Bh2DtuBDVXHLfauFDLtp7opKHyk65sz0ss=
 github.com/projectdiscovery/interactsh v1.1.9 h1:b77SaSGrO+DtivmWwqGGY2dmNlQC3Zgmwlaj9L4Oqvc=
 github.com/projectdiscovery/interactsh v1.1.9/go.mod h1:0FRQXCildcTLq7Tsa4BVZAsFCXhpWs4C4quKWigXb5I=
 github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb h1:MGtI4oE12ruWv11ZlPXXd7hl/uAaQZrFvrIDYDeVMd8=
 github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb/go.mod h1:vmgC0DTFCfoCLp0RAfsfYTZZan0QMVs+cmTbH6blfjk=
 github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE=
 github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI=
-github.com/projectdiscovery/mapcidr v1.1.16 h1:rjj1w5D6hbTsUQXYClLcGdfBEy9bryclgi70t0vBggo=
-github.com/projectdiscovery/mapcidr v1.1.16/go.mod h1:rGqpBhStdwOQ2uS62QM9qPsybwMwIhT7CTd2bxoHs8Q=
+github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQtls1wmXN/kM=
+github.com/projectdiscovery/mapcidr v1.1.34/go.mod h1:1+1R6OkKSAKtWDXE9RvxXtXPoajXTYX0eiEdkqlhQqQ=
 github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw1pfT6bg35NiN7yd1XKtJap5Nk6lMwQ0RNi8=
 github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc=
 github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7x7gHm028TJ5XwI8=
 github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI=
-github.com/projectdiscovery/ratelimit v0.0.35 h1:epEzFATOcXZ4tssV4Hax5Op9lrbUnQMEGMV5PoUpTKc=
-github.com/projectdiscovery/ratelimit v0.0.35/go.mod h1:mPqa8UpV5I7eAN5/ZcsjLiXMhjtVvZRrHtpBRsTPuyA=
-github.com/projectdiscovery/rawhttp v0.1.45 h1:5jssUybhRz2ljI3bW4kUO9MTvZ37ArG8pISYtgqzwBg=
-github.com/projectdiscovery/rawhttp v0.1.45/go.mod h1:oLWuZkyKi4rddPoLEN3TeAqyLvFNfr3dxmzV2kwpPvs=
+github.com/projectdiscovery/ratelimit v0.0.42 h1:110tBLaGUgyPK0DjLmMBQaaqW4tkmqsFidr/t6tyOy4=
+github.com/projectdiscovery/ratelimit v0.0.42/go.mod h1:ruhLiZ5liukSpG07p6eHTCPJUmTwOhxDSxQPulvC3/Y=
+github.com/projectdiscovery/rawhttp v0.1.49 h1:OPP9R/UZx/GFrcPRUs9fOS1dLwhg+2o7p3dByzkIhWM=
+github.com/projectdiscovery/rawhttp v0.1.49/go.mod h1:aaAaMsdzHPfw4yU3nbeP7NI3vy1ZjgoXw7l+m4Tnt94=
 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
-github.com/projectdiscovery/retryabledns v1.0.58 h1:ut1FSB9+GZ6zQIlKJFLqIz2RZs81EmkbsHTuIrWfYLE=
-github.com/projectdiscovery/retryabledns v1.0.58/go.mod h1:RobmKoNBgngAVE4H9REQtaLP1pa4TCyypHy1MWHT1mY=
-github.com/projectdiscovery/retryablehttp-go v1.0.55 h1:ADgugnl9jKkNXn5m/Zd8TGPq1P7GplYlqUNKm/qTmls=
-github.com/projectdiscovery/retryablehttp-go v1.0.55/go.mod h1:Kpvh4ruFPOEPYaYxgbFmlvBJr4lJKqpcbGvx1j0r/Ng=
+github.com/projectdiscovery/retryabledns v1.0.60 h1:f3TPeLnaJKrl9CbfsTfFTFnJY+UnyArH3mxxUUyR5ZQ=
+github.com/projectdiscovery/retryabledns v1.0.60/go.mod h1:T4Su40Wa9lVtRNMfMDFJi00g2T3FbTfwnKKkYON0WgU=
+github.com/projectdiscovery/retryablehttp-go v1.0.62 h1:ZY09NYRatoELHaPUcdRdWzHIJExjEqceHAfe7u/e9ZY=
+github.com/projectdiscovery/retryablehttp-go v1.0.62/go.mod h1:yhRy9Q3z0CuNUHIIPrveaubjVEwF4arKWT/5SQurQEE=
 github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
 github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
 github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
 github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0=
 github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1UzcNsC77A=
 github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo=
-github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7siFy9sj0A=
-github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE=
-github.com/projectdiscovery/useragent v0.0.40 h1:1LUhReSGPkhqsM5n40OOC9dIoNqMGs1dyGFJcOmg2Fo=
-github.com/projectdiscovery/useragent v0.0.40/go.mod h1:EvK1x3s948Gtqb/XOahXcauyejCL/rSgy5d1IAvsKT4=
-github.com/projectdiscovery/utils v0.0.88 h1:oYfCXM+8VHNLyH/H6cOibkuDUwHUAOBAMRNPFX6NPrs=
-github.com/projectdiscovery/utils v0.0.88/go.mod h1:lAWzFdGXtJRPKdhUu1Z46d8B8JbASTk1Z69WY6H/3kA=
-github.com/projectdiscovery/wappalyzergo v0.0.116 h1:xy+mBpwbYo/0PSzmJOQ/RXHomEh0D3nDBcbCxsW69m8=
-github.com/projectdiscovery/wappalyzergo v0.0.116/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8=
+github.com/projectdiscovery/uncover v1.0.8 h1:nE8bYJuwhqk0BEMRQRhNbjPGMy40A9gkiMk5xq6U4u0=
+github.com/projectdiscovery/uncover v1.0.8/go.mod h1:1SwsNqjwMkJEzJQ7lQr5AHIdYd+BQlyqhO5IwIYmIAM=
+github.com/projectdiscovery/useragent v0.0.52 h1:9SUPH0Epo3DJfB6PCDgETfMaD6nZ08sFvfgXTmPUAsU=
+github.com/projectdiscovery/useragent v0.0.52/go.mod h1:PUXHgShvaD8p3bihy1mY8tuBDhdk3M0yy4Z10Ajg2yQ=
+github.com/projectdiscovery/utils v0.1.0 h1:r7Z/s2CBktJ0bnSN410lzOhD8S/0IxmzmFxkQudYKps=
+github.com/projectdiscovery/utils v0.1.0/go.mod h1:RaBdJLTKF5FKZ/RtMeccqFBtpsSjaggVw6/oPTpDD40=
+github.com/projectdiscovery/wappalyzergo v0.1.0 h1:ZagOIKemBsNfCDRQeWavWEXtEjP8UiziuoRoPTwDAJ0=
+github.com/projectdiscovery/wappalyzergo v0.1.0/go.mod h1:wBYGKmA5BQp/NWsAy1q/jSH8N1LHWQ/LV26DuR+KzPM=
 github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
 github.com/projectdiscovery/yamldoc-go v1.0.4/go.mod h1:8PIPRcUD55UbtQdcfFR1hpIGRWG0P7alClXNGt1TBik=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
@@ -926,13 +922,11 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
 github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
-github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM=
-github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/redis/go-redis/v9 v9.1.0 h1:137FnGdk+EQdCbye1FW+qOEcY5S+SpY9T0NiuqvtfMY=
 github.com/redis/go-redis/v9 v9.1.0/go.mod h1:urWj3He21Dj5k4TK1y59xH8Uj6ATueP8AH1cY3lZl4c=
-github.com/refraction-networking/utls v1.6.1 h1:n1JG5karzdGWsI6iZmGrOv3SNzR4c+4M8J6KWGsk3lA=
-github.com/refraction-networking/utls v1.6.1/go.mod h1:+EbcQOvQvXoFV9AEKbuGlljt1doLRKAVY1jJHe9EtDo=
+github.com/refraction-networking/utls v1.6.6 h1:igFsYBUJPYM8Rno9xUuDoM5GQrVEqY4llzEXOkL43Ig=
+github.com/refraction-networking/utls v1.6.6/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0=
 github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
 github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
 github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
@@ -948,7 +942,6 @@ github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
-github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
 github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA=
 github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
@@ -1015,17 +1008,17 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
-github.com/tarunKoyalwar/go-echarts/v2 v2.1.1 h1:5fsXGPmK+i18J8cDgxy7AJkiXWBARpVTb0Gbv+bAzPo=
-github.com/tarunKoyalwar/go-echarts/v2 v2.1.1/go.mod h1:VEeyPT5Odx/UHeuxtIAHGu2+87MWGA5OBaZ120NFi/w=
+github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9 h1:GXIyLuIJ5Qk46lI8WJ83qHBZKUI3zhmMmuoY9HICUIQ=
+github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9/go.mod h1:uQdBQGrE1fZ2EyOs0pLcCDd1bBV4rSThieuIIGhXZ50=
 github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI=
 github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8=
 github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI=
 github.com/tidwall/btree v1.7.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY=
-github.com/tidwall/buntdb v1.3.0 h1:gdhWO+/YwoB2qZMeAU9JcWWsHSYU3OvcieYgFRS0zwA=
-github.com/tidwall/buntdb v1.3.0/go.mod h1:lZZrZUWzlyDJKlLQ6DKAy53LnG7m5kHyrEHvvcDmBpU=
+github.com/tidwall/buntdb v1.3.1 h1:HKoDF01/aBhl9RjYtbaLnvX9/OuenwvQiC3OP1CcL4o=
+github.com/tidwall/buntdb v1.3.1/go.mod h1:lZZrZUWzlyDJKlLQ6DKAy53LnG7m5kHyrEHvvcDmBpU=
 github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM=
-github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U=
+github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/grect v0.1.4 h1:dA3oIgNgWdSspFzn1kS4S/RDpZFLrIxAZOdJKjYapOg=
 github.com/tidwall/grect v0.1.4/go.mod h1:9FBsaYRaR0Tcy4UwefBX/UDcDcDy9V5jUcxHzv2jd5Q=
 github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8=
@@ -1056,8 +1049,8 @@ github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4d
 github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
 github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
-github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
-github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
+github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc=
+github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
@@ -1068,8 +1061,8 @@ github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+
 github.com/weppos/publicsuffix-go v0.12.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k=
 github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k=
 github.com/weppos/publicsuffix-go v0.30.0/go.mod h1:kBi8zwYnR0zrbm8RcuN1o9Fzgpnnn+btVN8uWPMyXAY=
-github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222 h1:h2JizvZl9aIj6za9S5AyrkU+OzIS4CetQthH/ejO+lg=
-github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222/go.mod h1:s41lQh6dIsDWIC1OWh7ChWJXLH0zkJ9KHZVqA7vHyuQ=
+github.com/weppos/publicsuffix-go v0.30.2 h1:Np18yzfMR90jNampWFs7iSh2sw/qCZkhL41/ffyihCU=
+github.com/weppos/publicsuffix-go v0.30.2/go.mod h1:/hGscit36Yt+wammfBBwdMdxBT8btsTt6KvwO9OvMyM=
 github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220927085643-dc0d00c92642/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE=
 github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
 github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
@@ -1132,15 +1125,15 @@ github.com/zmap/zcertificate v0.0.1/go.mod h1:q0dlN54Jm4NVSSuzisusQY0hqDWvu92C+T
 github.com/zmap/zcrypto v0.0.0-20201128221613-3719af1573cf/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ=
 github.com/zmap/zcrypto v0.0.0-20201211161100-e54a5822fb7e/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ=
 github.com/zmap/zcrypto v0.0.0-20230310154051-c8b263fd8300/go.mod h1:mOd4yUMgn2fe2nV9KXsa9AyQBFZGzygVPovsZR+Rl5w=
-github.com/zmap/zcrypto v0.0.0-20231219022726-a1f61fb1661c h1:U1b4THKcgOpJ+kILupuznNwPiURtwVW3e9alJvji9+s=
-github.com/zmap/zcrypto v0.0.0-20231219022726-a1f61fb1661c/go.mod h1:GSDpFDD4TASObxvfZfvpZZ3OWHIUHMlhVWlkOe4ewVk=
+github.com/zmap/zcrypto v0.0.0-20240512203510-0fef58d9a9db h1:IfONOhyZlf4qPt3ENPU+27mBbPjzTQ+swKpj7MJva9I=
+github.com/zmap/zcrypto v0.0.0-20240512203510-0fef58d9a9db/go.mod h1:mo/07mo6reDaiz6BzveCuYBWb1d+aX8Pf8Nh+Q57y2g=
 github.com/zmap/zflags v1.4.0-beta.1.0.20200204220219-9d95409821b6/go.mod h1:HXDUD+uue8yeLHr0eXx1lvY6CvMiHbTKw5nGmA9OUoo=
 github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706 h1:LaMyYFWQA7kh3ovPfAaFDTKlJu3JGng8khruOtsBVnE=
 github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706/go.mod h1:re2kMcs84XHb8Xl6RInt0emoKCuphfmfjHYuteviLHQ=
 github.com/zmap/zlint/v3 v3.0.0/go.mod h1:paGwFySdHIBEMJ61YjoqT4h7Ge+fdYG4sUQhnTb1lJ8=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA=
-go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
+go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0=
+go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ=
 go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
 go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
@@ -1151,8 +1144,8 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
-go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
-go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
@@ -1188,10 +1181,10 @@ golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
-golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
-golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
-golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
-golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
+golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
+golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1202,8 +1195,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA=
-golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08=
+golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM=
+golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -1226,8 +1219,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
-golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
+golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1283,9 +1276,10 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
 golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
-golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
-golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
+golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1295,8 +1289,8 @@ golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ
 golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
 golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
 golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
-golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU=
-golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk=
+golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI=
+golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -1310,8 +1304,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
-golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1383,12 +1377,11 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
-golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -1398,10 +1391,10 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
 golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
-golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
-golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
-golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
-golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
+golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
+golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
+golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1415,9 +1408,10 @@ golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1475,8 +1469,8 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc=
-golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
+golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw=
+golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1505,8 +1499,9 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
+google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=

integration_tests/fuzz/fuzz-multi-mode.yaml

@@ -0,0 +1,27 @@
+id: fuzz-multi-mode-test
+
+info:
+  name: multi-mode fuzzing test
+  author: pdteam
+  severity: info
+
+http:
+  - payloads:
+      inject:
+        - nuclei-v1
+        - nuclei-v2
+        - nuclei-v3
+
+    fuzzing:
+      - part: header
+        type: replace
+        mode: multiple
+        fuzz:
+          X-Client-Id: "{{inject}}"
+          X-Secret-Id: "{{inject}}"
+          
+    matchers-condition: or
+    matchers:
+      - type: word
+        words:
+          - "nuclei-v3"

integration_tests/profile-loader/basic.yml

@@ -0,0 +1,2 @@
+tags:
+  - kev

integration_tests/protocols/ssl/multi-req.yaml

@@ -0,0 +1,34 @@
+id: multi-req
+
+info:
+  name: Multi-Request
+  author: pdteam
+  severity: info
+
+ssl:
+  - address: "{{Host}}:{{Port}}"
+    min_version: ssl30
+    max_version: ssl30
+
+    extractors:
+      - type: json
+        json:
+          - " .tls_version"
+
+  - address: "{{Host}}:{{Port}}"
+    min_version: tls10
+    max_version: tls10
+
+    extractors:
+      - type: json
+        json:
+          - " .tls_version"
+
+  - address: "{{Host}}:{{Port}}"
+    min_version: tls11
+    max_version: tls11
+
+    extractors:
+      - type: json
+        json:
+          - " .tls_version"

internal/httpapi/apiendpoint.go

@@ -0,0 +1,112 @@
+package httpapi
+
+import (
+	"encoding/json"
+	"net/http"
+	"time"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/js/compiler"
+	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+)
+
+type Concurrency struct {
+	BulkSize              int    `json:"bulk_size"`
+	Threads               int    `json:"threads"`
+	RateLimit             int    `json:"rate_limit"`
+	RateLimitDuration     string `json:"rate_limit_duration"`
+	PayloadConcurrency    int    `json:"payload_concurrency"`
+	ProbeConcurrency      int    `json:"probe_concurrency"`
+	JavascriptConcurrency int    `json:"javascript_concurrency"`
+}
+
+// Server represents the HTTP server that handles the concurrency settings endpoints.
+type Server struct {
+	addr   string
+	config *types.Options
+}
+
+// New creates a new instance of Server.
+func New(addr string, config *types.Options) *Server {
+	return &Server{
+		addr:   addr,
+		config: config,
+	}
+}
+
+// Start initializes the server and its routes, then starts listening on the specified address.
+func (s *Server) Start() error {
+	http.HandleFunc("/api/concurrency", s.handleConcurrency)
+	if err := http.ListenAndServe(s.addr, nil); err != nil {
+		return err
+	}
+	return nil
+}
+
+// handleConcurrency routes the request based on its method to the appropriate handler.
+func (s *Server) handleConcurrency(w http.ResponseWriter, r *http.Request) {
+	switch r.Method {
+	case http.MethodGet:
+		s.getSettings(w, r)
+	case http.MethodPut:
+		s.updateSettings(w, r)
+	default:
+		http.Error(w, "Unsupported HTTP method", http.StatusMethodNotAllowed)
+	}
+}
+
+// GetSettings handles GET requests and returns the current concurrency settings
+func (s *Server) getSettings(w http.ResponseWriter, _ *http.Request) {
+	concurrencySettings := Concurrency{
+		BulkSize:              s.config.BulkSize,
+		Threads:               s.config.TemplateThreads,
+		RateLimit:             s.config.RateLimit,
+		RateLimitDuration:     s.config.RateLimitDuration.String(),
+		PayloadConcurrency:    s.config.PayloadConcurrency,
+		ProbeConcurrency:      s.config.ProbeConcurrency,
+		JavascriptConcurrency: compiler.PoolingJsVmConcurrency,
+	}
+	w.Header().Set("Content-Type", "application/json")
+	if err := json.NewEncoder(w).Encode(concurrencySettings); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+// UpdateSettings handles PUT requests to update the concurrency settings
+func (s *Server) updateSettings(w http.ResponseWriter, r *http.Request) {
+	var newSettings Concurrency
+	if err := json.NewDecoder(r.Body).Decode(&newSettings); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	if newSettings.RateLimitDuration != "" {
+		if duration, err := time.ParseDuration(newSettings.RateLimitDuration); err == nil {
+			s.config.RateLimitDuration = duration
+		} else {
+			http.Error(w, "Invalid duration format", http.StatusBadRequest)
+			return
+		}
+	}
+	if newSettings.BulkSize > 0 {
+		s.config.BulkSize = newSettings.BulkSize
+	}
+	if newSettings.Threads > 0 {
+		s.config.TemplateThreads = newSettings.Threads
+	}
+	if newSettings.RateLimit > 0 {
+		s.config.RateLimit = newSettings.RateLimit
+	}
+	if newSettings.PayloadConcurrency > 0 {
+		s.config.PayloadConcurrency = newSettings.PayloadConcurrency
+	}
+	if newSettings.ProbeConcurrency > 0 {
+		s.config.ProbeConcurrency = newSettings.ProbeConcurrency
+	}
+	if newSettings.JavascriptConcurrency > 0 {
+		compiler.PoolingJsVmConcurrency = newSettings.JavascriptConcurrency
+		s.config.JsConcurrency = newSettings.JavascriptConcurrency // no-op on speed change
+	}
+
+	w.WriteHeader(http.StatusOK)
+}

internal/pdcp/writer.go

@@ -9,6 +9,7 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"regexp"
 	"sync/atomic"
 	"time"
 
@@ -27,9 +28,13 @@ const (
 	appendEndpoint = "/v1/scans/%s/import"
 	flushTimer     = time.Duration(1) * time.Minute
 	MaxChunkSize   = 1024 * 1024 * 4 // 4 MB
+	xidRe          = `^[a-z0-9]{20}$`
 )
 
-var _ output.Writer = &UploadWriter{}
+var (
+	xidRegex               = regexp.MustCompile(xidRe)
+	_        output.Writer = &UploadWriter{}
+)
 
 // UploadWriter is a writer that uploads its output to pdcp
 // server to enable web dashboard and more
@@ -41,6 +46,7 @@ type UploadWriter struct {
 	cancel    context.CancelFunc
 	done      chan struct{}
 	scanID    string
+	scanName  string
 	counter   atomic.Int32
 }
 
@@ -86,8 +92,17 @@ func NewUploadWriter(ctx context.Context, creds *pdcpauth.PDCPCredentials) (*Upl
 }
 
 // SetScanID sets the scan id for the upload writer
-func (u *UploadWriter) SetScanID(id string) {
+func (u *UploadWriter) SetScanID(id string) error {
+	if !xidRegex.MatchString(id) {
+		return fmt.Errorf("invalid scan id provided")
+	}
 	u.scanID = id
+	return nil
+}
+
+// SetScanName sets the scan name for the upload writer
+func (u *UploadWriter) SetScanName(name string) {
+	u.scanName = name
 }
 
 func (u *UploadWriter) autoCommit(ctx context.Context, r *io.PipeReader) {
@@ -220,7 +235,13 @@ func (u *UploadWriter) getRequest(bin []byte) (*retryablehttp.Request, error) {
 		return nil, errorutil.NewWithErr(err).Msgf("could not create cloud upload request")
 	}
 	// add pdtm meta params
-	req.URL.RawQuery = updateutils.GetpdtmParams(config.Version)
+	req.URL.Params.Merge(updateutils.GetpdtmParams(config.Version))
+	// if it is upload endpoint also include name if it exists
+	if u.scanName != "" && req.URL.Path == uploadEndpoint {
+		req.URL.Params.Add("name", u.scanName)
+	}
+	req.URL.Update()
+
 	req.Header.Set(pdcpauth.ApiKeyHeaderName, u.creds.APIKey)
 	req.Header.Set("Content-Type", "application/octet-stream")
 	req.Header.Set("Accept", "application/json")

internal/runner/inputs.go

@@ -1,6 +1,7 @@
 package runner
 
 import (
+	"context"
 	"sync/atomic"
 	"time"
 
@@ -16,8 +17,6 @@ import (
 	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
-var GlobalProbeBulkSize = 50
-
 // initializeTemplatesHTTPInput initializes the http form of input
 // for any loaded http templates if input is in non-standard format.
 func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
@@ -31,11 +30,6 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 	}
 	gologger.Info().Msgf("Running httpx on input host")
 
-	var bulkSize = GlobalProbeBulkSize
-	if r.options.BulkSize > GlobalProbeBulkSize {
-		bulkSize = r.options.BulkSize
-	}
-
 	httpxOptions := httpx.DefaultOptions
 	httpxOptions.RetryMax = r.options.Retries
 	httpxOptions.Timeout = time.Duration(r.options.Timeout) * time.Second
@@ -45,10 +39,8 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 		return nil, errors.Wrap(err, "could not create httpx client")
 	}
 
-	shouldFollowGlobalProbeBulkSize := bulkSize == GlobalProbeBulkSize
-
 	// Probe the non-standard URLs and store them in cache
-	swg, err := syncutil.New(syncutil.WithSize(bulkSize))
+	swg, err := syncutil.New(syncutil.WithSize(r.options.BulkSize))
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create adaptive group")
 	}
@@ -58,8 +50,10 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 			return true
 		}
 
-		if shouldFollowGlobalProbeBulkSize && swg.Size != GlobalProbeBulkSize {
-			swg.Resize(GlobalProbeBulkSize)
+		if r.options.ProbeConcurrency > 0 && swg.Size != r.options.ProbeConcurrency {
+			if err := swg.Resize(context.Background(), r.options.ProbeConcurrency); err != nil {
+				gologger.Error().Msgf("Could not resize workpool: %s\n", err)
+			}
 		}
 
 		swg.Add()

internal/runner/lazy.go

@@ -1,6 +1,7 @@
 package runner
 
 import (
+	"context"
 	"fmt"
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/authprovider/authx"
@@ -71,7 +72,8 @@ func GetLazyAuthFetchCallback(opts *AuthLazyFetchOptions) authx.LazyFetchSecret
 		tmpl := tmpls[0]
 		// add args to tmpl here
 		vars := map[string]interface{}{}
-		ctx := scan.NewScanContext(contextargs.NewWithInput(d.Input))
+		mainCtx := context.Background()
+		ctx := scan.NewScanContext(mainCtx, contextargs.NewWithInput(mainCtx, d.Input))
 		for _, v := range d.Variables {
 			vars[v.Key] = v.Value
 			ctx.Input.Add(v.Key, v.Value)

internal/runner/options.go

@@ -3,6 +3,7 @@ package runner
 import (
 	"bufio"
 	"fmt"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -18,6 +19,7 @@ import (
 	"github.com/projectdiscovery/gologger/levels"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
@@ -25,6 +27,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonl"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/sarif"
+	"github.com/projectdiscovery/nuclei/v3/pkg/templates/extensions"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml"
 	fileutil "github.com/projectdiscovery/utils/file"
@@ -74,6 +77,31 @@ func ParseOptions(options *types.Options) {
 		}
 		os.Exit(0)
 	}
+
+	defaultProfilesPath := filepath.Join(config.DefaultConfig.GetTemplateDir(), "profiles")
+	if options.ListTemplateProfiles {
+		gologger.Print().Msgf(
+			"\nListing available %v nuclei template profiles for %v",
+			config.DefaultConfig.TemplateVersion,
+			config.DefaultConfig.TemplatesDirectory,
+		)
+		templatesRootDir := config.DefaultConfig.GetTemplateDir()
+		err := filepath.WalkDir(defaultProfilesPath, func(iterItem string, d fs.DirEntry, err error) error {
+			ext := filepath.Ext(iterItem)
+			isYaml := ext == extensions.YAML || ext == extensions.YML
+			if err != nil || d.IsDir() || !isYaml {
+				return nil
+			}
+			if profileRelPath, err := filepath.Rel(templatesRootDir, iterItem); err == nil {
+				gologger.Print().Msgf("%s (%s)\n", profileRelPath, strings.TrimSuffix(filepath.Base(iterItem), ext))
+			}
+			return nil
+		})
+		if err != nil {
+			gologger.Error().Msgf("%s\n", err)
+		}
+		os.Exit(0)
+	}
 	if options.StoreResponseDir != DefaultDumpTrafficOutputFolder && !options.StoreResponse {
 		gologger.Debug().Msgf("Store response directory specified, enabling \"store-resp\" flag automatically\n")
 		options.StoreResponse = true
@@ -87,7 +115,12 @@ func ParseOptions(options *types.Options) {
 	// Load the resolvers if user asked for them
 	loadResolvers(options)
 
-	err := protocolinit.Init(options)
+	err := protocolstate.Init(options)
+	if err != nil {
+		gologger.Fatal().Msgf("Could not initialize protocol state: %s\n", err)
+	}
+
+	err = protocolinit.Init(options)
 	if err != nil {
 		gologger.Fatal().Msgf("Could not initialize protocols: %s\n", err)
 	}

internal/runner/runner.go

@@ -32,6 +32,7 @@ import (
 
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/internal/colorizer"
+	"github.com/projectdiscovery/nuclei/v3/internal/httpapi"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk"
@@ -52,6 +53,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/uncover"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/excludematchers"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
+	httpProtocol "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
@@ -88,8 +90,9 @@ type Runner struct {
 	pdcpUploadErrMsg string
 	inputProvider    provider.InputProvider
 	//general purpose temporary directory
-	tmpDir string
-	parser parser.Parser
+	tmpDir          string
+	parser          parser.Parser
+	httpApiEndpoint *httpapi.Server
 }
 
 const pprofServerAddress = "127.0.0.1:8086"
@@ -221,6 +224,17 @@ func New(options *types.Options) (*Runner, error) {
 		}()
 	}
 
+	if options.HttpApiEndpoint != "" {
+		apiServer := httpapi.New(options.HttpApiEndpoint, options)
+		gologger.Info().Msgf("Listening api endpoint on: %s", options.HttpApiEndpoint)
+		runner.httpApiEndpoint = apiServer
+		go func() {
+			if err := apiServer.Start(); err != nil {
+				gologger.Error().Msgf("Failed to start API server: %s", err)
+			}
+		}()
+	}
+
 	if (len(options.Templates) == 0 || !options.NewTemplates || (options.TargetsFilePath == "" && !options.Stdin && len(options.Targets) == 0)) && options.UpdateTemplates {
 		os.Exit(0)
 	}
@@ -345,6 +359,10 @@ func (r *Runner) runStandardEnumeration(executerOpts protocols.ExecutorOptions,
 
 // Close releases all the resources and cleans up
 func (r *Runner) Close() {
+	// dump hosterrors cache
+	if r.hostErrors != nil {
+		r.hostErrors.Close()
+	}
 	if r.output != nil {
 		r.output.Close()
 	}
@@ -400,7 +418,11 @@ func (r *Runner) setupPDCPUpload(writer output.Writer) output.Writer {
 		return writer
 	}
 	if r.options.ScanID != "" {
-		uploadWriter.SetScanID(r.options.ScanID)
+		// ignore and use empty scan id if invalid
+		_ = uploadWriter.SetScanID(r.options.ScanID)
+	}
+	if r.options.ScanName != "" {
+		uploadWriter.SetScanName(r.options.ScanName)
 	}
 	return output.NewMultiWriter(writer, uploadWriter)
 }
@@ -446,7 +468,7 @@ func (r *Runner) RunEnumeration() error {
 		Parser:             r.parser,
 	}
 
-	if env.GetEnvOrDefault("NUCLEI_ARGS", "") == "req_url_pattern=true" {
+	if config.DefaultConfig.IsDebugArgEnabled(config.DebugExportURLPattern) {
 		// Go StdLib style experimental/debug feature switch
 		executorOpts.ExportReqURLPattern = true
 	}
@@ -496,6 +518,23 @@ func (r *Runner) RunEnumeration() error {
 		return errors.Wrap(err, "Could not create loader.")
 	}
 
+	// list all templates or tags as specified by user.
+	// This uses a separate parser to reduce time taken as
+	// normally nuclei does a lot of compilation and stuff
+	// for templates, which we don't want for these simp
+	if r.options.TemplateList || r.options.TemplateDisplay || r.options.TagList {
+		if err := store.LoadTemplatesOnlyMetadata(); err != nil {
+			return err
+		}
+
+		if r.options.TagList {
+			r.listAvailableStoreTags(store)
+		} else {
+			r.listAvailableStoreTemplates(store)
+		}
+		os.Exit(0)
+	}
+
 	if r.options.Validate {
 		if err := store.ValidateTemplates(); err != nil {
 			return err
@@ -526,12 +565,6 @@ func (r *Runner) RunEnumeration() error {
 			_ = r.inputProvider.SetWithExclusions(host)
 		}
 	}
-	// list all templates
-	if r.options.TemplateList || r.options.TemplateDisplay {
-		r.listAvailableStoreTemplates(store)
-		os.Exit(0)
-	}
-
 	// display execution info like version , templates used etc
 	r.displayExecutionInfo(store)
 
@@ -655,7 +688,7 @@ func (r *Runner) executeTemplatesInput(store *loader.Store, engine *core.Engine)
 	if r.inputProvider == nil {
 		return nil, errors.New("no input provider found")
 	}
-	results := engine.ExecuteScanWithOpts(finalTemplates, r.inputProvider, r.options.DisableClustering)
+	results := engine.ExecuteScanWithOpts(context.Background(), finalTemplates, r.inputProvider, r.options.DisableClustering)
 	return results, nil
 }
 
@@ -684,6 +717,7 @@ func (r *Runner) displayExecutionInfo(store *loader.Store) {
 	} else {
 		stats.DisplayAsWarning(templates.SkippedCodeTmplTamperedStats)
 	}
+	stats.DisplayAsWarning(httpProtocol.SetThreadToCountZero)
 	stats.ForceDisplayWarning(templates.SkippedUnsignedStats)
 	stats.ForceDisplayWarning(templates.SkippedRequestSignatureStats)
 

internal/runner/templates.go

@@ -3,9 +3,11 @@ package runner
 import (
 	"bytes"
 	"path/filepath"
+	"sort"
 	"strings"
 
 	"github.com/alecthomas/chroma/quick"
+	jsoniter "github.com/json-iterator/go"
 	"github.com/logrusorgru/aurora"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
@@ -71,6 +73,40 @@ func (r *Runner) listAvailableStoreTemplates(store *loader.Store) {
 	}
 }
 
+func (r *Runner) listAvailableStoreTags(store *loader.Store) {
+	gologger.Print().Msgf(
+		"\nListing available %v nuclei tags for %v",
+		config.DefaultConfig.TemplateVersion,
+		config.DefaultConfig.TemplatesDirectory,
+	)
+	tagsMap := make(map[string]int)
+	for _, tpl := range store.Templates() {
+		for _, tag := range tpl.Info.Tags.ToSlice() {
+			tagsMap[tag]++
+		}
+	}
+	type kv struct {
+		Key   string `json:"tag"`
+		Value int    `json:"count"`
+	}
+	var tagsList []kv
+	for k, v := range tagsMap {
+		tagsList = append(tagsList, kv{k, v})
+	}
+	sort.Slice(tagsList, func(i, j int) bool {
+		return tagsList[i].Value > tagsList[j].Value
+	})
+
+	for _, tag := range tagsList {
+		if r.options.JSONL {
+			marshalled, _ := jsoniter.Marshal(tag)
+			gologger.Silent().Msgf("%s\n", string(marshalled))
+		} else {
+			gologger.Silent().Msgf("%s (%d)\n", tag.Key, tag.Value)
+		}
+	}
+}
+
 func (r *Runner) highlightTemplate(body *[]byte) ([]byte, error) {
 	var buf bytes.Buffer
 	// YAML lexer, true color terminal formatter and monokai style

lib/config.go

@@ -10,6 +10,7 @@ import (
 	"github.com/projectdiscovery/ratelimit"
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/authprovider"
+	"github.com/projectdiscovery/nuclei/v3/pkg/catalog"
 	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"github.com/projectdiscovery/nuclei/v3/pkg/progress"
@@ -393,7 +394,7 @@ func EnablePassiveMode() NucleiSDKOptions {
 	}
 }
 
-// WithAuthOptions allows setting a custom authprovider implementation
+// WithAuthProvider allows setting a custom authprovider implementation
 func WithAuthProvider(provider authprovider.AuthProvider) NucleiSDKOptions {
 	return func(e *NucleiEngine) error {
 		e.authprovider = provider
@@ -425,3 +426,11 @@ func SignedTemplatesOnly() NucleiSDKOptions {
 		return nil
 	}
 }
+
+// WithCatalog uses a supplied catalog
+func WithCatalog(cat catalog.Catalog) NucleiSDKOptions {
+	return func(e *NucleiEngine) error {
+		e.catalog = cat
+		return nil
+	}
+}

lib/multi.go

@@ -58,6 +58,22 @@ func createEphemeralObjects(base *NucleiEngine, opts *types.Options) (*unsafeOpt
 	return u, nil
 }
 
+// closeEphemeralObjects closes all resources used by ephemeral nuclei objects/instances/types
+func closeEphemeralObjects(u *unsafeOptions) {
+	if u.executerOpts.RateLimiter != nil {
+		u.executerOpts.RateLimiter.Stop()
+	}
+	// dereference all objects that were inherited from base nuclei engine
+	// since these are meant to be closed globally by base nuclei engine
+	u.executerOpts.Output = nil
+	u.executerOpts.IssuesClient = nil
+	u.executerOpts.Interactsh = nil
+	u.executerOpts.HostErrorsCache = nil
+	u.executerOpts.Progress = nil
+	u.executerOpts.Catalog = nil
+	u.executerOpts.Parser = nil
+}
+
 // ThreadSafeNucleiEngine is a tweaked version of nuclei.Engine whose methods are thread-safe
 // and can be used concurrently. Non-thread-safe methods start with Global prefix
 type ThreadSafeNucleiEngine struct {
@@ -95,11 +111,11 @@ func (e *ThreadSafeNucleiEngine) GlobalResultCallback(callback func(event *outpu
 	e.eng.resultCallbacks = []func(*output.ResultEvent){callback}
 }
 
-// ExecuteWithCallback executes templates on targets and calls callback on each result(only if results are found)
+// ExecuteNucleiWithOptsCtx executes templates on targets and calls callback on each result(only if results are found)
 // This method can be called concurrently and it will use some global resources but can be runned parallelly
 // by invoking this method with different options and targets
 // Note: Not all options are thread-safe. this method will throw error if you try to use non-thread-safe options
-func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOpts(targets []string, opts ...NucleiSDKOptions) error {
+func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOptsCtx(ctx context.Context, targets []string, opts ...NucleiSDKOptions) error {
 	baseOpts := *e.eng.opts
 	tmpEngine := &NucleiEngine{opts: &baseOpts, mode: threadSafe}
 	for _, option := range opts {
@@ -112,6 +128,8 @@ func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOpts(targets []string, opts ..
 	if err != nil {
 		return err
 	}
+	// cleanup and stop all resources
+	defer closeEphemeralObjects(unsafeOpts)
 
 	// load templates
 	workflowLoader, err := workflow.NewLoader(&unsafeOpts.executerOpts)
@@ -138,12 +156,18 @@ func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOpts(targets []string, opts ..
 	engine := core.New(tmpEngine.opts)
 	engine.SetExecuterOptions(unsafeOpts.executerOpts)
 
-	_ = engine.ExecuteScanWithOpts(store.Templates(), inputProvider, false)
+	_ = engine.ExecuteScanWithOpts(context.Background(), store.Templates(), inputProvider, false)
 
 	engine.WorkPool().Wait()
 	return nil
 }
 
+// ExecuteNucleiWithOpts is same as ExecuteNucleiWithOptsCtx but with default context
+// This is a placeholder and will be deprecated in future major release
+func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOpts(targets []string, opts ...NucleiSDKOptions) error {
+	return e.ExecuteNucleiWithOptsCtx(context.Background(), targets, opts...)
+}
+
 // Close all resources used by nuclei engine
 func (e *ThreadSafeNucleiEngine) Close() {
 	e.eng.Close()

lib/sdk.go

@@ -3,10 +3,11 @@ package nuclei
 import (
 	"bufio"
 	"bytes"
+	"context"
 	"io"
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/authprovider"
-	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk"
+	"github.com/projectdiscovery/nuclei/v3/pkg/catalog"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
 	"github.com/projectdiscovery/nuclei/v3/pkg/core"
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
@@ -17,6 +18,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
@@ -63,7 +65,7 @@ type NucleiEngine struct {
 
 	// unexported core fields
 	interactshClient *interactsh.Client
-	catalog          *disk.DiskCatalog
+	catalog          catalog.Catalog
 	rateLimiter      *ratelimit.Limiter
 	store            *loader.Store
 	httpxClient      providerTypes.InputLivenessProbe
@@ -183,15 +185,43 @@ func (e *NucleiEngine) SignTemplate(tmplSigner *signer.TemplateSigner, data []by
 
 // Close all resources used by nuclei engine
 func (e *NucleiEngine) Close() {
-	e.interactshClient.Close()
-	e.rc.Close()
-	e.customWriter.Close()
-	e.hostErrCache.Close()
-	e.executerOpts.RateLimiter.Stop()
+	if e.interactshClient != nil {
+		e.interactshClient.Close()
+	}
+	if e.rc != nil {
+		e.rc.Close()
+	}
+	if e.customWriter != nil {
+		e.customWriter.Close()
+	}
+	if e.customProgress != nil {
+		e.customProgress.Stop()
+	}
+	if e.hostErrCache != nil {
+		e.hostErrCache.Close()
+	}
+	if e.executerOpts.RateLimiter != nil {
+		e.executerOpts.RateLimiter.Stop()
+	}
+	if e.rateLimiter != nil {
+		e.rateLimiter.Stop()
+	}
+	// close global shared resources
+	protocolstate.Close()
+	if e.inputProvider != nil {
+		e.inputProvider.Close()
+	}
+	if e.browserInstance != nil {
+		e.browserInstance.Close()
+	}
+	if e.httpxClient != nil {
+		_ = e.httpxClient.Close()
+	}
 }
 
-// ExecuteWithCallback executes templates on targets and calls callback on each result(only if results are found)
-func (e *NucleiEngine) ExecuteWithCallback(callback ...func(event *output.ResultEvent)) error {
+// ExecuteCallbackWithCtx executes templates on targets and calls callback on each result(only if results are found)
+// enable matcher-status option if you expect this callback to be called for all results regardless if it matched or not
+func (e *NucleiEngine) ExecuteCallbackWithCtx(ctx context.Context, callback ...func(event *output.ResultEvent)) error {
 	if !e.templatesLoaded {
 		_ = e.LoadAllTemplates()
 	}
@@ -210,15 +240,23 @@ func (e *NucleiEngine) ExecuteWithCallback(callback ...func(event *output.Result
 	}
 	e.resultCallbacks = append(e.resultCallbacks, filtered...)
 
-	_ = e.engine.ExecuteScanWithOpts(e.store.Templates(), e.inputProvider, false)
+	_ = e.engine.ExecuteScanWithOpts(context.Background(), e.store.Templates(), e.inputProvider, false)
 	defer e.engine.WorkPool().Wait()
 	return nil
 }
 
+// ExecuteWithCallback is same as ExecuteCallbackWithCtx but with default context
+// Note this is deprecated and will be removed in future major release
+func (e *NucleiEngine) ExecuteWithCallback(callback ...func(event *output.ResultEvent)) error {
+	return e.ExecuteCallbackWithCtx(context.Background(), callback...)
+}
+
+// Options return nuclei Type Options
 func (e *NucleiEngine) Options() *types.Options {
 	return e.opts
 }
 
+// Engine returns core Executer of nuclei
 func (e *NucleiEngine) Engine() *core.Engine {
 	return e.engine
 }

lib/sdk_private.go

@@ -25,7 +25,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
@@ -35,6 +34,8 @@ import (
 	"github.com/projectdiscovery/ratelimit"
 )
 
+var sharedInit sync.Once = sync.Once{}
+
 // applyRequiredDefaults to options
 func (e *NucleiEngine) applyRequiredDefaults() {
 	mockoutput := testutils.NewMockOutputWriter(e.opts.OmitTemplate)
@@ -116,8 +117,10 @@ func (e *NucleiEngine) init() error {
 
 	e.parser = templates.NewParser()
 
-	_ = protocolstate.Init(e.opts)
-	_ = protocolinit.Init(e.opts)
+	sharedInit.Do(func() {
+		_ = protocolinit.Init(e.opts)
+	})
+
 	e.applyRequiredDefaults()
 	var err error
 
@@ -146,7 +149,9 @@ func (e *NucleiEngine) init() error {
 		return err
 	}
 
-	e.catalog = disk.NewCatalog(config.DefaultConfig.TemplatesDirectory)
+	if e.catalog == nil {
+		e.catalog = disk.NewCatalog(config.DefaultConfig.TemplatesDirectory)
+	}
 
 	e.executerOpts = protocols.ExecutorOptions{
 		Output:          e.customWriter,

lib/sdk_test.go

@@ -1,60 +0,0 @@
-package nuclei_test
-
-import (
-	"testing"
-
-	nuclei "github.com/projectdiscovery/nuclei/v3/lib"
-	"github.com/stretchr/testify/require"
-)
-
-func TestSimpleNuclei(t *testing.T) {
-	ne, err := nuclei.NewNucleiEngine(
-		nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}),
-		nuclei.EnableStatsWithOpts(nuclei.StatsOptions{JSON: true}),
-	)
-	require.Nil(t, err)
-	ne.LoadTargets([]string{"scanme.sh"}, false) // probe non http/https target is set to false here
-	// when callback is nil it nuclei will print JSON output to stdout
-	err = ne.ExecuteWithCallback(nil)
-	require.Nil(t, err)
-	defer ne.Close()
-}
-
-func TestSimpleNucleiRemote(t *testing.T) {
-	ne, err := nuclei.NewNucleiEngine(
-		nuclei.WithTemplatesOrWorkflows(
-			nuclei.TemplateSources{
-				RemoteTemplates: []string{"https://cloud.projectdiscovery.io/public/nameserver-fingerprint.yaml"},
-			},
-		),
-	)
-	require.Nil(t, err)
-	ne.LoadTargets([]string{"scanme.sh"}, false) // probe non http/https target is set to false here
-	err = ne.LoadAllTemplates()
-	require.Nil(t, err, "could not load templates")
-	// when callback is nil it nuclei will print JSON output to stdout
-	err = ne.ExecuteWithCallback(nil)
-	require.Nil(t, err)
-	defer ne.Close()
-}
-
-func TestThreadSafeNuclei(t *testing.T) {
-	// create nuclei engine with options
-	ne, err := nuclei.NewThreadSafeNucleiEngine()
-	require.Nil(t, err)
-
-	// scan 1 = run dns templates on scanme.sh
-	t.Run("scanme.sh", func(t *testing.T) {
-		err = ne.ExecuteNucleiWithOpts([]string{"scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}))
-		require.Nil(t, err)
-	})
-
-	// scan 2 = run dns templates on honey.scanme.sh
-	t.Run("honey.scanme.sh", func(t *testing.T) {
-		err = ne.ExecuteNucleiWithOpts([]string{"honey.scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}))
-		require.Nil(t, err)
-	})
-
-	// wait for all scans to finish
-	defer ne.Close()
-}

lib/tests/sdk_test.go

@@ -0,0 +1,132 @@
+package sdk_test
+
+import (
+	"os"
+	"os/exec"
+	"testing"
+	"time"
+
+	nuclei "github.com/projectdiscovery/nuclei/v3/lib"
+	"github.com/projectdiscovery/utils/env"
+	"github.com/stretchr/testify/require"
+	"github.com/tarunKoyalwar/goleak"
+)
+
+var knownLeaks = []goleak.Option{
+	// prettyify the output and generate dependency graph and more details instead of just stack output
+	goleak.Pretty(),
+	// net/http transport maintains idle connections which are closed with cooldown
+	// hence they don't count as leaks
+	goleak.IgnoreAnyFunction("net/http.(*http2ClientConn).readLoop"),
+}
+
+func TestSimpleNuclei(t *testing.T) {
+	fn := func() {
+		defer func() {
+			// resources like leveldb have a delay to commit in-memory resources
+			// to disk, typically 1-2 seconds, so we wait for 2 seconds
+			time.Sleep(2 * time.Second)
+			goleak.VerifyNone(t, knownLeaks...)
+		}()
+		ne, err := nuclei.NewNucleiEngine(
+			nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}), // filter dns templates
+			nuclei.EnableStatsWithOpts(nuclei.StatsOptions{JSON: true}),
+		)
+		require.Nil(t, err)
+		ne.LoadTargets([]string{"scanme.sh"}, false) // probe non http/https target is set to false here
+		// when callback is nil it nuclei will print JSON output to stdout
+		err = ne.ExecuteWithCallback(nil)
+		require.Nil(t, err)
+		defer ne.Close()
+	}
+
+	// this is shared test so needs to be run as seperate process
+	if env.GetEnvOrDefault("TestSimpleNuclei", false) {
+		// run as new process
+		cmd := exec.Command(os.Args[0], "-test.run=TestSimpleNuclei")
+		cmd.Env = append(os.Environ(), "TestSimpleNuclei=true")
+		out, err := cmd.CombinedOutput()
+		if err != nil {
+			t.Fatalf("process ran with error %s, output: %s", err, out)
+		}
+	} else {
+		fn()
+	}
+}
+
+func TestSimpleNucleiRemote(t *testing.T) {
+	fn := func() {
+		defer func() {
+			// resources like leveldb have a delay to commit in-memory resources
+			// to disk, typically 1-2 seconds, so we wait for 2 seconds
+			time.Sleep(2 * time.Second)
+			goleak.VerifyNone(t, knownLeaks...)
+		}()
+		ne, err := nuclei.NewNucleiEngine(
+			nuclei.WithTemplatesOrWorkflows(
+				nuclei.TemplateSources{
+					RemoteTemplates: []string{"https://cloud.projectdiscovery.io/public/nameserver-fingerprint.yaml"},
+				},
+			),
+		)
+		require.Nil(t, err)
+		ne.LoadTargets([]string{"scanme.sh"}, false) // probe non http/https target is set to false here
+		err = ne.LoadAllTemplates()
+		require.Nil(t, err, "could not load templates")
+		// when callback is nil it nuclei will print JSON output to stdout
+		err = ne.ExecuteWithCallback(nil)
+		require.Nil(t, err)
+		defer ne.Close()
+	}
+	// this is shared test so needs to be run as seperate process
+	if env.GetEnvOrDefault("TestSimpleNucleiRemote", false) {
+		cmd := exec.Command(os.Args[0], "-test.run=TestSimpleNucleiRemote")
+		cmd.Env = append(os.Environ(), "TestSimpleNucleiRemote=true")
+		out, err := cmd.CombinedOutput()
+		if err != nil {
+			t.Fatalf("process ran with error %s, output: %s", err, out)
+		}
+	} else {
+		fn()
+	}
+}
+
+func TestThreadSafeNuclei(t *testing.T) {
+	fn := func() {
+		defer func() {
+			// resources like leveldb have a delay to commit in-memory resources
+			// to disk, typically 1-2 seconds, so we wait for 2 seconds
+			time.Sleep(2 * time.Second)
+			goleak.VerifyNone(t, knownLeaks...)
+		}()
+		// create nuclei engine with options
+		ne, err := nuclei.NewThreadSafeNucleiEngine()
+		require.Nil(t, err)
+
+		// scan 1 = run dns templates on scanme.sh
+		t.Run("scanme.sh", func(t *testing.T) {
+			err = ne.ExecuteNucleiWithOpts([]string{"scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}))
+			require.Nil(t, err)
+		})
+
+		// scan 2 = run dns templates on honey.scanme.sh
+		t.Run("honey.scanme.sh", func(t *testing.T) {
+			err = ne.ExecuteNucleiWithOpts([]string{"honey.scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}))
+			require.Nil(t, err)
+		})
+
+		// wait for all scans to finish
+		defer ne.Close()
+	}
+
+	if env.GetEnvOrDefault("TestThreadSafeNuclei", false) {
+		cmd := exec.Command(os.Args[0], "-test.run=TestThreadSafeNuclei")
+		cmd.Env = append(os.Environ(), "TestThreadSafeNuclei=true")
+		out, err := cmd.CombinedOutput()
+		if err != nil {
+			t.Fatalf("process ran with error %s, output: %s", err, out)
+		}
+	} else {
+		fn()
+	}
+}

pkg/authprovider/authx/file.go

@@ -219,7 +219,7 @@ func GetAuthDataFromFile(file string) (*Authx, error) {
 	return GetAuthDataFromJSON(bin)
 }
 
-// GetTemplateIDsFromSecretFile reads the template IDs from the secret file
+// GetTemplatePathsFromSecretFile reads the template IDs from the secret file
 func GetTemplatePathsFromSecretFile(file string) ([]string, error) {
 	auth, err := GetAuthDataFromFile(file)
 	if err != nil {

pkg/catalog/config/constants.go

@@ -31,7 +31,7 @@ const (
 	CLIConfigFileName               = "config.yaml"
 	ReportingConfigFilename         = "reporting-config.yaml"
 	// Version is the current version of nuclei
-	Version = `v3.2.4`
+	Version = `v3.2.8`
 	// Directory Names of custom templates
 	CustomS3TemplatesDirName     = "s3"
 	CustomGitHubTemplatesDirName = "github"
@@ -68,3 +68,13 @@ func trimDevIfExists(version string) string {
 	}
 	return version
 }
+
+// similar to go pattern of enabling debug related features
+// we add custom/extra switches for debugging purposes
+const (
+	// DebugArgHostErrorStats is used to print host error stats
+	// when it is closed
+	DebugArgHostErrorStats = "host-error-stats"
+	// DebugExportReqURLPattern is used to export request URL pattern
+	DebugExportURLPattern = "req-url-pattern"
+)

pkg/catalog/config/nucleiconfig.go

@@ -46,9 +46,10 @@ type Config struct {
 	LatestNucleiIgnoreHash       string `json:"nuclei-latest-ignore-hash,omitempty"`
 
 	// internal / unexported fields
-	disableUpdates bool   `json:"-"` // disable updates both version check and template updates
-	homeDir        string `json:"-"` //  User Home Directory
-	configDir      string `json:"-"` //  Nuclei Global Config Directory
+	disableUpdates bool     `json:"-"` // disable updates both version check and template updates
+	homeDir        string   `json:"-"` //  User Home Directory
+	configDir      string   `json:"-"` //  Nuclei Global Config Directory
+	debugArgs      []string `json:"-"` // debug args
 }
 
 // IsCustomTemplate determines whether a given template is custom-built or part of the official Nuclei templates.
@@ -166,12 +167,12 @@ func (c *Config) GetTemplateIndexFilePath() string {
 	return filepath.Join(c.TemplatesDirectory, NucleiTemplatesIndexFileName)
 }
 
-// GetTemplatesConfigFilePath returns checksum file path of nuclei templates
+// GetChecksumFilePath returns checksum file path of nuclei templates
 func (c *Config) GetChecksumFilePath() string {
 	return filepath.Join(c.TemplatesDirectory, NucleiTemplatesCheckSumFileName)
 }
 
-// GetCLIOptsConfigFilePath returns the nuclei cli config file path
+// GetFlagsConfigFilePath returns the nuclei cli config file path
 func (c *Config) GetFlagsConfigFilePath() string {
 	return filepath.Join(c.configDir, CLIConfigFileName)
 }
@@ -329,6 +330,46 @@ func (c *Config) copyIgnoreFile() {
 	}
 }
 
+// IsDebugArgEnabled checks if debug arg is enabled
+// this could be a feature specific to debugging like PPROF or printing stats
+// of max host error etc
+func (c *Config) IsDebugArgEnabled(arg string) bool {
+	for _, v := range c.debugArgs {
+		if v == arg {
+			return true
+		}
+	}
+	return false
+}
+
+// parseDebugArgs from string
+func (c *Config) parseDebugArgs(data string) {
+	// use space as seperator instead of commas
+	tmp := strings.Fields(data)
+	for _, v := range tmp {
+		key := v
+		value := ""
+		// if it is key value pair then split it
+		if strings.Contains(v, "=") {
+			parts := strings.SplitN(v, "=", 2)
+			if len(parts) != 2 {
+				continue
+			}
+			key, value = strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1])
+		}
+		if value == "false" || value == "0" {
+			// if false or disabled then skip
+			continue
+		}
+		switch key {
+		case DebugArgHostErrorStats:
+			c.debugArgs = append(c.debugArgs, DebugArgHostErrorStats)
+		case DebugExportURLPattern:
+			c.debugArgs = append(c.debugArgs, DebugExportURLPattern)
+		}
+	}
+}
+
 func init() {
 	// first attempt to migrate all files from old config directory to new config directory
 	goflags.AttemptConfigMigration() // regardless how many times this is called it will only migrate once based on condition
@@ -377,6 +418,7 @@ func init() {
 	// and even if it is changed we don't follow it since it is not expected behavior
 	// If custom templates are in default locations only then they are loaded while running nuclei
 	DefaultConfig.SetTemplatesDir(DefaultConfig.TemplatesDirectory)
+	DefaultConfig.parseDebugArgs(env.GetEnvOrDefault("NUCLEI_ARGS", ""))
 }
 
 // Add Default Config adds default when .templates-config.json file is not present

pkg/catalog/loader/loader.go

@@ -255,6 +255,46 @@ func init() {
 	templateIDPathMap = make(map[string]string)
 }
 
+// LoadTemplatesOnlyMetadata loads only the metadata of the templates
+func (store *Store) LoadTemplatesOnlyMetadata() error {
+	templatePaths, errs := store.config.Catalog.GetTemplatesPath(store.finalTemplates)
+	store.logErroredTemplates(errs)
+
+	filteredTemplatePaths := store.pathFilter.Match(templatePaths)
+
+	validPaths := make(map[string]struct{})
+	for templatePath := range filteredTemplatePaths {
+		loaded, err := store.config.ExecutorOptions.Parser.LoadTemplate(templatePath, store.tagFilter, nil, store.config.Catalog)
+		if loaded || store.pathFilter.MatchIncluded(templatePath) {
+			validPaths[templatePath] = struct{}{}
+		}
+		if err != nil {
+			if strings.Contains(err.Error(), templates.ErrExcluded.Error()) {
+				stats.Increment(templates.TemplatesExcludedStats)
+				if config.DefaultConfig.LogAllEvents {
+					gologger.Print().Msgf("[%v] %v\n", aurora.Yellow("WRN").String(), err.Error())
+				}
+				continue
+			}
+			gologger.Warning().Msg(err.Error())
+		}
+	}
+	parserItem, ok := store.config.ExecutorOptions.Parser.(*templates.Parser)
+	if !ok {
+		return errors.New("invalid parser")
+	}
+	templatesCache := parserItem.Cache()
+
+	for templatePath := range validPaths {
+		template, _, _ := templatesCache.Has(templatePath)
+		if template != nil {
+			template.Path = templatePath
+			store.templates = append(store.templates, template)
+		}
+	}
+	return nil
+}
+
 // ValidateTemplates takes a list of templates and validates them
 // erroring out on discovering any faulty templates.
 func (store *Store) ValidateTemplates() error {

pkg/core/execute_options.go

@@ -1,6 +1,7 @@
 package core
 
 import (
+	"context"
 	"sync"
 	"sync/atomic"
 
@@ -20,18 +21,18 @@ import (
 //
 // All the execution logic for the templates/workflows happens in this part
 // of the engine.
-func (e *Engine) Execute(templates []*templates.Template, target provider.InputProvider) *atomic.Bool {
-	return e.ExecuteScanWithOpts(templates, target, false)
+func (e *Engine) Execute(ctx context.Context, templates []*templates.Template, target provider.InputProvider) *atomic.Bool {
+	return e.ExecuteScanWithOpts(ctx, templates, target, false)
 }
 
 // ExecuteWithResults a list of templates with results
-func (e *Engine) ExecuteWithResults(templatesList []*templates.Template, target provider.InputProvider, callback func(*output.ResultEvent)) *atomic.Bool {
+func (e *Engine) ExecuteWithResults(ctx context.Context, templatesList []*templates.Template, target provider.InputProvider, callback func(*output.ResultEvent)) *atomic.Bool {
 	e.Callback = callback
-	return e.ExecuteScanWithOpts(templatesList, target, false)
+	return e.ExecuteScanWithOpts(ctx, templatesList, target, false)
 }
 
 // ExecuteScanWithOpts executes scan with given scanStrategy
-func (e *Engine) ExecuteScanWithOpts(templatesList []*templates.Template, target provider.InputProvider, noCluster bool) *atomic.Bool {
+func (e *Engine) ExecuteScanWithOpts(ctx context.Context, templatesList []*templates.Template, target provider.InputProvider, noCluster bool) *atomic.Bool {
 	results := &atomic.Bool{}
 	selfcontainedWg := &sync.WaitGroup{}
 
@@ -83,14 +84,14 @@ func (e *Engine) ExecuteScanWithOpts(templatesList []*templates.Template, target
 	}
 
 	// Execute All SelfContained in parallel
-	e.executeAllSelfContained(selfContained, results, selfcontainedWg)
+	e.executeAllSelfContained(ctx, selfContained, results, selfcontainedWg)
 
 	strategyResult := &atomic.Bool{}
 	switch e.options.ScanStrategy {
 	case scanstrategy.TemplateSpray.String():
-		strategyResult = e.executeTemplateSpray(filtered, target)
+		strategyResult = e.executeTemplateSpray(ctx, filtered, target)
 	case scanstrategy.HostSpray.String():
-		strategyResult = e.executeHostSpray(filtered, target)
+		strategyResult = e.executeHostSpray(ctx, filtered, target)
 	}
 
 	results.CompareAndSwap(false, strategyResult.Load())
@@ -100,7 +101,7 @@ func (e *Engine) ExecuteScanWithOpts(templatesList []*templates.Template, target
 }
 
 // executeTemplateSpray executes scan using template spray strategy where targets are iterated over each template
-func (e *Engine) executeTemplateSpray(templatesList []*templates.Template, target provider.InputProvider) *atomic.Bool {
+func (e *Engine) executeTemplateSpray(ctx context.Context, templatesList []*templates.Template, target provider.InputProvider) *atomic.Bool {
 	results := &atomic.Bool{}
 
 	// wp is workpool that contains different waitgroups for
@@ -108,6 +109,12 @@ func (e *Engine) executeTemplateSpray(templatesList []*templates.Template, targe
 	wp := e.GetWorkPool()
 
 	for _, template := range templatesList {
+		select {
+		case <-ctx.Done():
+			return results
+		default:
+		}
+
 		// resize check point - nop if there are no changes
 		wp.RefreshWithConfig(e.GetWorkPoolConfig())
 
@@ -125,7 +132,7 @@ func (e *Engine) executeTemplateSpray(templatesList []*templates.Template, targe
 			// All other request types are executed here
 			// Note: executeTemplateWithTargets creates goroutines and blocks
 			// given template is executed on all targets
-			e.executeTemplateWithTargets(tpl, target, results)
+			e.executeTemplateWithTargets(ctx, tpl, target, results)
 		}(template)
 	}
 	wp.Wait()
@@ -133,15 +140,21 @@ func (e *Engine) executeTemplateSpray(templatesList []*templates.Template, targe
 }
 
 // executeHostSpray executes scan using host spray strategy where templates are iterated over each target
-func (e *Engine) executeHostSpray(templatesList []*templates.Template, target provider.InputProvider) *atomic.Bool {
+func (e *Engine) executeHostSpray(ctx context.Context, templatesList []*templates.Template, target provider.InputProvider) *atomic.Bool {
 	results := &atomic.Bool{}
 	wp, _ := syncutil.New(syncutil.WithSize(e.options.BulkSize + e.options.HeadlessBulkSize))
 
 	target.Iterate(func(value *contextargs.MetaInput) bool {
+		select {
+		case <-ctx.Done():
+			return false
+		default:
+		}
+
 		wp.Add()
 		go func(targetval *contextargs.MetaInput) {
 			defer wp.Done()
-			e.executeTemplatesOnTarget(templatesList, targetval, results)
+			e.executeTemplatesOnTarget(ctx, templatesList, targetval, results)
 		}(value)
 		return true
 	})

pkg/core/executors.go

@@ -1,6 +1,7 @@
 package core
 
 import (
+	"context"
 	"sync"
 	"sync/atomic"
 
@@ -17,14 +18,14 @@ import (
 // Executors are low level executors that deals with template execution on a target
 
 // executeAllSelfContained executes all self contained templates that do not use `target`
-func (e *Engine) executeAllSelfContained(alltemplates []*templates.Template, results *atomic.Bool, sg *sync.WaitGroup) {
+func (e *Engine) executeAllSelfContained(ctx context.Context, alltemplates []*templates.Template, results *atomic.Bool, sg *sync.WaitGroup) {
 	for _, v := range alltemplates {
 		sg.Add(1)
 		go func(template *templates.Template) {
 			defer sg.Done()
 			var err error
 			var match bool
-			ctx := scan.NewScanContext(contextargs.New())
+			ctx := scan.NewScanContext(ctx, contextargs.New(ctx))
 			if e.Callback != nil {
 				if results, err := template.Executer.ExecuteWithResults(ctx); err != nil {
 					for _, result := range results {
@@ -45,7 +46,7 @@ func (e *Engine) executeAllSelfContained(alltemplates []*templates.Template, res
 }
 
 // executeTemplateWithTarget executes a given template on x targets (with a internal targetpool(i.e concurrency))
-func (e *Engine) executeTemplateWithTargets(template *templates.Template, target provider.InputProvider, results *atomic.Bool) {
+func (e *Engine) executeTemplateWithTargets(ctx context.Context, template *templates.Template, target provider.InputProvider, results *atomic.Bool) {
 	// this is target pool i.e max target to execute
 	wg := e.workPool.InputPool(template.Type())
 
@@ -77,6 +78,12 @@ func (e *Engine) executeTemplateWithTargets(template *templates.Template, target
 	}
 
 	target.Iterate(func(scannedValue *contextargs.MetaInput) bool {
+		select {
+		case <-ctx.Done():
+			return false // exit
+		default:
+		}
+
 		// Best effort to track the host progression
 		// skips indexes lower than the minimum in-flight at interruption time
 		var skip bool
@@ -100,7 +107,7 @@ func (e *Engine) executeTemplateWithTargets(template *templates.Template, target
 		currentInfo.Unlock()
 
 		// Skip if the host has had errors
-		if e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(scannedValue.ID()) {
+		if e.executerOpts.HostErrorsCache != nil && e.executerOpts.HostErrorsCache.Check(contextargs.NewWithMetaInput(ctx, scannedValue)) {
 			return true
 		}
 
@@ -114,9 +121,9 @@ func (e *Engine) executeTemplateWithTargets(template *templates.Template, target
 
 			var match bool
 			var err error
-			ctxArgs := contextargs.New()
+			ctxArgs := contextargs.New(ctx)
 			ctxArgs.MetaInput = value
-			ctx := scan.NewScanContext(ctxArgs)
+			ctx := scan.NewScanContext(ctx, ctxArgs)
 			switch template.Type() {
 			case types.WorkflowProtocol:
 				match = e.executeWorkflow(ctx, template.CompiledWorkflow)
@@ -149,7 +156,7 @@ func (e *Engine) executeTemplateWithTargets(template *templates.Template, target
 }
 
 // executeTemplatesOnTarget execute given templates on given single target
-func (e *Engine) executeTemplatesOnTarget(alltemplates []*templates.Template, target *contextargs.MetaInput, results *atomic.Bool) {
+func (e *Engine) executeTemplatesOnTarget(ctx context.Context, alltemplates []*templates.Template, target *contextargs.MetaInput, results *atomic.Bool) {
 	// all templates are executed on single target
 
 	// wp is workpool that contains different waitgroups for
@@ -158,6 +165,12 @@ func (e *Engine) executeTemplatesOnTarget(alltemplates []*templates.Template, ta
 	wp := e.GetWorkPool()
 
 	for _, tpl := range alltemplates {
+		select {
+		case <-ctx.Done():
+			return
+		default:
+		}
+
 		// resize check point - nop if there are no changes
 		wp.RefreshWithConfig(e.GetWorkPoolConfig())
 
@@ -173,9 +186,9 @@ func (e *Engine) executeTemplatesOnTarget(alltemplates []*templates.Template, ta
 
 			var match bool
 			var err error
-			ctxArgs := contextargs.New()
+			ctxArgs := contextargs.New(ctx)
 			ctxArgs.MetaInput = value
-			ctx := scan.NewScanContext(ctxArgs)
+			ctx := scan.NewScanContext(ctx, ctxArgs)
 			switch template.Type() {
 			case types.WorkflowProtocol:
 				match = e.executeWorkflow(ctx, template.CompiledWorkflow)
@@ -199,52 +212,3 @@ func (e *Engine) executeTemplatesOnTarget(alltemplates []*templates.Template, ta
 	}
 	wp.Wait()
 }
-
-type ChildExecuter struct {
-	e *Engine
-
-	results *atomic.Bool
-}
-
-// Close closes the executer returning bool results
-func (e *ChildExecuter) Close() *atomic.Bool {
-	e.e.workPool.Wait()
-	return e.results
-}
-
-// Execute executes a template and URLs
-func (e *ChildExecuter) Execute(template *templates.Template, value *contextargs.MetaInput) {
-	templateType := template.Type()
-
-	// resize check point - nop if there are no changes
-	e.e.WorkPool().RefreshWithConfig(e.e.GetWorkPoolConfig())
-
-	var wg *syncutil.AdaptiveWaitGroup
-	if templateType == types.HeadlessProtocol {
-		wg = e.e.workPool.Headless
-	} else {
-		wg = e.e.workPool.Default
-	}
-
-	wg.Add()
-	go func(tpl *templates.Template) {
-		defer wg.Done()
-
-		ctxArgs := contextargs.New()
-		ctxArgs.MetaInput = value
-		ctx := scan.NewScanContext(ctxArgs)
-		match, err := template.Executer.Execute(ctx)
-		if err != nil {
-			gologger.Warning().Msgf("[%s] Could not execute step: %s\n", e.e.executerOpts.Colorizer.BrightBlue(template.ID), err)
-		}
-		e.results.CompareAndSwap(false, match)
-	}(template)
-}
-
-// ExecuteWithOpts executes with the full options
-func (e *Engine) ChildExecuter() *ChildExecuter {
-	return &ChildExecuter{
-		e:       e,
-		results: &atomic.Bool{},
-	}
-}

pkg/core/workflow_execute.go

@@ -21,7 +21,7 @@ func (e *Engine) executeWorkflow(ctx *scan.ScanContext, w *workflows.Workflow) b
 
 	// at this point we should be at the start root execution of a workflow tree, hence we create global shared instances
 	workflowCookieJar, _ := cookiejar.New(nil)
-	ctxArgs := contextargs.New()
+	ctxArgs := contextargs.New(ctx.Context())
 	ctxArgs.MetaInput = ctx.Input.MetaInput
 	ctxArgs.CookieJar = workflowCookieJar
 
@@ -98,7 +98,7 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan
 			}
 			if err != nil {
 				if w.Options.HostErrorsCache != nil {
-					w.Options.HostErrorsCache.MarkFailed(ctx.Input.MetaInput.ID(), err)
+					w.Options.HostErrorsCache.MarkFailed(ctx.Input, err)
 				}
 				if len(template.Executers) == 1 {
 					mainErr = err
@@ -139,7 +139,7 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan
 							defer swg.Done()
 
 							// create a new context with the same input but with unset callbacks
-							subCtx := scan.NewScanContext(ctx.Input)
+							subCtx := scan.NewScanContext(ctx.Context(), ctx.Input)
 							if err := e.runWorkflowStep(subtemplate, subCtx, results, swg, w); err != nil {
 								gologger.Warning().Msgf(workflowStepExecutionError, subtemplate.Template, err)
 							}
@@ -165,7 +165,7 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan
 
 			go func(template *workflows.WorkflowTemplate) {
 				// create a new context with the same input but with unset callbacks
-				subCtx := scan.NewScanContext(ctx.Input)
+				subCtx := scan.NewScanContext(ctx.Context(), ctx.Input)
 				if err := e.runWorkflowStep(template, subCtx, results, swg, w); err != nil {
 					gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err)
 				}

pkg/core/workflow_execute_test.go

@@ -1,6 +1,7 @@
 package core
 
 import (
+	"context"
 	"testing"
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/stringslice"
@@ -25,8 +26,8 @@ func TestWorkflowsSimple(t *testing.T) {
 	}}
 
 	engine := &Engine{}
-	input := contextargs.NewWithInput("https://test.com")
-	ctx := scan.NewScanContext(input)
+	input := contextargs.NewWithInput(context.Background(), "https://test.com")
+	ctx := scan.NewScanContext(context.Background(), input)
 	matched := engine.executeWorkflow(ctx, workflow)
 	require.True(t, matched, "could not get correct match value")
 }
@@ -49,8 +50,8 @@ func TestWorkflowsSimpleMultiple(t *testing.T) {
 	}}
 
 	engine := &Engine{}
-	input := contextargs.NewWithInput("https://test.com")
-	ctx := scan.NewScanContext(input)
+	input := contextargs.NewWithInput(context.Background(), "https://test.com")
+	ctx := scan.NewScanContext(context.Background(), input)
 	matched := engine.executeWorkflow(ctx, workflow)
 	require.True(t, matched, "could not get correct match value")
 
@@ -77,8 +78,8 @@ func TestWorkflowsSubtemplates(t *testing.T) {
 	}}
 
 	engine := &Engine{}
-	input := contextargs.NewWithInput("https://test.com")
-	ctx := scan.NewScanContext(input)
+	input := contextargs.NewWithInput(context.Background(), "https://test.com")
+	ctx := scan.NewScanContext(context.Background(), input)
 	matched := engine.executeWorkflow(ctx, workflow)
 	require.True(t, matched, "could not get correct match value")
 
@@ -103,8 +104,8 @@ func TestWorkflowsSubtemplatesNoMatch(t *testing.T) {
 	}}
 
 	engine := &Engine{}
-	input := contextargs.NewWithInput("https://test.com")
-	ctx := scan.NewScanContext(input)
+	input := contextargs.NewWithInput(context.Background(), "https://test.com")
+	ctx := scan.NewScanContext(context.Background(), input)
 	matched := engine.executeWorkflow(ctx, workflow)
 	require.False(t, matched, "could not get correct match value")
 
@@ -134,8 +135,8 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) {
 	}}
 
 	engine := &Engine{}
-	input := contextargs.NewWithInput("https://test.com")
-	ctx := scan.NewScanContext(input)
+	input := contextargs.NewWithInput(context.Background(), "https://test.com")
+	ctx := scan.NewScanContext(context.Background(), input)
 	matched := engine.executeWorkflow(ctx, workflow)
 	require.True(t, matched, "could not get correct match value")
 
@@ -165,8 +166,8 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) {
 	}}
 
 	engine := &Engine{}
-	input := contextargs.NewWithInput("https://test.com")
-	ctx := scan.NewScanContext(input)
+	input := contextargs.NewWithInput(context.Background(), "https://test.com")
+	ctx := scan.NewScanContext(context.Background(), input)
 	matched := engine.executeWorkflow(ctx, workflow)
 	require.False(t, matched, "could not get correct match value")
 

pkg/core/workpool.go

@@ -1,6 +1,9 @@
 package core
 
 import (
+	"context"
+
+	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	syncutil "github.com/projectdiscovery/utils/sync"
 )
@@ -71,14 +74,18 @@ func (w *WorkPool) RefreshWithConfig(config WorkPoolConfig) {
 	if w.config.HeadlessInputConcurrency != config.HeadlessInputConcurrency {
 		w.config.HeadlessInputConcurrency = config.HeadlessInputConcurrency
 	}
-	w.Refresh()
+	w.Refresh(context.Background())
 }
 
-func (w *WorkPool) Refresh() {
+func (w *WorkPool) Refresh(ctx context.Context) {
 	if w.Default.Size != w.config.TypeConcurrency {
-		w.Default.Resize(w.config.TypeConcurrency)
+		if err := w.Default.Resize(ctx, w.config.TypeConcurrency); err != nil {
+			gologger.Warning().Msgf("Could not resize workpool: %s\n", err)
+		}
 	}
 	if w.Headless.Size != w.config.HeadlessTypeConcurrency {
-		w.Headless.Resize(w.config.HeadlessTypeConcurrency)
+		if err := w.Headless.Resize(ctx, w.config.HeadlessTypeConcurrency); err != nil {
+			gologger.Warning().Msgf("Could not resize workpool: %s\n", err)
+		}
 	}
 }

pkg/fuzz/component/headers.go

@@ -83,6 +83,9 @@ func (q *Header) Delete(key string) error {
 func (q *Header) Rebuild() (*retryablehttp.Request, error) {
 	cloned := q.req.Clone(context.Background())
 	q.value.parsed.Iterate(func(key string, value any) bool {
+		if strings.TrimSpace(key) == "" {
+			return true
+		}
 		if strings.EqualFold(key, "Host") {
 			return true
 		}

pkg/fuzz/execute.go

@@ -57,6 +57,8 @@ type GeneratedRequest struct {
 	DynamicValues map[string]interface{}
 	// Component is the component for the request
 	Component component.Component
+	// Parameter being fuzzed
+	Parameter string
 }
 
 // Execute executes a fuzzing rule accepting a callback on which
@@ -211,11 +213,19 @@ func (rule *Rule) executeRuleValues(input *ExecuteRuleInput, ruleComponent compo
 		})
 		// if mode is multiple now build and execute it
 		if rule.modeType == multipleModeType {
+			rule.Fuzz.KV.Iterate(func(key, value string) bool {
+				var evaluated string
+				evaluated, input.InteractURLs = rule.executeEvaluate(input, key, "", value, input.InteractURLs)
+				if err := ruleComponent.SetValue(key, evaluated); err != nil {
+					return true
+				}
+				return true
+			})
 			req, err := ruleComponent.Rebuild()
 			if err != nil {
 				return err
 			}
-			if gotErr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent); gotErr != nil {
+			if gotErr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, ""); gotErr != nil {
 				return gotErr
 			}
 		}

pkg/fuzz/parts.go

@@ -68,7 +68,7 @@ func (rule *Rule) executePartComponentOnValues(input *ExecuteRuleInput, payloadS
 				return err
 			}
 
-			if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent); qerr != nil {
+			if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, key); qerr != nil {
 				return qerr
 			}
 			// fmt.Printf("executed with value: %s\n", evaluated)
@@ -90,7 +90,7 @@ func (rule *Rule) executePartComponentOnValues(input *ExecuteRuleInput, payloadS
 		if err != nil {
 			return err
 		}
-		if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent); qerr != nil {
+		if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, ""); qerr != nil {
 			err = qerr
 			return err
 		}
@@ -125,7 +125,7 @@ func (rule *Rule) executePartComponentOnKV(input *ExecuteRuleInput, payload Valu
 				return err
 			}
 
-			if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent); qerr != nil {
+			if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, key); qerr != nil {
 				return err
 			}
 
@@ -144,12 +144,13 @@ func (rule *Rule) executePartComponentOnKV(input *ExecuteRuleInput, payload Valu
 }
 
 // execWithInput executes a rule with input via callback
-func (rule *Rule) execWithInput(input *ExecuteRuleInput, httpReq *retryablehttp.Request, interactURLs []string, component component.Component) error {
+func (rule *Rule) execWithInput(input *ExecuteRuleInput, httpReq *retryablehttp.Request, interactURLs []string, component component.Component, parameter string) error {
 	request := GeneratedRequest{
 		Request:       httpReq,
 		InteractURLs:  interactURLs,
 		DynamicValues: input.Values,
 		Component:     component,
+		Parameter:     parameter,
 	}
 	if !input.Callback(request) {
 		return types.ErrNoMoreRequests

pkg/input/formats/json/json.go

@@ -28,9 +28,10 @@ var _ formats.Format = &JSONFormat{}
 type proxifyRequest struct {
 	URL     string `json:"url"`
 	Request struct {
-		Header map[string]string `json:"header"`
-		Body   string            `json:"body"`
-		Raw    string            `json:"raw"`
+		Header   map[string]string `json:"header"`
+		Body     string            `json:"body"`
+		Raw      string            `json:"raw"`
+		Endpoint string            `json:"endpoint"`
 	} `json:"request"`
 }
 
@@ -63,6 +64,9 @@ func (j *JSONFormat) Parse(input string, resultsCb formats.ParseReqRespCallback)
 			return errors.Wrap(err, "could not decode json file")
 		}
 
+		if request.URL == "" && request.Request.Endpoint != "" {
+			request.URL = request.Request.Endpoint
+		}
 		rawRequest, err := types.ParseRawRequestWithURL(request.Request.Raw, request.URL)
 		if err != nil {
 			gologger.Warning().Msgf("jsonl: Could not parse raw request %s: %s\n", request.URL, err)

pkg/input/formats/openapi/generator.go

@@ -179,6 +179,10 @@ func generateRequestsFromOp(opts *generateReqOptions) error {
 	for _, parameter := range reqParams {
 		value := parameter.Value
 
+		if value.Schema == nil || value.Schema.Value == nil {
+			continue
+		}
+
 		// paramValue or default value to use
 		var paramValue interface{}
 

pkg/input/provider/list/hmap_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils/expand"
+	"github.com/projectdiscovery/utils/auth/pdcp"
 	"github.com/stretchr/testify/require"
 )
 
@@ -68,7 +69,7 @@ func (m *mockDnsHandler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) {
 		msg.Authoritative = true
 		domain := msg.Question[0].Name
 		msg.Answer = append(msg.Answer, &dns.AAAA{
-			Hdr:  dns.RR_Header{Name: domain, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60},
+			Hdr:  dns.RR_Header{Name: domain, Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 60},
 			AAAA: net.ParseIP("2400:6180:0:d0::91:1001"),
 		})
 	}
@@ -153,6 +154,13 @@ func Test_scanallips_normalizeStoreInputValue(t *testing.T) {
 }
 
 func Test_expandASNInputValue(t *testing.T) {
+	// skip this test if pdcp keys are not present
+	h := pdcp.PDCPCredHandler{}
+	creds, err := h.GetCreds()
+	if err != nil || creds == nil || creds.APIKey == "" {
+		t.Logf("Skipping asnmap test as pdcp keys are not present")
+		t.SkipNow()
+	}
 	tests := []struct {
 		asn                string
 		expectedOutputFile string

pkg/input/types/http.go

@@ -12,6 +12,7 @@ import (
 	"sync"
 
 	"github.com/projectdiscovery/retryablehttp-go"
+	"github.com/projectdiscovery/useragent"
 	"github.com/projectdiscovery/utils/conversion"
 	mapsutil "github.com/projectdiscovery/utils/maps"
 	urlutil "github.com/projectdiscovery/utils/url"
@@ -73,6 +74,10 @@ func (rr *RequestResponse) BuildRequest() (*retryablehttp.Request, error) {
 			req.Header.Add(k, v)
 			return true
 		})
+		if req.Header.Get("User-Agent") == "" {
+			userAgent := useragent.PickRandom()
+			req.Header.Set("User-Agent", userAgent.Raw)
+		}
 		rr.req = req
 	})
 	return rr.req, rr.reqErr

pkg/input/types/probe.go

@@ -4,4 +4,6 @@ package types
 type InputLivenessProbe interface {
 	// ProbeURL probes the scheme for a URL. first HTTPS is tried
 	ProbeURL(input string) (string, error)
+	// Close closes the liveness probe
+	Close() error
 }

pkg/installer/versioncheck.go

@@ -83,6 +83,10 @@ func UpdateIgnoreFile() error {
 }
 
 func doVersionCheck(isSDK bool) error {
+	// we use global retryablehttp client so its not immeditely gc'd if any references are held
+	// and according our config we have idle connections which are shown as leaked by goleak in tests
+	// i.e we close all idle connections after our use and it doesn't affect any other part of the code
+	defer retryableHttpClient.HTTPClient.CloseIdleConnections()
 	resp, err := retryableHttpClient.Get(pdtmNucleiVersionEndpoint + "?" + getpdtmParams(isSDK))
 	if err != nil {
 		return err

pkg/js/compiler/compiler.go

@@ -10,9 +10,15 @@ import (
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
 	contextutil "github.com/projectdiscovery/utils/context"
+	"github.com/projectdiscovery/utils/errkit"
 	stringsutil "github.com/projectdiscovery/utils/strings"
 )
 
+var (
+	// ErrJSExecDeadline is the error returned when alloted time for script execution exceeds
+	ErrJSExecDeadline = errkit.New("js engine execution deadline exceeded").SetKind(errkit.ErrKindDeadline).Build()
+)
+
 // Compiler provides a runtime to execute goja runtime
 // based javascript scripts efficiently while also
 // providing them access to custom modules defined in libs/.
@@ -37,6 +43,8 @@ type ExecuteOptions struct {
 	// Source is original source of the script
 	Source *string
 
+	Context context.Context
+
 	// Manually exported objects
 	exports map[string]interface{}
 }
@@ -77,13 +85,13 @@ func (c *Compiler) Execute(code string, args *ExecuteArgs) (ExecuteResult, error
 	if err != nil {
 		return nil, err
 	}
-	return c.ExecuteWithOptions(p, args, &ExecuteOptions{})
+	return c.ExecuteWithOptions(p, args, &ExecuteOptions{Context: context.Background()})
 }
 
 // ExecuteWithOptions executes a script with the provided options.
 func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs, opts *ExecuteOptions) (ExecuteResult, error) {
 	if opts == nil {
-		opts = &ExecuteOptions{}
+		opts = &ExecuteOptions{Context: context.Background()}
 	}
 	if args == nil {
 		args = NewExecuteArgs()
@@ -105,7 +113,7 @@ func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs,
 	}
 
 	// execute with context and timeout
-	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(opts.Timeout)*time.Second)
+	ctx, cancel := context.WithTimeoutCause(opts.Context, time.Duration(opts.Timeout)*time.Second, ErrJSExecDeadline)
 	defer cancel()
 	// execute the script
 	results, err := contextutil.ExecFuncWithTwoReturns(ctx, func() (val goja.Value, err error) {
@@ -117,6 +125,9 @@ func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs,
 		return ExecuteProgram(program, args, opts)
 	})
 	if err != nil {
+		if val, ok := err.(*goja.Exception); ok {
+			err = val.Unwrap()
+		}
 		return nil, err
 	}
 	var res ExecuteResult

pkg/js/compiler/init.go

@@ -1,6 +1,8 @@
 package compiler
 
-import "github.com/projectdiscovery/nuclei/v3/pkg/types"
+import (
+	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+)
 
 // jsprotocolInit
 
@@ -9,6 +11,7 @@ var (
 	JsProtocolTimeout       = 10
 	PoolingJsVmConcurrency  = 100
 	NonPoolingVMConcurrency = 20
+	JsTimeoutMultiplier     = 1.5
 )
 
 // Init initializes the javascript protocol
@@ -21,7 +24,9 @@ func Init(opts *types.Options) error {
 		// 100 is reasonable default
 		opts.JsConcurrency = 100
 	}
-	JsProtocolTimeout = opts.Timeout
+	// we have dialer timeout set to 10s so js needs to be at least
+	// 15s to return the actual error if not it will be a dialer timeout
+	JsProtocolTimeout = int(float64(opts.Timeout) * JsTimeoutMultiplier)
 	PoolingJsVmConcurrency = opts.JsConcurrency
 	PoolingJsVmConcurrency -= NonPoolingVMConcurrency
 	return nil

pkg/js/compiler/non-pool.go

@@ -8,7 +8,7 @@ import (
 )
 
 var (
-	ephemeraljsc, _ = syncutil.New(syncutil.WithSize(NonPoolingVMConcurrency))
+	ephemeraljsc    *syncutil.AdaptiveWaitGroup
 	lazyFixedSgInit = sync.OnceFunc(func() {
 		ephemeraljsc, _ = syncutil.New(syncutil.WithSize(NonPoolingVMConcurrency))
 	})

pkg/js/compiler/pool.go

@@ -2,6 +2,7 @@ package compiler
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"fmt"
 	"reflect"
@@ -55,10 +56,12 @@ var (
 	lazySgInit = sync.OnceFunc(func() {
 		pooljsc, _ = syncutil.New(syncutil.WithSize(PoolingJsVmConcurrency))
 	})
-	sgResizeCheck = func() {
+	sgResizeCheck = func(ctx context.Context) {
 		// resize check point
 		if pooljsc.Size != PoolingJsVmConcurrency {
-			pooljsc.Resize(PoolingJsVmConcurrency)
+			if err := pooljsc.Resize(ctx, PoolingJsVmConcurrency); err != nil {
+				gologger.Warning().Msgf("Could not resize workpool: %s\n", err)
+			}
 		}
 	}
 )
@@ -122,7 +125,7 @@ func executeWithPoolingProgram(p *goja.Program, args *ExecuteArgs, opts *Execute
 	// its unknown (most likely cannot be done) to limit max js runtimes at a moment without making it static
 	// unlike sync.Pool which reacts to GC and its purposes is to reuse objects rather than creating new ones
 	lazySgInit()
-	sgResizeCheck()
+	sgResizeCheck(opts.Context)
 
 	pooljsc.Add()
 	defer pooljsc.Done()

pkg/js/libs/postgres/memo.postgres.go

@@ -5,7 +5,7 @@ import (
 	"errors"
 	"fmt"
 
-	_ "github.com/lib/pq"
+	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/utils/pgwrap"
 
 	utils "github.com/projectdiscovery/nuclei/v3/pkg/js/utils"
 

pkg/js/libs/postgres/postgres.go

@@ -9,10 +9,11 @@ import (
 	"time"
 
 	"github.com/go-pg/pg"
-	_ "github.com/lib/pq"
 	"github.com/praetorian-inc/fingerprintx/pkg/plugins"
 	postgres "github.com/praetorian-inc/fingerprintx/pkg/plugins/services/postgresql"
 	utils "github.com/projectdiscovery/nuclei/v3/pkg/js/utils"
+	"github.com/projectdiscovery/nuclei/v3/pkg/js/utils/pgwrap"
+	_ "github.com/projectdiscovery/nuclei/v3/pkg/js/utils/pgwrap"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 )
 
@@ -100,10 +101,11 @@ func executeQuery(host string, port int, username string, password string, dbNam
 	target := net.JoinHostPort(host, fmt.Sprintf("%d", port))
 
 	connStr := fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=disable", username, password, target, dbName)
-	db, err := sql.Open("postgres", connStr)
+	db, err := sql.Open(pgwrap.PGWrapDriver, connStr)
 	if err != nil {
 		return nil, err
 	}
+	defer db.Close()
 
 	rows, err := db.Query(query)
 	if err != nil {
@@ -143,12 +145,19 @@ func connect(host string, port int, username string, password string, dbName str
 
 	target := net.JoinHostPort(host, fmt.Sprintf("%d", port))
 
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
 	db := pg.Connect(&pg.Options{
-		Addr:     target,
-		User:     username,
-		Password: password,
-		Database: dbName,
-	})
+		Addr:               target,
+		User:               username,
+		Password:           password,
+		Database:           dbName,
+		Dialer: func(network, addr string) (net.Conn, error) {
+			return protocolstate.Dialer.Dial(context.Background(), network, addr)
+		},
+		IdleCheckFrequency: -1,
+	}).WithContext(ctx).WithTimeout(10 * time.Second)
 	defer db.Close()
 
 	_, err := db.Exec("select 1")

pkg/js/utils/pgwrap/pgwrap.go

@@ -0,0 +1,53 @@
+package pgwrap
+
+import (
+	"context"
+	"database/sql"
+	"database/sql/driver"
+	"net"
+	"time"
+
+	"github.com/lib/pq"
+	"github.com/projectdiscovery/fastdialer/fastdialer"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
+)
+
+const (
+	PGWrapDriver = "pgwrap"
+)
+
+type pgDial struct {
+	fd *fastdialer.Dialer
+}
+
+func (p *pgDial) Dial(network, address string) (net.Conn, error) {
+	return p.fd.Dial(context.TODO(), network, address)
+}
+
+func (p *pgDial) DialTimeout(network, address string, timeout time.Duration) (net.Conn, error) {
+	ctx, cancel := context.WithTimeoutCause(context.Background(), timeout, fastdialer.ErrDialTimeout)
+	defer cancel()
+	return p.fd.Dial(ctx, network, address)
+}
+
+func (p *pgDial) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
+	return p.fd.Dial(ctx, network, address)
+}
+
+// Unfortunately lib/pq does not provide easy to customize or
+// replace dialer so we need to hijack it by wrapping it in our own
+// driver and register it as postgres driver
+
+// PgDriver is the Postgres database driver.
+type PgDriver struct{}
+
+// Open opens a new connection to the database. name is a connection string.
+// Most users should only use it through database/sql package from the standard
+// library.
+func (d PgDriver) Open(name string) (driver.Conn, error) {
+	return pq.DialOpen(&pgDial{fd: protocolstate.Dialer}, name)
+}
+
+func init() {
+	sql.Register(PGWrapDriver, &PgDriver{})
+}

pkg/model/model_test.go

@@ -80,7 +80,7 @@ func TestUnmarshal(t *testing.T) {
 	templateName := "Test Template"
 	authors := []string{"forgedhallpass", "ice3man"}
 	tags := []string{"cve", "misc"}
-	references := []string{"http://test.com", "http://domain.com"}
+	references := []string{"http://test.com", "http://Domain.com"}
 
 	dynamicKey1 := "customDynamicKey1"
 	dynamicKey2 := "customDynamicKey2"
@@ -109,7 +109,7 @@ func TestUnmarshal(t *testing.T) {
   author: ` + strings.Join(authors, ", ") + `
   tags: ` + strings.Join(tags, ", ") + `
   severity: critical
-  reference: ` + strings.Join(references, ", ") + `
+  reference: ` + strings.Join(references, ",") + `
   metadata:
      ` + dynamicKey1 + `: ` + dynamicKeysMap[dynamicKey1].(string) + `
      ` + dynamicKey2 + `: ` + dynamicKeysMap[dynamicKey2].(string) + `

pkg/model/types/severity/severities.go

@@ -26,6 +26,14 @@ func (severities *Severities) Set(values string) error {
 	return nil
 }
 
+func (severities Severities) MarshalYAML() (interface{}, error) {
+	var stringSeverities = make([]string, 0, len(severities))
+	for _, severity := range severities {
+		stringSeverities = append(stringSeverities, severity.String())
+	}
+	return stringSeverities, nil
+}
+
 func (severities *Severities) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	var stringSliceValue stringslice.StringSlice
 	if err := unmarshal(&stringSliceValue); err != nil {

pkg/model/types/severity/severity_test.go

@@ -74,3 +74,12 @@ func TestMarshalJSON(t *testing.T) {
 	}
 	require.Equal(t, "[\"low\",\"medium\"]", string(data), "could not marshal json")
 }
+
+func TestSeveritiesMarshalYaml(t *testing.T) {
+	unmarshalled := Severities{Low, Medium}
+	marshalled, err := yaml.Marshal(unmarshalled)
+	if err != nil {
+		panic(err)
+	}
+	require.Equal(t, "- low\n- medium\n", string(marshalled), "could not marshal yaml")
+}

pkg/model/types/stringslice/stringslice_raw.go

@@ -12,6 +12,15 @@ func (rawStringSlice *RawStringSlice) Normalize(value string) string {
 	return value
 }
 
+func (rawStringSlice *RawStringSlice) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	marshalledSlice, err := marshalStringToSlice(unmarshal)
+	if err != nil {
+		return err
+	}
+	rawStringSlice.Value = marshalledSlice
+	return nil
+}
+
 func (rawStringSlice RawStringSlice) JSONSchemaAlias() any {
 	return StringOrSlice("")
 }

pkg/output/format_screen.go

@@ -106,5 +106,21 @@ func (w *StandardWriter) formatScreen(output *ResultEvent) []byte {
 		}
 		builder.WriteString("]")
 	}
+
+	// If it is a fuzzing output, enrich with additional
+	// metadata for the match.
+	if output.IsFuzzingResult {
+		if output.FuzzingParameter != "" {
+			builder.WriteString(" [")
+			builder.WriteString(output.FuzzingPosition)
+			builder.WriteRune(':')
+			builder.WriteString(w.aurora.BrightMagenta(output.FuzzingParameter).String())
+			builder.WriteString("]")
+		}
+
+		builder.WriteString(" [")
+		builder.WriteString(output.FuzzingMethod)
+		builder.WriteString("]")
+	}
 	return builder.Bytes()
 }

pkg/output/output.go

@@ -4,6 +4,7 @@ import (
 	"encoding/base64"
 	"fmt"
 	"io"
+	"log/slog"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -27,9 +28,12 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators"
 	protocolUtils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	"github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils"
+	"github.com/projectdiscovery/utils/errkit"
 	fileutil "github.com/projectdiscovery/utils/file"
 	osutils "github.com/projectdiscovery/utils/os"
+	urlutil "github.com/projectdiscovery/utils/url"
 )
 
 // Writer is an interface which writes output to somewhere for nuclei events.
@@ -175,6 +179,14 @@ type ResultEvent struct {
 	// must be enabled by setting protocols.ExecuterOptions.ExportReqURLPattern to true
 	ReqURLPattern string `json:"req_url_pattern,omitempty"`
 
+	// Fields related to HTTP Fuzzing functionality of nuclei.
+	// The output contains additional fields when the result is
+	// for a fuzzing template.
+	IsFuzzingResult  bool   `json:"is_fuzzing_result,omitempty"`
+	FuzzingMethod    string `json:"fuzzing_method,omitempty"`
+	FuzzingParameter string `json:"fuzzing_parameter,omitempty"`
+	FuzzingPosition  string `json:"fuzzing_position,omitempty"`
+
 	FileToIndexPosition map[string]int `json:"-"`
 	Error               string         `json:"error,omitempty"`
 }
@@ -291,10 +303,13 @@ func (w *StandardWriter) Write(event *ResultEvent) error {
 
 // JSONLogRequest is a trace/error log request written to file
 type JSONLogRequest struct {
-	Template string `json:"template"`
-	Input    string `json:"input"`
-	Error    string `json:"error"`
-	Type     string `json:"type"`
+	Template string      `json:"template"`
+	Type     string      `json:"type"`
+	Input    string      `json:"input"`
+	Address  string      `json:"address"`
+	Error    string      `json:"error"`
+	Kind     string      `json:"kind,omitempty"`
+	Attrs    interface{} `json:"attrs,omitempty"`
 }
 
 // Request writes a log the requests trace log
@@ -307,12 +322,43 @@ func (w *StandardWriter) Request(templatePath, input, requestType string, reques
 		Input:    input,
 		Type:     requestType,
 	}
-	if unwrappedErr := utils.UnwrapError(requestErr); unwrappedErr != nil {
-		request.Error = unwrappedErr.Error()
+	parsed, _ := urlutil.ParseAbsoluteURL(input, false)
+	if parsed != nil {
+		request.Address = parsed.Hostname()
+		port := parsed.Port()
+		if port == "" {
+			switch parsed.Scheme {
+			case urlutil.HTTP:
+				port = "80"
+			case urlutil.HTTPS:
+				port = "443"
+			}
+		}
+		request.Address += ":" + port
+	}
+	errX := errkit.FromError(requestErr)
+	if errX == nil {
+		request.Error = "none"
 	} else {
-		request.Error = "none"
+		request.Kind = errkit.ErrKindUnknown.String()
+		var cause error
+		if len(errX.Errors()) > 1 {
+			cause = errX.Errors()[0]
+		}
+		if cause == nil {
+			cause = errX
+		}
+		cause = tryParseCause(cause)
+		request.Error = cause.Error()
+		request.Kind = errkit.GetErrorKind(requestErr, nucleierr.ErrTemplateLogic).String()
+		if len(errX.Attrs()) > 0 {
+			request.Attrs = slog.GroupValue(errX.Attrs()...)
+		}
+	}
+	// check if address slog attr is avaiable in error if set use it
+	if val := errkit.GetAttrValue(requestErr, "address"); val.Any() != nil {
+		request.Address = val.String()
 	}
-
 	data, err := jsoniter.Marshal(request)
 	if err != nil {
 		return
@@ -444,3 +490,24 @@ func (w *StandardWriter) WriteStoreDebugData(host, templateID, eventType string,
 	}
 
 }
+
+// tryParseCause tries to parse the cause of given error
+// this is legacy support due to use of errorutil in existing libraries
+// but this should not be required once all libraries are updated
+func tryParseCause(err error) error {
+	if err == nil {
+		return nil
+	}
+	msg := err.Error()
+	if strings.HasPrefix(msg, "ReadStatusLine:") {
+		// last index is actual error (from rawhttp)
+		parts := strings.Split(msg, ":")
+		return errkit.New(strings.TrimSpace(parts[len(parts)-1]))
+	}
+	if strings.Contains(msg, "read ") {
+		// same here
+		parts := strings.Split(msg, ":")
+		return errkit.New(strings.TrimSpace(parts[len(parts)-1]))
+	}
+	return err
+}

pkg/output/output_test.go

@@ -30,7 +30,7 @@ func TestStandardWriterRequest(t *testing.T) {
 		require.NoError(t, err)
 		w.Request("path", "input", "http", nil)
 
-		require.Equal(t, `{"template":"path","input":"input","error":"none","type":"http"}`, traceWriter.String())
+		require.Equal(t, `{"template":"path","type":"http","input":"input","address":"input:","error":"none"}`, traceWriter.String())
 		require.Empty(t, errorWriter.String())
 	})
 
@@ -47,7 +47,7 @@ func TestStandardWriterRequest(t *testing.T) {
 			fmt.Errorf("GET https://example.com/tcpconfig.html/tcpconfig.html giving up after 2 attempts: %w", errors.New("context deadline exceeded (Client.Timeout exceeded while awaiting headers)")),
 		)
 
-		require.Equal(t, `{"template":"misconfiguration/tcpconfig.yaml","input":"https://example.com/tcpconfig.html","error":"context deadline exceeded (Client.Timeout exceeded while awaiting headers)","type":"http"}`, errorWriter.String())
+		require.Equal(t, `{"template":"misconfiguration/tcpconfig.yaml","type":"http","input":"https://example.com/tcpconfig.html","address":"example.com:443","error":"context deadline exceeded (Client.Timeout exceeded while awaiting headers)","kind":"unknown-error"}`, errorWriter.String())
 	})
 }
 

pkg/protocols/code/code.go

@@ -32,6 +32,7 @@ import (
 	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	contextutil "github.com/projectdiscovery/utils/context"
+	"github.com/projectdiscovery/utils/errkit"
 	errorutil "github.com/projectdiscovery/utils/errors"
 )
 
@@ -41,7 +42,10 @@ const (
 )
 
 var (
+	// pythonEnvRegexCompiled is the compiled regex for python environment variables
 	pythonEnvRegexCompiled = regexp.MustCompile(pythonEnvRegex)
+	// ErrCodeExecutionDeadline is the error returned when alloted time for script execution exceeds
+	ErrCodeExecutionDeadline = errkit.New("code execution deadline exceeded").SetKind(errkit.ErrKindDeadline).Build()
 )
 
 // Request is a request for the SSL protocol
@@ -199,6 +203,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 				Source:   &request.PreCondition,
 				Callback: registerPreConditionFunctions,
 				Cleanup:  cleanUpPreConditionFunctions,
+				Context:  input.Context(),
 			})
 		if err != nil {
 			return errorutil.NewWithTag(request.TemplateID, "could not execute pre-condition: %s", err)
@@ -213,7 +218,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 		}
 	}
 
-	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second)
+	ctx, cancel := context.WithTimeoutCause(input.Context(), time.Duration(timeout)*time.Second, ErrCodeExecutionDeadline)
 	defer cancel()
 	// Note: we use contextutil despite the fact that gozero accepts context as argument
 	gOutput, err := contextutil.ExecFuncWithTwoReturns(ctx, func() (*gozerotypes.Result, error) {

pkg/protocols/code/code_test.go

@@ -3,6 +3,7 @@
 package code
 
 import (
+	"context"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -31,7 +32,7 @@ func TestCodeProtocol(t *testing.T) {
 	require.Nil(t, err, "could not compile code request")
 
 	var gotEvent output.InternalEvent
-	ctxArgs := contextargs.NewWithInput("")
+	ctxArgs := contextargs.NewWithInput(context.Background(), "")
 	err = request.ExecuteWithResults(ctxArgs, nil, nil, func(event *output.InternalWrappedEvent) {
 		gotEvent = event.InternalEvent
 	})

pkg/protocols/common/automaticscan/automaticscan.go

@@ -1,6 +1,7 @@
 package automaticscan
 
 import (
+	"context"
 	"io"
 	"net/http"
 	"os"
@@ -55,7 +56,6 @@ type Service struct {
 	engine             *core.Engine
 	target             provider.InputProvider
 	wappalyzer         *wappalyzer.Wappalyze
-	childExecuter      *core.ChildExecuter
 	httpclient         *retryablehttp.Client
 	templateDirs       []string // root Template Directories
 	technologyMappings map[string]string
@@ -94,7 +94,6 @@ func New(opts Options) (*Service, error) {
 		return nil, err
 	}
 
-	childExecuter := opts.Engine.ChildExecuter()
 	httpclient, err := httpclientpool.Get(opts.ExecuterOpts.Options, &httpclientpool.Configuration{
 		Connection: &httpclientpool.ConnectionConfiguration{
 			DisableKeepAlive: httputil.ShouldDisableKeepAlive(opts.ExecuterOpts.Options),
@@ -110,7 +109,6 @@ func New(opts Options) (*Service, error) {
 		target:             opts.Target,
 		wappalyzer:         wappalyzer,
 		templateDirs:       templateDirs, // fix this
-		childExecuter:      childExecuter,
 		httpclient:         httpclient,
 		technologyMappings: mappingData,
 		techTemplates:      techDetectTemplates,
@@ -189,7 +187,7 @@ func (s *Service) executeAutomaticScanOnTarget(input *contextargs.MetaInput) {
 	execOptions.Progress = &testutils.MockProgressClient{} // stats are not supported yet due to centralized logic and cannot be reinitialized
 	eng.SetExecuterOptions(execOptions)
 
-	tmp := eng.ExecuteScanWithOpts(finalTemplates, provider.NewSimpleInputProviderWithUrls(input.Input), true)
+	tmp := eng.ExecuteScanWithOpts(context.Background(), finalTemplates, provider.NewSimpleInputProviderWithUrls(input.Input), true)
 	s.hasResults.Store(tmp.Load())
 }
 
@@ -244,7 +242,9 @@ func (s *Service) getTagsUsingWappalyzer(input *contextargs.MetaInput) []string
 
 // getTagsUsingDetectionTemplates returns tags using detection templates
 func (s *Service) getTagsUsingDetectionTemplates(input *contextargs.MetaInput) ([]string, int) {
-	ctxArgs := contextargs.NewWithInput(input.Input)
+	ctx := context.Background()
+
+	ctxArgs := contextargs.NewWithInput(ctx, input.Input)
 
 	// execute tech detection templates on target
 	tags := map[string]struct{}{}
@@ -256,7 +256,7 @@ func (s *Service) getTagsUsingDetectionTemplates(input *contextargs.MetaInput) (
 		sg.Add()
 		go func(template *templates.Template) {
 			defer sg.Done()
-			ctx := scan.NewScanContext(ctxArgs)
+			ctx := scan.NewScanContext(ctx, ctxArgs)
 			ctx.OnResult = func(event *output.InternalWrappedEvent) {
 				if event == nil {
 					return

pkg/protocols/common/contextargs/contextargs.go

@@ -1,6 +1,7 @@
 package contextargs
 
 import (
+	"context"
 	"net/http/cookiejar"
 	"strings"
 	"sync/atomic"
@@ -19,6 +20,8 @@ var (
 
 // Context implements a shared context struct to share information across multiple templates within a workflow
 type Context struct {
+	ctx context.Context
+
 	// Meta is the target for the executor
 	MetaInput *MetaInput
 
@@ -30,17 +33,25 @@ type Context struct {
 }
 
 // Create a new contextargs instance
-func New() *Context {
-	return NewWithInput("")
+func New(ctx context.Context) *Context {
+	return NewWithInput(ctx, "")
+}
+
+// NewWithMetaInput creates a new contextargs instance with meta input
+func NewWithMetaInput(ctx context.Context, input *MetaInput) *Context {
+	n := New(ctx)
+	n.MetaInput = input
+	return n
 }
 
 // Create a new contextargs instance with input string
-func NewWithInput(input string) *Context {
+func NewWithInput(ctx context.Context, input string) *Context {
 	jar, err := cookiejar.New(nil)
 	if err != nil {
 		gologger.Error().Msgf("contextargs: could not create cookie jar: %s\n", err)
 	}
 	return &Context{
+		ctx:       ctx,
 		MetaInput: &MetaInput{Input: input},
 		CookieJar: jar,
 		args: &mapsutil.SyncLockMap[string, interface{}]{
@@ -50,6 +61,11 @@ func NewWithInput(input string) *Context {
 	}
 }
 
+// Context returns the context of the current contextargs
+func (ctx *Context) Context() context.Context {
+	return ctx.ctx
+}
+
 // Set the specific key-value pair
 func (ctx *Context) Set(key string, value interface{}) {
 	_ = ctx.args.Set(key, value)
@@ -158,9 +174,27 @@ func (ctx *Context) HasArgs() bool {
 
 func (ctx *Context) Clone() *Context {
 	newCtx := &Context{
+		ctx:       ctx.ctx,
 		MetaInput: ctx.MetaInput.Clone(),
 		args:      ctx.args.Clone(),
 		CookieJar: ctx.CookieJar,
 	}
 	return newCtx
 }
+
+// GetCopyIfHostOutdated returns a new contextargs if the host is outdated
+func GetCopyIfHostOutdated(ctx *Context, url string) *Context {
+	if ctx.MetaInput.Input == "" {
+		newctx := ctx.Clone()
+		newctx.MetaInput.Input = url
+		return newctx
+	}
+	orig, _ := urlutil.Parse(ctx.MetaInput.Input)
+	newURL, _ := urlutil.Parse(url)
+	if orig != nil && newURL != nil && orig.Host != newURL.Host {
+		newCtx := ctx.Clone()
+		newCtx.MetaInput.Input = newURL.Host
+		return newCtx
+	}
+	return ctx
+}

pkg/protocols/common/contextargs/metainput.go

@@ -4,11 +4,13 @@ import (
 	"bytes"
 	"crypto/md5"
 	"fmt"
+	"net"
 	"strings"
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/types"
 	urlutil "github.com/projectdiscovery/utils/url"
+	"github.com/segmentio/ksuid"
 )
 
 // MetaInput represents a target with metadata (TODO: replace with https://github.com/projectdiscovery/metainput)
@@ -47,6 +49,56 @@ func (metaInput *MetaInput) URL() (*urlutil.URL, error) {
 	return instance, nil
 }
 
+// Port returns the port of the target
+// if port is not present then empty string is returned
+func (metaInput *MetaInput) Port() string {
+	target, err := urlutil.ParseAbsoluteURL(metaInput.Input, false)
+	if err != nil {
+		return ""
+	}
+	return target.Port()
+}
+
+// Address return the remote address of target
+// Note: it does not resolve the domain to ip
+// it is meant to be used by hosterrorsCache if invalid metainput
+// is provided then it uses a random ksuid as hostname to avoid skipping valid targets
+func (metaInput *MetaInput) Address() string {
+	var hostname, port string
+	target, err := urlutil.ParseAbsoluteURL(metaInput.Target(), false)
+	if err != nil {
+		if metaInput.CustomIP == "" {
+			// since this is used in hosterrorscache we add a random id
+			// which will never be used to avoid skipping valid targets
+			hostname = fmt.Sprintf("invalid-%s", ksuid.New().String())
+		}
+	} else {
+		hostname = target.Hostname()
+		port = target.Port()
+		if port == "" {
+			switch target.Scheme {
+			case urlutil.HTTP:
+				port = "80"
+			case urlutil.HTTPS:
+				port = "443"
+			default:
+				port = "80"
+			}
+		}
+	}
+	if metaInput.CustomIP != "" {
+		hostname = metaInput.CustomIP
+	}
+	if port == "" {
+		if strings.HasPrefix(hostname, "http://") {
+			port = "80"
+		} else if strings.HasPrefix(hostname, "https://") {
+			port = "443"
+		}
+	}
+	return net.JoinHostPort(hostname, port)
+}
+
 // ID returns a unique id/hash for metainput
 func (metaInput *MetaInput) ID() string {
 	if metaInput.CustomIP != "" {

pkg/protocols/common/hosterrorscache/hosterrorscache.go

@@ -10,17 +10,25 @@ import (
 
 	"github.com/bluele/gcache"
 	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
+	"github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr"
+	"github.com/projectdiscovery/utils/errkit"
 )
 
 // CacheInterface defines the signature of the hosterrorscache so that
 // users of Nuclei as embedded lib may implement their own cache
 type CacheInterface interface {
-	SetVerbose(verbose bool)            // log verbosely
-	Close()                             // close the cache
-	Check(value string) bool            // return true if the host should be skipped
-	MarkFailed(value string, err error) // record a failure (and cause) for the host
+	SetVerbose(verbose bool)                        // log verbosely
+	Close()                                         // close the cache
+	Check(ctx *contextargs.Context) bool            // return true if the host should be skipped
+	MarkFailed(ctx *contextargs.Context, err error) // record a failure (and cause) for the host
 }
 
+var (
+	_ CacheInterface = (*Cache)(nil)
+)
+
 // Cache is a cache for host based errors. It allows skipping
 // certain hosts based on an error threshold.
 //
@@ -34,8 +42,10 @@ type Cache struct {
 }
 
 type cacheItem struct {
-	errors atomic.Int32
 	sync.Once
+	errors         atomic.Int32
+	isPermanentErr bool
+	cause          error // optional cause
 }
 
 const DefaultMaxHostsCount = 10000
@@ -55,6 +65,16 @@ func (c *Cache) SetVerbose(verbose bool) {
 
 // Close closes the host errors cache
 func (c *Cache) Close() {
+	if config.DefaultConfig.IsDebugArgEnabled(config.DebugArgHostErrorStats) {
+		items := c.failedTargets.GetALL(false)
+		for k, v := range items {
+			val, ok := v.(*cacheItem)
+			if !ok {
+				continue
+			}
+			gologger.Info().Label("MaxHostErrorStats").Msgf("Host: %s, Errors: %d", k, val.errors.Load())
+		}
+	}
 	c.failedTargets.Purge()
 }
 
@@ -88,14 +108,19 @@ func (c *Cache) normalizeCacheValue(value string) string {
 //   - URL: https?:// type
 //   - Host:port type
 //   - host type
-func (c *Cache) Check(value string) bool {
-	finalValue := c.normalizeCacheValue(value)
+func (c *Cache) Check(ctx *contextargs.Context) bool {
+	finalValue := c.GetKeyFromContext(ctx, nil)
 
 	existingCacheItem, err := c.failedTargets.GetIFPresent(finalValue)
 	if err != nil {
 		return false
 	}
 	existingCacheItemValue := existingCacheItem.(*cacheItem)
+	if existingCacheItemValue.isPermanentErr {
+		// skipping permanent errors is expected so verbose instead of info
+		gologger.Verbose().Msgf("Skipped %s from target list as found unresponsive permanently: %s", finalValue, existingCacheItemValue.cause)
+		return true
+	}
 
 	if existingCacheItemValue.errors.Load() >= int32(c.MaxHostError) {
 		existingCacheItemValue.Do(func() {
@@ -107,15 +132,22 @@ func (c *Cache) Check(value string) bool {
 }
 
 // MarkFailed marks a host as failed previously
-func (c *Cache) MarkFailed(value string, err error) {
+func (c *Cache) MarkFailed(ctx *contextargs.Context, err error) {
 	if !c.checkError(err) {
 		return
 	}
-	finalValue := c.normalizeCacheValue(value)
+	finalValue := c.GetKeyFromContext(ctx, err)
 	existingCacheItem, err := c.failedTargets.GetIFPresent(finalValue)
 	if err != nil || existingCacheItem == nil {
 		newItem := &cacheItem{errors: atomic.Int32{}}
 		newItem.errors.Store(1)
+		if errkit.IsKind(err, errkit.ErrKindNetworkPermanent) {
+			// skip this address altogether
+			// permanent errors are always permanent hence this is created once
+			// and never updated so no need to synchronize
+			newItem.isPermanentErr = true
+			newItem.cause = err
+		}
 		_ = c.failedTargets.Set(finalValue, newItem)
 		return
 	}
@@ -124,19 +156,77 @@ func (c *Cache) MarkFailed(value string, err error) {
 	_ = c.failedTargets.Set(finalValue, existingCacheItemValue)
 }
 
-var reCheckError = regexp.MustCompile(`(no address found for host|Client\.Timeout exceeded while awaiting headers|could not resolve host|connection refused|connection reset by peer)`)
+// GetKeyFromContext returns the key for the cache from the context
+func (c *Cache) GetKeyFromContext(ctx *contextargs.Context, err error) string {
+	// Note:
+	// ideally any changes made to remote addr in template like {{Hostname}}:81 etc
+	// should be reflected in contextargs but it is not yet reflected in some cases
+	// and needs refactor of ScanContext + ContextArgs to achieve that
+	// i.e why we use real address from error if present
+	address := ctx.MetaInput.Address()
+	// get address override from error
+	if err != nil {
+		tmp := errkit.GetAttrValue(err, "address")
+		if tmp.Any() != nil {
+			address = tmp.String()
+		}
+	}
+	finalValue := c.normalizeCacheValue(address)
+	return finalValue
+}
+
+var reCheckError = regexp.MustCompile(`(no address found for host|could not resolve host|connection refused|connection reset by peer|could not connect to any address found for host|timeout awaiting response headers)`)
 
 // checkError checks if an error represents a type that should be
 // added to the host skipping table.
+// it first parses error and extracts the cause and checks for blacklisted
+// or common errors that should be skipped
 func (c *Cache) checkError(err error) bool {
 	if err == nil {
 		return false
 	}
-	errString := err.Error()
-	for _, msg := range c.TrackError {
-		if strings.Contains(errString, msg) {
-			return true
+	kind := errkit.GetErrorKind(err, nucleierr.ErrTemplateLogic)
+	switch kind {
+	case nucleierr.ErrTemplateLogic:
+		// these are errors that are not related to the target
+		// and are due to template logic
+		return false
+	case errkit.ErrKindNetworkTemporary:
+		// these should not be counted as host errors
+		return false
+	case errkit.ErrKindNetworkPermanent:
+		// these should be counted as host errors
+		return true
+	case errkit.ErrKindDeadline:
+		// these should not be counted as host errors
+		return false
+	default:
+		// parse error for furthur processing
+		errX := errkit.FromError(err)
+		tmp := errX.Cause()
+		cause := tmp.Error()
+		if strings.Contains(cause, "ReadStatusLine:") && strings.Contains(cause, "read: connection reset by peer") {
+			// this is a FP and should not be counted as a host error
+			// because server closes connection when it reads corrupted bytes which we send via rawhttp
+			return false
 		}
+		if strings.HasPrefix(cause, "ReadStatusLine:") {
+			// error is present in last part when using rawhttp
+			// this will be fixed once errkit is used everywhere
+			lastIndex := strings.LastIndex(cause, ":")
+			if lastIndex == -1 {
+				lastIndex = 0
+			}
+			if lastIndex >= len(cause)-1 {
+				lastIndex = 0
+			}
+			cause = cause[lastIndex+1:]
+		}
+		for _, msg := range c.TrackError {
+			if strings.Contains(cause, msg) {
+				return true
+			}
+		}
+		return reCheckError.MatchString(cause)
 	}
-	return reCheckError.MatchString(errString)
 }

pkg/protocols/common/hosterrorscache/hosterrorscache_test.go

@@ -1,11 +1,13 @@
 package hosterrorscache
 
 import (
+	"context"
 	"fmt"
 	"sync"
 	"sync/atomic"
 	"testing"
 
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
 	"github.com/stretchr/testify/require"
 )
 
@@ -13,8 +15,8 @@ func TestCacheCheck(t *testing.T) {
 	cache := New(3, DefaultMaxHostsCount, nil)
 
 	for i := 0; i < 100; i++ {
-		cache.MarkFailed("test", fmt.Errorf("could not resolve host"))
-		got := cache.Check("test")
+		cache.MarkFailed(newCtxArgs("test"), fmt.Errorf("could not resolve host"))
+		got := cache.Check(newCtxArgs("test"))
 		if i < 2 {
 			// till 3 the host is not flagged to skip
 			require.False(t, got)
@@ -24,7 +26,7 @@ func TestCacheCheck(t *testing.T) {
 		}
 	}
 
-	value := cache.Check("test")
+	value := cache.Check(newCtxArgs("test"))
 	require.Equal(t, true, value, "could not get checked value")
 }
 
@@ -32,8 +34,8 @@ func TestTrackErrors(t *testing.T) {
 	cache := New(3, DefaultMaxHostsCount, []string{"custom error"})
 
 	for i := 0; i < 100; i++ {
-		cache.MarkFailed("custom", fmt.Errorf("got: nested: custom error"))
-		got := cache.Check("custom")
+		cache.MarkFailed(newCtxArgs("custom"), fmt.Errorf("got: nested: custom error"))
+		got := cache.Check(newCtxArgs("custom"))
 		if i < 2 {
 			// till 3 the host is not flagged to skip
 			require.False(t, got)
@@ -42,7 +44,7 @@ func TestTrackErrors(t *testing.T) {
 			require.True(t, got)
 		}
 	}
-	value := cache.Check("custom")
+	value := cache.Check(newCtxArgs("custom"))
 	require.Equal(t, true, value, "could not get checked value")
 }
 
@@ -73,16 +75,18 @@ func TestCacheMarkFailed(t *testing.T) {
 
 	tests := []struct {
 		host     string
-		expected int
+		expected int32
 	}{
 		{"http://example.com:80", 1},
 		{"example.com:80", 2},
-		{"example.com", 1},
+		// earlier if port is not provided then port was omitted
+		// but from now it will default to appropriate http scheme based port with 80 as default
+		{"example.com:443", 1},
 	}
 
 	for _, test := range tests {
-		normalizedCacheValue := cache.normalizeCacheValue(test.host)
-		cache.MarkFailed(test.host, fmt.Errorf("no address found for host"))
+		normalizedCacheValue := cache.GetKeyFromContext(newCtxArgs(test.host), nil)
+		cache.MarkFailed(newCtxArgs(test.host), fmt.Errorf("no address found for host"))
 		failedTarget, err := cache.failedTargets.Get(normalizedCacheValue)
 		require.Nil(t, err)
 		require.NotNil(t, failedTarget)
@@ -102,7 +106,7 @@ func TestCacheMarkFailedConcurrent(t *testing.T) {
 	}{
 		{"http://example.com:80", 200},
 		{"example.com:80", 200},
-		{"example.com", 100},
+		{"example.com:443", 100},
 	}
 
 	// the cache is not atomic during items creation, so we pre-create them with counter to zero
@@ -120,14 +124,14 @@ func TestCacheMarkFailedConcurrent(t *testing.T) {
 			wg.Add(1)
 			go func() {
 				defer wg.Done()
-				cache.MarkFailed(currentTest.host, fmt.Errorf("could not resolve host"))
+				cache.MarkFailed(newCtxArgs(currentTest.host), fmt.Errorf("could not resolve host"))
 			}()
 		}
 	}
 	wg.Wait()
 
 	for _, test := range tests {
-		require.True(t, cache.Check(test.host))
+		require.True(t, cache.Check(newCtxArgs(test.host)))
 
 		normalizedCacheValue := cache.normalizeCacheValue(test.host)
 		failedTarget, err := cache.failedTargets.Get(normalizedCacheValue)
@@ -139,3 +143,8 @@ func TestCacheMarkFailedConcurrent(t *testing.T) {
 		require.EqualValues(t, test.expected, value.errors.Load())
 	}
 }
+
+func newCtxArgs(value string) *contextargs.Context {
+	ctx := contextargs.NewWithInput(context.TODO(), value)
+	return ctx
+}

pkg/protocols/common/protocolinit/init.go

@@ -13,7 +13,6 @@ import (
 
 // Init initializes the client pools for the protocols
 func Init(options *types.Options) error {
-
 	if err := protocolstate.Init(options); err != nil {
 		return err
 	}
@@ -40,4 +39,5 @@ func Init(options *types.Options) error {
 
 func Close() {
 	protocolstate.Dialer.Close()
+	protocolstate.Dialer = nil
 }

pkg/protocols/common/protocolstate/memguardian.go

@@ -1,6 +1,7 @@
 package protocolstate
 
 import (
+	"context"
 	"sync"
 	"time"
 
@@ -14,20 +15,27 @@ var (
 	MaxThreadsOnLowMemory          = env.GetEnvOrDefault("MEMGUARDIAN_THREADS", 0)
 	MaxBytesBufferAllocOnLowMemory = env.GetEnvOrDefault("MEMGUARDIAN_ALLOC", 0)
 	memTimer                       *time.Ticker
+	cancelFunc                     context.CancelFunc
 )
 
-func StartActiveMemGuardian() {
+func StartActiveMemGuardian(ctx context.Context) {
 	if memguardian.DefaultMemGuardian == nil {
 		return
 	}
 
-	memTimer := time.NewTicker(memguardian.DefaultInterval)
+	memTimer = time.NewTicker(memguardian.DefaultInterval)
+	ctx, cancelFunc = context.WithCancel(ctx)
 	go func() {
-		for range memTimer.C {
-			if IsLowOnMemory() {
-				_ = GlobalGuardBytesBufferAlloc()
-			} else {
-				GlobalRestoreBytesBufferAlloc()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-memTimer.C:
+				if IsLowOnMemory() {
+					_ = GlobalGuardBytesBufferAlloc()
+				} else {
+					GlobalRestoreBytesBufferAlloc()
+				}
 			}
 		}
 	}()
@@ -38,7 +46,10 @@ func StopActiveMemGuardian() {
 		return
 	}
 
-	memTimer.Stop()
+	if memTimer != nil {
+		memTimer.Stop()
+		cancelFunc()
+	}
 }
 
 func IsLowOnMemory() bool {

pkg/protocols/common/protocolstate/state.go

@@ -148,7 +148,7 @@ func Init(options *types.Options) error {
 		return Dialer.Dial(ctx, "tcp", addr)
 	})
 
-	StartActiveMemGuardian()
+	StartActiveMemGuardian(context.Background())
 
 	return nil
 }
@@ -180,6 +180,7 @@ func interfaceAddress(interfaceName string) (net.IP, error) {
 		if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
 			if ipnet.IP.To4() != nil {
 				address = ipnet.IP
+				break
 			}
 		}
 	}
@@ -209,6 +210,7 @@ func interfaceAddresses(interfaceName string) ([]net.Addr, error) {
 func Close() {
 	if Dialer != nil {
 		Dialer.Close()
+		Dialer = nil
 	}
 	StopActiveMemGuardian()
 }

pkg/protocols/common/uncover/uncover.go

@@ -11,6 +11,7 @@ import (
 	"github.com/projectdiscovery/uncover"
 	"github.com/projectdiscovery/uncover/sources"
 	mapsutil "github.com/projectdiscovery/utils/maps"
+	sliceutil "github.com/projectdiscovery/utils/slice"
 	stringsutil "github.com/projectdiscovery/utils/strings"
 )
 
@@ -84,9 +85,21 @@ func GetUncoverTargetsFromMetadata(ctx context.Context, templates []*templates.T
 			if queriesMap[engine] == nil {
 				queriesMap[engine] = []string{}
 			}
-			queriesMap[engine] = append(queriesMap[engine], fmt.Sprint(v))
+			switch v := v.(type) {
+			case []interface{}:
+				qs := queriesMap[engine]
+				for _, vv := range v {
+					qs = append(qs, fmt.Sprint(vv))
+				}
+				queriesMap[engine] = qs
+			default:
+				queriesMap[engine] = append(queriesMap[engine], fmt.Sprint(v))
+			}
 		}
 	}
+	for engine, queries := range queriesMap {
+		queriesMap[engine] = sliceutil.Dedupe(queries)
+	}
 	keys := mapsutil.GetKeys(queriesMap)
 	gologger.Info().Msgf("Running uncover queries from template against: %s", strings.Join(keys, ","))
 	result := make(chan string, runtime.NumCPU())
@@ -97,8 +110,11 @@ func GetUncoverTargetsFromMetadata(ctx context.Context, templates []*templates.T
 		// TODO: add support for map[engine]queries in uncover
 		// Note below implementation is intentionally sequential to avoid burning all the API keys
 		counter := 0
-
+	outerLoop:
 		for eng, queries := range queriesMap {
+			if opts.Limit > 0 && counter >= opts.Limit {
+				break
+			}
 			// create new uncover options for each engine
 			uncoverOpts := &uncover.Options{
 				Agents:        []string{eng},
@@ -114,18 +130,20 @@ func GetUncoverTargetsFromMetadata(ctx context.Context, templates []*templates.T
 				gologger.Error().Msgf("Could not get targets using %v engine from uncover: %s", eng, err)
 				return
 			}
+
+		innerLoop:
 			for {
 				select {
 				case <-ctx.Done():
 					return
 				case res, ok := <-ch:
 					if !ok {
-						return
+						continue outerLoop
 					}
 					result <- res
 					counter++
 					if opts.Limit > 0 && counter >= opts.Limit {
-						return
+						break innerLoop
 					}
 				}
 			}

pkg/protocols/dns/request.go

@@ -80,9 +80,17 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
 				break
 			}
 
+			select {
+			case <-input.Context().Done():
+				return input.Context().Err()
+			default:
+			}
+
 			// resize check point - nop if there are no changes
 			if shouldFollowGlobal && swg.Size != request.options.Options.PayloadConcurrency {
-				swg.Resize(request.options.Options.PayloadConcurrency)
+				if err := swg.Resize(input.Context(), request.options.Options.PayloadConcurrency); err != nil {
+					return err
+				}
 			}
 
 			value = generators.MergeMaps(vars, value)

pkg/protocols/dns/request_test.go

@@ -1,6 +1,7 @@
 package dns
 
 import (
+	"context"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -37,7 +38,7 @@ func TestDNSExecuteWithResults(t *testing.T) {
 				Name:  "test",
 				Part:  "raw",
 				Type:  matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
-				Words: []string{"93.184.216.34"},
+				Words: []string{"93.184.215.14"},
 			}},
 			Extractors: []*extractors.Extractor{{
 				Part:  "raw",
@@ -54,7 +55,7 @@ func TestDNSExecuteWithResults(t *testing.T) {
 	t.Run("domain-valid", func(t *testing.T) {
 		metadata := make(output.InternalEvent)
 		previous := make(output.InternalEvent)
-		ctxArgs := contextargs.NewWithInput("example.com")
+		ctxArgs := contextargs.NewWithInput(context.Background(), "example.com")
 		err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
 			finalEvent = event
 		})
@@ -64,13 +65,13 @@ func TestDNSExecuteWithResults(t *testing.T) {
 	require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results")
 	require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results")
 	require.Equal(t, 1, len(finalEvent.Results[0].ExtractedResults), "could not get correct number of extracted results")
-	require.Equal(t, "93.184.216.34", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
+	require.Equal(t, "93.184.215.14", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
 	finalEvent = nil
 
 	t.Run("url-to-domain", func(t *testing.T) {
 		metadata := make(output.InternalEvent)
 		previous := make(output.InternalEvent)
-		err := request.ExecuteWithResults(contextargs.NewWithInput("https://example.com"), metadata, previous, func(event *output.InternalWrappedEvent) {
+		err := request.ExecuteWithResults(contextargs.NewWithInput(context.Background(), "https://example.com"), metadata, previous, func(event *output.InternalWrappedEvent) {
 			finalEvent = event
 		})
 		require.Nil(t, err, "could not execute dns request")
@@ -79,6 +80,6 @@ func TestDNSExecuteWithResults(t *testing.T) {
 	require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results")
 	require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results")
 	require.Equal(t, 1, len(finalEvent.Results[0].ExtractedResults), "could not get correct number of extracted results")
-	require.Equal(t, "93.184.216.34", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
+	require.Equal(t, "93.184.215.14", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
 	finalEvent = nil
 }

pkg/protocols/file/request_test.go

@@ -1,6 +1,7 @@
 package file
 
 import (
+	"context"
 	"os"
 	"path/filepath"
 	"testing"
@@ -67,7 +68,7 @@ func TestFileExecuteWithResults(t *testing.T) {
 	t.Run("valid", func(t *testing.T) {
 		metadata := make(output.InternalEvent)
 		previous := make(output.InternalEvent)
-		ctxArgs := contextargs.NewWithInput(tempDir)
+		ctxArgs := contextargs.NewWithInput(context.Background(), tempDir)
 		err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
 			finalEvent = event
 		})

pkg/protocols/headless/engine/page_actions.go

@@ -24,6 +24,7 @@ import (
 	protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
 	httputil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/http"
 	contextutil "github.com/projectdiscovery/utils/context"
+	"github.com/projectdiscovery/utils/errkit"
 	errorutil "github.com/projectdiscovery/utils/errors"
 	fileutil "github.com/projectdiscovery/utils/file"
 	folderutil "github.com/projectdiscovery/utils/folder"
@@ -35,6 +36,8 @@ import (
 var (
 	errinvalidArguments = errorutil.New("invalid arguments provided")
 	ErrLFAccessDenied   = errorutil.New("Use -allow-local-file-access flag to enable local file access")
+	// ErrActionExecDealine is the error returned when alloted time for action execution exceeds
+	ErrActionExecDealine = errkit.New("headless action execution deadline exceeded").SetKind(errkit.ErrKindDeadline).Build()
 )
 
 const (
@@ -619,7 +622,7 @@ func (p *Page) WaitEvent(act *Action, out map[string]string) (func() error, erro
 	// Just wait the event to happen
 	waitFunc := func() (err error) {
 		// execute actual wait event
-		ctx, cancel := context.WithTimeout(context.Background(), maxDuration)
+		ctx, cancel := context.WithTimeoutCause(context.Background(), maxDuration, ErrActionExecDealine)
 		defer cancel()
 		err = contextutil.ExecFunc(ctx, p.page.WaitEvent(waitEvent))
 		return

pkg/protocols/headless/engine/page_actions_test.go

@@ -1,6 +1,7 @@
 package engine
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"math/rand"
@@ -595,7 +596,7 @@ func testHeadless(t *testing.T, actions []*Action, timeout time.Duration, handle
 	ts := httptest.NewServer(http.HandlerFunc(handler))
 	defer ts.Close()
 
-	input := contextargs.NewWithInput(ts.URL)
+	input := contextargs.NewWithInput(context.Background(), ts.URL)
 	input.CookieJar, err = cookiejar.New(nil)
 	require.Nil(t, err)
 
@@ -674,7 +675,7 @@ func TestBlockedHeadlessURLS(t *testing.T) {
 			{ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}},
 		}
 
-		data, page, err := instance.Run(contextargs.NewWithInput(ts.URL), actions, nil, &Options{Timeout: 20 * time.Second, Options: opts}) // allow file access in test
+		data, page, err := instance.Run(contextargs.NewWithInput(context.Background(), ts.URL), actions, nil, &Options{Timeout: 20 * time.Second, Options: opts}) // allow file access in test
 		require.Error(t, err, "expected error for url %s got %v", testcase, data)
 		require.True(t, stringsutil.ContainsAny(err.Error(), "net::ERR_ACCESS_DENIED", "failed to parse url", "Cannot navigate to invalid URL", "net::ERR_ABORTED", "net::ERR_INVALID_URL"), "found different error %v for testcases %v", err, testcase)
 		require.Len(t, data, 0, "expected no data for url %s got %v", testcase, data)

pkg/protocols/headless/request.go

@@ -44,7 +44,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
 		if err != nil {
 			return err
 		}
-		input = contextargs.NewWithInput(url)
+		input = contextargs.NewWithInput(input.Context(), url)
 	}
 
 	if request.options.Browser.UserAgent() == "" {

pkg/protocols/http/build_request.go

@@ -277,7 +277,7 @@ func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, data st
 			return nil, fmt.Errorf("malformed request supplied")
 		}
 
-		if err := expressions.ContainsUnresolvedVariables(parts[1]); err != nil {
+		if err := expressions.ContainsUnresolvedVariables(parts[1]); err != nil && !r.request.SkipVariablesCheck {
 			return nil, ErrUnresolvedVars.Msgf(parts[1])
 		}
 
@@ -296,7 +296,7 @@ func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, data st
 		}
 		return r.generateRawRequest(ctx, data, parsed, values, payloads)
 	}
-	if err := expressions.ContainsUnresolvedVariables(data); err != nil {
+	if err := expressions.ContainsUnresolvedVariables(data); err != nil && !r.request.SkipVariablesCheck {
 		// early exit: if there are any unresolved variables in `path` after evaluation
 		// then return early since this will definitely fail
 		return nil, ErrUnresolvedVars.Msgf(data)

pkg/protocols/http/build_request_test.go

@@ -40,7 +40,7 @@ func TestMakeRequestFromModal(t *testing.T) {
 
 	generator := request.newGenerator(false)
 	inputData, payloads, _ := generator.nextValue()
-	req, err := generator.Make(context.Background(), contextargs.NewWithInput("https://example.com"), inputData, payloads, map[string]interface{}{})
+	req, err := generator.Make(context.Background(), contextargs.NewWithInput(context.Background(), "https://example.com"), inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	if req.request.URL == nil {
 		t.Fatalf("url is nil in generator make")
@@ -70,13 +70,13 @@ func TestMakeRequestFromModalTrimSuffixSlash(t *testing.T) {
 
 	generator := request.newGenerator(false)
 	inputData, payloads, _ := generator.nextValue()
-	req, err := generator.Make(context.Background(), contextargs.NewWithInput("https://example.com/test.php"), inputData, payloads, map[string]interface{}{})
+	req, err := generator.Make(context.Background(), contextargs.NewWithInput(context.Background(), "https://example.com/test.php"), inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	require.Equal(t, "https://example.com/test.php?query=example", req.request.URL.String(), "could not get correct request path")
 
 	generator = request.newGenerator(false)
 	inputData, payloads, _ = generator.nextValue()
-	req, err = generator.Make(context.Background(), contextargs.NewWithInput("https://example.com/test/"), inputData, payloads, map[string]interface{}{})
+	req, err = generator.Make(context.Background(), contextargs.NewWithInput(context.Background(), "https://example.com/test/"), inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	require.Equal(t, "https://example.com/test/?query=example", req.request.URL.String(), "could not get correct request path")
 }
@@ -110,13 +110,13 @@ Accept-Encoding: gzip`},
 
 	generator := request.newGenerator(false)
 	inputData, payloads, _ := generator.nextValue()
-	req, err := generator.Make(context.Background(), contextargs.NewWithInput("https://example.com"), inputData, payloads, map[string]interface{}{})
+	req, err := generator.Make(context.Background(), contextargs.NewWithInput(context.Background(), "https://example.com"), inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	authorization := req.request.Header.Get("Authorization")
 	require.Equal(t, "Basic admin:admin", authorization, "could not get correct authorization headers from raw")
 
 	inputData, payloads, _ = generator.nextValue()
-	req, err = generator.Make(context.Background(), contextargs.NewWithInput("https://example.com"), inputData, payloads, map[string]interface{}{})
+	req, err = generator.Make(context.Background(), contextargs.NewWithInput(context.Background(), "https://example.com"), inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	authorization = req.request.Header.Get("Authorization")
 	require.Equal(t, "Basic admin:guest", authorization, "could not get correct authorization headers from raw")
@@ -151,13 +151,13 @@ Accept-Encoding: gzip`},
 
 	generator := request.newGenerator(false)
 	inputData, payloads, _ := generator.nextValue()
-	req, err := generator.Make(context.Background(), contextargs.NewWithInput("https://example.com"), inputData, payloads, map[string]interface{}{})
+	req, err := generator.Make(context.Background(), contextargs.NewWithInput(context.Background(), "https://example.com"), inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	authorization := req.request.Header.Get("Authorization")
 	require.Equal(t, "Basic YWRtaW46YWRtaW4=", authorization, "could not get correct authorization headers from raw")
 
 	inputData, payloads, _ = generator.nextValue()
-	req, err = generator.Make(context.Background(), contextargs.NewWithInput("https://example.com"), inputData, payloads, map[string]interface{}{})
+	req, err = generator.Make(context.Background(), contextargs.NewWithInput(context.Background(), "https://example.com"), inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	authorization = req.request.Header.Get("Authorization")
 	require.Equal(t, "Basic YWRtaW46Z3Vlc3Q=", authorization, "could not get correct authorization headers from raw")
@@ -195,7 +195,7 @@ func TestMakeRequestFromModelUniqueInteractsh(t *testing.T) {
 	require.Nil(t, err, "could not create interactsh client")
 
 	inputData, payloads, _ := generator.nextValue()
-	got, err := generator.Make(context.Background(), contextargs.NewWithInput("https://example.com"), inputData, payloads, map[string]interface{}{})
+	got, err := generator.Make(context.Background(), contextargs.NewWithInput(context.Background(), "https://example.com"), inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 
 	// check if all the interactsh markers are replaced with unique urls

pkg/protocols/http/http.go

@@ -9,7 +9,6 @@ import (
 	json "github.com/json-iterator/go"
 	"github.com/pkg/errors"
 
-	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/fuzz"
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators"
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
@@ -19,6 +18,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
 	httputil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/http"
+	"github.com/projectdiscovery/nuclei/v3/pkg/utils/stats"
 	"github.com/projectdiscovery/rawhttp"
 	"github.com/projectdiscovery/retryablehttp-go"
 	fileutil "github.com/projectdiscovery/utils/file"
@@ -455,7 +455,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
 			}
 		}
 		if hasNamedInternalExtractor && hasMultipleRequests {
-			gologger.Warning().Label(options.TemplateID).Msgf("Setting thread count to 0 because dynamic extractors are not supported with payloads yet")
+			stats.Increment(SetThreadToCountZero)
 			request.Threads = 0
 		} else {
 			// specifically for http requests high concurrency and and threads will lead to memory exausthion, hence reduce the maximum parallelism
@@ -490,3 +490,11 @@ func (request *Request) Requests() int {
 	}
 	return len(request.Path)
 }
+
+const (
+	SetThreadToCountZero = "set-thread-count-to-zero"
+)
+
+func init() {
+	stats.NewEntry(SetThreadToCountZero, "Setting thread count to 0 for %d templates, dynamic extractors are not supported with payloads yet")
+}

pkg/protocols/http/httpclientpool/clientpool.go

@@ -35,14 +35,28 @@ var (
 	forceMaxRedirects int
 	normalClient      *retryablehttp.Client
 	clientPool        *mapsutil.SyncLockMap[string, *retryablehttp.Client]
+	// MaxResponseHeaderTimeout is the timeout for response headers
+	// to be read from the server (this prevents infinite hang started by server if any)
+	// Note: this will be overridden temporarily when using @timeout request annotation
+	MaxResponseHeaderTimeout = time.Duration(10) * time.Second
+	// HttpTimeoutMultiplier is the multiplier for the http timeout
+	HttpTimeoutMultiplier = 3
 )
 
+// GetHttpTimeout returns the http timeout for the client
+func GetHttpTimeout(opts *types.Options) time.Duration {
+	return time.Duration(opts.Timeout*HttpTimeoutMultiplier) * time.Second
+}
+
 // Init initializes the clientpool implementation
 func Init(options *types.Options) error {
 	// Don't create clients if already created in the past.
 	if normalClient != nil {
 		return nil
 	}
+	if options.Timeout > 10 {
+		MaxResponseHeaderTimeout = time.Duration(options.Timeout) * time.Second
+	}
 	if options.ShouldFollowHTTPRedirects() {
 		forceMaxRedirects = options.MaxRedirects
 	}
@@ -101,6 +115,8 @@ type Configuration struct {
 	RedirectFlow RedirectFlow
 	// Connection defines custom connection configuration
 	Connection *ConnectionConfiguration
+	// ResponseHeaderTimeout is the timeout for response body to be read from the server
+	ResponseHeaderTimeout time.Duration
 }
 
 // Hash returns the hash of the configuration to allow client pooling
@@ -119,13 +135,15 @@ func (c *Configuration) Hash() string {
 	builder.WriteString(strconv.FormatBool(c.DisableCookie))
 	builder.WriteString("c")
 	builder.WriteString(strconv.FormatBool(c.Connection != nil))
+	builder.WriteString("r")
+	builder.WriteString(strconv.FormatInt(int64(c.ResponseHeaderTimeout.Seconds()), 10))
 	hash := builder.String()
 	return hash
 }
 
 // HasStandardOptions checks whether the configuration requires custom settings
 func (c *Configuration) HasStandardOptions() bool {
-	return c.Threads == 0 && c.MaxRedirects == 0 && c.RedirectFlow == DontFollowRedirect && c.DisableCookie && c.Connection == nil && !c.NoTimeout
+	return c.Threads == 0 && c.MaxRedirects == 0 && c.RedirectFlow == DontFollowRedirect && c.DisableCookie && c.Connection == nil && !c.NoTimeout && c.ResponseHeaderTimeout == 0
 }
 
 // GetRawHTTP returns the rawhttp request client
@@ -139,7 +157,7 @@ func GetRawHTTP(options *types.Options) *rawhttp.Client {
 		} else if Dialer != nil {
 			rawHttpOptions.FastDialer = Dialer
 		}
-		rawHttpOptions.Timeout = time.Duration(options.Timeout) * time.Second
+		rawHttpOptions.Timeout = GetHttpTimeout(options)
 		rawHttpClient = rawhttp.NewClient(rawHttpOptions)
 	}
 	return rawHttpClient
@@ -172,6 +190,9 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
 	maxIdleConns := 0
 	maxConnsPerHost := 0
 	maxIdleConnsPerHost := -1
+	// do not split given timeout into chunks for retry
+	// because this won't work on slow hosts
+	retryableHttpOptions.NoAdjustTimeout = true
 
 	if configuration.Threads > 0 || options.ScanStrategy == scanstrategy.HostSpray.String() {
 		// Single host
@@ -225,6 +246,12 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
 		return nil, errors.Wrap(err, "could not create client certificate")
 	}
 
+	// responseHeaderTimeout is max timeout for response headers to be read
+	responseHeaderTimeout := MaxResponseHeaderTimeout
+	if configuration.ResponseHeaderTimeout != 0 {
+		responseHeaderTimeout = configuration.ResponseHeaderTimeout
+	}
+
 	transport := &http.Transport{
 		ForceAttemptHTTP2: options.ForceAttemptHTTP2,
 		DialContext:       Dialer.Dial,
@@ -237,11 +264,12 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
 			}
 			return Dialer.DialTLS(ctx, network, addr)
 		},
-		MaxIdleConns:        maxIdleConns,
-		MaxIdleConnsPerHost: maxIdleConnsPerHost,
-		MaxConnsPerHost:     maxConnsPerHost,
-		TLSClientConfig:     tlsConfig,
-		DisableKeepAlives:   disableKeepAlives,
+		MaxIdleConns:          maxIdleConns,
+		MaxIdleConnsPerHost:   maxIdleConnsPerHost,
+		MaxConnsPerHost:       maxConnsPerHost,
+		TLSClientConfig:       tlsConfig,
+		DisableKeepAlives:     disableKeepAlives,
+		ResponseHeaderTimeout: responseHeaderTimeout,
 	}
 
 	if types.ProxyURL != "" {
@@ -288,7 +316,7 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
 		CheckRedirect: makeCheckRedirectFunc(redirectFlow, maxRedirects),
 	}
 	if !configuration.NoTimeout {
-		httpclient.Timeout = time.Duration(options.Timeout) * time.Second
+		httpclient.Timeout = GetHttpTimeout(options)
 	}
 	client := retryablehttp.NewWithHTTPClient(httpclient, retryableHttpOptions)
 	if jar != nil {
@@ -340,7 +368,7 @@ func makeCheckRedirectFunc(redirectType RedirectFlow, maxRedirects int) checkRed
 	}
 }
 
-func checkMaxRedirects(req *http.Request, via []*http.Request, maxRedirects int) error {
+func checkMaxRedirects(_ *http.Request, via []*http.Request, maxRedirects int) error {
 	if maxRedirects == 0 {
 		if len(via) > defaultMaxRedirects {
 			return http.ErrUseLastResponse

pkg/protocols/http/httpclientpool/options.go

@@ -0,0 +1,54 @@
+package httpclientpool
+
+import (
+	"io"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/projectdiscovery/rawhttp"
+)
+
+// WithCustomTimeout is a configuration for custom timeout
+type WithCustomTimeout struct {
+	Timeout time.Duration
+}
+
+// RawHttpRequestOpts is a configuration for raw http request
+type RawHttpRequestOpts struct {
+	// Method is the http method to use
+	Method string
+	// URL is the url to request
+	URL string
+	// Path is request path to use
+	Path string
+	// Headers is the headers to use
+	Headers map[string][]string
+	// Body is the body to use
+	Body io.Reader
+	// Options is more client related options
+	Options *rawhttp.Options
+}
+
+// SendRawRequest sends a raw http request with the provided options and returns http response
+func SendRawRequest(client *rawhttp.Client, opts *RawHttpRequestOpts) (*http.Response, error) {
+	resp, err := client.DoRawWithOptions(opts.Method, opts.URL, opts.Path, opts.Headers, opts.Body, opts.Options)
+	if err != nil {
+		cause := err.Error()
+		if strings.Contains(cause, "ReadStatusLine: ") && strings.Contains(cause, "read: connection reset by peer") {
+			// this error is caused when rawhttp client sends a corrupted or malformed request packet to server
+			// some servers may attempt gracefully shutdown but most will just abruptly close the connection which results
+			// in a connection reset by peer error and this can be safely assumed as 400 Bad Request in terms of normal http flow
+			req, _ := http.NewRequest(opts.Method, opts.URL, opts.Body)
+			req.Header = opts.Headers
+			resp = &http.Response{
+				Request:    req,
+				StatusCode: http.StatusBadRequest,
+				Status:     http.StatusText(http.StatusBadRequest),
+				Body:       io.NopCloser(strings.NewReader("")),
+			}
+			return resp, nil
+		}
+	}
+	return resp, err
+}

pkg/protocols/http/httputils/spm.go

@@ -5,6 +5,7 @@ import (
 	"sync"
 
 	syncutil "github.com/projectdiscovery/utils/sync"
+	"golang.org/x/exp/maps"
 )
 
 // WorkPoolType is the type of work pool to use
@@ -19,7 +20,7 @@ const (
 
 // StopAtFirstMatchHandler is a handler that executes
 // request and stops on first match
-type StopAtFirstMatchHandler[T any] struct {
+type StopAtFirstMatchHandler[T comparable] struct {
 	once sync.Once
 	// Result Channel
 	ResultChan chan T
@@ -33,12 +34,14 @@ type StopAtFirstMatchHandler[T any] struct {
 	ctx         context.Context
 	cancel      context.CancelFunc
 	internalWg  *sync.WaitGroup
-	results     []T
+	results     map[T]struct{}
+	onResult    func(T)
 	stopEnabled bool
+	maxResults  int
 }
 
 // NewBlockingSPMHandler creates a new stop at first match handler
-func NewBlockingSPMHandler[T any](ctx context.Context, size int, spm bool) *StopAtFirstMatchHandler[T] {
+func NewBlockingSPMHandler[T comparable](ctx context.Context, size int, maxResults int, spm bool) *StopAtFirstMatchHandler[T] {
 	ctx1, cancel := context.WithCancel(ctx)
 
 	awg, _ := syncutil.New(syncutil.WithSize(size))
@@ -51,6 +54,8 @@ func NewBlockingSPMHandler[T any](ctx context.Context, size int, spm bool) *Stop
 		ctx:         ctx1,
 		cancel:      cancel,
 		stopEnabled: spm,
+		results:     make(map[T]struct{}),
+		maxResults:  maxResults,
 	}
 	s.internalWg.Add(1)
 	go s.run(ctx)
@@ -58,7 +63,7 @@ func NewBlockingSPMHandler[T any](ctx context.Context, size int, spm bool) *Stop
 }
 
 // NewNonBlockingSPMHandler creates a new stop at first match handler
-func NewNonBlockingSPMHandler[T any](ctx context.Context, spm bool) *StopAtFirstMatchHandler[T] {
+func NewNonBlockingSPMHandler[T comparable](ctx context.Context, maxResults int, spm bool) *StopAtFirstMatchHandler[T] {
 	ctx1, cancel := context.WithCancel(ctx)
 	s := &StopAtFirstMatchHandler[T]{
 		ResultChan:  make(chan T, 1),
@@ -68,6 +73,8 @@ func NewNonBlockingSPMHandler[T any](ctx context.Context, spm bool) *StopAtFirst
 		ctx:         ctx1,
 		cancel:      cancel,
 		stopEnabled: spm,
+		results:     make(map[T]struct{}),
+		maxResults:  maxResults,
 	}
 	s.internalWg.Add(1)
 	go s.run(ctx)
@@ -82,6 +89,25 @@ func (h *StopAtFirstMatchHandler[T]) Trigger() {
 	}
 }
 
+// Cancel cancels spm context
+func (h *StopAtFirstMatchHandler[T]) Cancel() {
+	h.cancel()
+}
+
+// SetOnResult callback
+// this is not thread safe
+func (h *StopAtFirstMatchHandler[T]) SetOnResultCallback(fn func(T)) {
+	if h.onResult != nil {
+		tmp := h.onResult
+		h.onResult = func(t T) {
+			tmp(t)
+			fn(t)
+		}
+	} else {
+		h.onResult = fn
+	}
+}
+
 // MatchCallback is called when a match is found
 // input fn should be the callback that is intended to be called
 // if stop at first is enabled and other conditions are met
@@ -97,14 +123,23 @@ func (h *StopAtFirstMatchHandler[T]) MatchCallback(fn func()) {
 // run runs the internal handler
 func (h *StopAtFirstMatchHandler[T]) run(ctx context.Context) {
 	defer h.internalWg.Done()
+
 	for {
 		select {
 		case <-ctx.Done():
+			return
 		case val, ok := <-h.ResultChan:
 			if !ok {
 				return
 			}
-			h.results = append(h.results, val)
+			if h.onResult != nil {
+				h.onResult(val)
+			}
+			if len(h.results) >= h.maxResults {
+				// skip or do not store the result
+				continue
+			}
+			h.results[val] = struct{}{}
 		}
 	}
 }
@@ -114,6 +149,11 @@ func (h *StopAtFirstMatchHandler[T]) Done() <-chan struct{} {
 	return h.ctx.Done()
 }
 
+// Cancelled returns true if the context is cancelled
+func (h *StopAtFirstMatchHandler[T]) Cancelled() bool {
+	return h.ctx.Err() != nil
+}
+
 // FoundFirstMatch returns true if first match was found
 // in stop at first match mode
 func (h *StopAtFirstMatchHandler[T]) FoundFirstMatch() bool {
@@ -143,10 +183,11 @@ func (h *StopAtFirstMatchHandler[T]) Release() {
 	}
 }
 
-func (h *StopAtFirstMatchHandler[T]) Resize(size int) {
+func (h *StopAtFirstMatchHandler[T]) Resize(ctx context.Context, size int) error {
 	if h.sgPool.Size != size {
-		h.sgPool.Resize(size)
+		return h.sgPool.Resize(ctx, size)
 	}
+	return nil
 }
 
 func (h *StopAtFirstMatchHandler[T]) Size() int {
@@ -168,5 +209,5 @@ func (h *StopAtFirstMatchHandler[T]) Wait() {
 
 // CombinedResults returns the combined results
 func (h *StopAtFirstMatchHandler[T]) CombinedResults() []T {
-	return h.results
+	return maps.Keys(h.results)
 }

pkg/protocols/http/request.go

@@ -35,8 +35,10 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signerpool"
 	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	"github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr"
 	"github.com/projectdiscovery/rawhttp"
 	convUtil "github.com/projectdiscovery/utils/conversion"
+	"github.com/projectdiscovery/utils/errkit"
 	errorutil "github.com/projectdiscovery/utils/errors"
 	httpUtils "github.com/projectdiscovery/utils/http"
 	"github.com/projectdiscovery/utils/reader"
@@ -47,12 +49,17 @@ import (
 
 const (
 	defaultMaxWorkers = 150
+	// max unique errors to store & combine
+	// when executing requests in parallel
+	maxErrorsWhenParallel = 3
 )
 
 var (
-	MaxBodyRead = int64(1 << 22) // 4MB using shift operator
+	MaxBodyRead = int64(10 * 1024 * 1024) // 10MB
 	// ErrMissingVars is error occured when variables are missing
-	ErrMissingVars = errors.New("stop execution due to unresolved variables")
+	ErrMissingVars = errkit.New("stop execution due to unresolved variables").SetKind(nucleierr.ErrTemplateLogic).Build()
+	// ErrHttpEngineRequestDeadline is error occured when request deadline set by http request engine is exceeded
+	ErrHttpEngineRequestDeadline = errkit.New("http request engine deadline exceeded").SetKind(errkit.ErrKindDeadline).Build()
 )
 
 // Type returns the type of the protocol request
@@ -111,7 +118,13 @@ func (request *Request) executeRaceRequest(input *contextargs.Context, previous
 	}
 
 	shouldStop := (request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch || request.options.StopAtFirstMatch)
-	spmHandler := httputils.NewNonBlockingSPMHandler[error](ctx, shouldStop)
+
+	childCtx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	spmHandler := httputils.NewNonBlockingSPMHandler[error](childCtx, maxErrorsWhenParallel, shouldStop)
+	defer spmHandler.Cancel()
+
 	gotMatches := &atomic.Bool{}
 	// wrappedCallback is a callback that wraps the original callback
 	// to implement stop at first match logic
@@ -132,12 +145,29 @@ func (request *Request) executeRaceRequest(input *contextargs.Context, previous
 		}
 	}
 
+	// look for unresponsive hosts and cancel inflight requests as well
+	spmHandler.SetOnResultCallback(func(err error) {
+		if err == nil {
+			return
+		}
+		// marks thsi host as unresponsive if applicable
+		request.markUnresponsiveAddress(input, err)
+		if request.isUnresponsiveAddress(input) {
+			// stop all inflight requests
+			spmHandler.Cancel()
+		}
+	})
+
 	for i := 0; i < request.RaceNumberRequests; i++ {
+		if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(input) {
+			// stop sending more requests condition is met
+			break
+		}
 		spmHandler.Acquire()
 		// execute http request
 		go func(httpRequest *generatedRequest) {
 			defer spmHandler.Release()
-			if spmHandler.FoundFirstMatch() {
+			if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(input) {
 				// stop sending more requests condition is met
 				return
 			}
@@ -175,7 +205,13 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 	// Stop-at-first-match logic while executing requests
 	// parallely using threads
 	shouldStop := (request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch || request.options.StopAtFirstMatch)
-	spmHandler := httputils.NewBlockingSPMHandler[error](context.Background(), maxWorkers, shouldStop)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	spmHandler := httputils.NewBlockingSPMHandler[error](ctx, maxWorkers, maxErrorsWhenParallel, shouldStop)
+	defer spmHandler.Cancel()
+
 	// wrappedCallback is a callback that wraps the original callback
 	// to implement stop at first match logic
 	wrappedCallback := func(event *output.InternalWrappedEvent) {
@@ -194,6 +230,19 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 		}
 	}
 
+	// look for unresponsive hosts and cancel inflight requests as well
+	spmHandler.SetOnResultCallback(func(err error) {
+		if err == nil {
+			return
+		}
+		// marks thsi host as unresponsive if applicable
+		request.markUnresponsiveAddress(input, err)
+		if request.isUnresponsiveAddress(input) {
+			// stop all inflight requests
+			spmHandler.Cancel()
+		}
+	})
+
 	// iterate payloads and make requests
 	generator := request.newGenerator(false)
 	for {
@@ -202,9 +251,22 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 			break
 		}
 
+		select {
+		case <-input.Context().Done():
+			return input.Context().Err()
+		default:
+		}
+
 		// resize check point - nop if there are no changes
 		if shouldFollowGlobal && spmHandler.Size() != request.options.Options.PayloadConcurrency {
-			spmHandler.Resize(request.options.Options.PayloadConcurrency)
+			if err := spmHandler.Resize(input.Context(), request.options.Options.PayloadConcurrency); err != nil {
+				return err
+			}
+		}
+
+		// break if stop at first match is found or host is unresponsive
+		if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(input) {
+			break
 		}
 
 		ctx := request.newContext(input)
@@ -219,22 +281,30 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 		if input.MetaInput.Input == "" {
 			input.MetaInput.Input = generatedHttpRequest.URL()
 		}
+		updatedInput := contextargs.GetCopyIfHostOutdated(input, generatedHttpRequest.URL())
+		if request.isUnresponsiveAddress(updatedInput) {
+			// skip on unresponsive host no need to continue
+			spmHandler.Cancel()
+			return nil
+		}
 		spmHandler.Acquire()
 		go func(httpRequest *generatedRequest) {
 			defer spmHandler.Release()
-			if spmHandler.FoundFirstMatch() {
+			if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(updatedInput) || spmHandler.Cancelled() {
+				return
+			}
+			// putting ratelimiter here prevents any unnecessary waiting if any
+			request.options.RateLimitTake()
+
+			// after ratelimit take, check if we need to stop
+			if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(updatedInput) || spmHandler.Cancelled() {
 				return
 			}
 
 			select {
 			case <-spmHandler.Done():
 				return
-			case spmHandler.ResultChan <- func() error {
-				// putting ratelimiter here prevents any unnecessary waiting if any
-				request.options.RateLimitTake()
-				previous := make(map[string]interface{})
-				return request.executeRequest(input, httpRequest, previous, false, wrappedCallback, 0)
-			}():
+			case spmHandler.ResultChan <- request.executeRequest(input, httpRequest, make(map[string]interface{}), false, wrappedCallback, 0):
 				return
 			}
 		}(generatedHttpRequest)
@@ -276,12 +346,16 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
 		maxWorkers = pipeOptions.MaxPendingRequests
 	}
 
-	// Stop-at-first-match logic while executing requests
-	// parallely using threads
 	// Stop-at-first-match logic while executing requests
 	// parallely using threads
 	shouldStop := (request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch || request.options.StopAtFirstMatch)
-	spmHandler := httputils.NewBlockingSPMHandler[error](context.Background(), maxWorkers, shouldStop)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	spmHandler := httputils.NewBlockingSPMHandler[error](ctx, maxWorkers, maxErrorsWhenParallel, shouldStop)
+	defer spmHandler.Cancel()
+
 	// wrappedCallback is a callback that wraps the original callback
 	// to implement stop at first match logic
 	wrappedCallback := func(event *output.InternalWrappedEvent) {
@@ -300,11 +374,36 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
 		}
 	}
 
+	// look for unresponsive hosts and cancel inflight requests as well
+	spmHandler.SetOnResultCallback(func(err error) {
+		if err == nil {
+			return
+		}
+		// marks thsi host as unresponsive if applicable
+		request.markUnresponsiveAddress(input, err)
+		if request.isUnresponsiveAddress(input) {
+			// stop all inflight requests
+			spmHandler.Cancel()
+		}
+	})
+
 	for {
 		inputData, payloads, ok := generator.nextValue()
 		if !ok {
 			break
 		}
+
+		select {
+		case <-input.Context().Done():
+			return input.Context().Err()
+		default:
+		}
+
+		if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(input) || spmHandler.Cancelled() {
+			// skip if first match is found
+			break
+		}
+
 		ctx := request.newContext(input)
 		generatedHttpRequest, err := generator.Make(ctx, input, inputData, payloads, dynamicValues)
 		if err != nil {
@@ -314,15 +413,20 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
 		if input.MetaInput.Input == "" {
 			input.MetaInput.Input = generatedHttpRequest.URL()
 		}
+		updatedInput := contextargs.GetCopyIfHostOutdated(input, generatedHttpRequest.URL())
+		if request.isUnresponsiveAddress(updatedInput) {
+			// skip on unresponsive host no need to continue
+			spmHandler.Cancel()
+			return nil
+		}
 		generatedHttpRequest.pipelinedClient = pipeClient
 		spmHandler.Acquire()
 		go func(httpRequest *generatedRequest) {
 			defer spmHandler.Release()
-			if spmHandler.FoundFirstMatch() {
+			if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(updatedInput) {
 				// skip if first match is found
 				return
 			}
-
 			select {
 			case <-spmHandler.Done():
 				return
@@ -378,8 +482,9 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 			request.options.RateLimitTake()
 
 			ctx := request.newContext(input)
-			ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Duration(request.options.Options.Timeout)*time.Second)
+			ctxWithTimeout, cancel := context.WithTimeoutCause(ctx, httpclientpool.GetHttpTimeout(request.options.Options), ErrHttpEngineRequestDeadline)
 			defer cancel()
+
 			generatedHttpRequest, err := generator.Make(ctxWithTimeout, input, data, payloads, dynamicValue)
 			if err != nil {
 				if err == types.ErrNoMoreRequests {
@@ -388,6 +493,12 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 				request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
 				return true, err
 			}
+			// ideally if http template used a custom port or hostname
+			// we would want to update it in input but currently templateCtx logic
+			// is closely tied to contextargs.Context so we are temporarily creating
+			// a copy and using it to check for host errors etc
+			// but this should be replaced once templateCtx is refactored properly
+			updatedInput := contextargs.GetCopyIfHostOutdated(input, generatedHttpRequest.URL())
 
 			if generatedHttpRequest.customCancelFunction != nil {
 				defer generatedHttpRequest.customCancelFunction()
@@ -398,7 +509,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 				input.MetaInput.Input = generatedHttpRequest.URL()
 			}
 			// Check if hosts keep erroring
-			if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input.MetaInput.ID()) {
+			if request.isUnresponsiveAddress(updatedInput) {
 				return true, nil
 			}
 			var gotMatches bool
@@ -437,12 +548,13 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 				return true, nil
 			}
 			if execReqErr != nil {
-				if request.options.HostErrorsCache != nil {
-					request.options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err)
-				}
+				// if applicable mark the host as unresponsive
+				request.markUnresponsiveAddress(updatedInput, execReqErr)
 				requestErr = errorutil.NewWithErr(execReqErr).Msgf("got err while executing %v", generatedHttpRequest.URL())
+				request.options.Progress.IncrementFailedRequestsBy(1)
+			} else {
+				request.options.Progress.IncrementRequests()
 			}
-			request.options.Progress.IncrementRequests()
 
 			// If this was a match, and we want to stop at first match, skip all further requests.
 			shouldStopAtFirstMatch := generatedHttpRequest.original.options.Options.StopAtFirstMatch || generatedHttpRequest.original.options.StopAtFirstMatch || request.StopAtFirstMatch
@@ -456,6 +568,13 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 		if !ok {
 			break
 		}
+
+		select {
+		case <-input.Context().Done():
+			return input.Context().Err()
+		default:
+		}
+
 		var gotErr error
 		var skip bool
 		if len(gotDynamicValues) > 0 {
@@ -482,6 +601,10 @@ const drainReqSize = int64(8 * 1024)
 
 // executeRequest executes the actual generated request and returns error if occurred
 func (request *Request) executeRequest(input *contextargs.Context, generatedRequest *generatedRequest, previousEvent output.InternalEvent, hasInteractMatchers bool, processEvent protocols.OutputEventCallback, requestCount int) (err error) {
+	// Check if hosts keep erroring
+	if request.isUnresponsiveAddress(input) {
+		return fmt.Errorf("hostErrorsCache : host %s is unresponsive", input.MetaInput.Input)
+	}
 
 	// wrap one more callback for validation and fixing event
 	callback := func(event *output.InternalWrappedEvent) {
@@ -612,7 +735,16 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 			inputUrl = url.String()
 		}
 		formedURL = fmt.Sprintf("%s%s", inputUrl, generatedRequest.rawRequest.Path)
-		resp, err = generatedRequest.original.rawhttpClient.DoRawWithOptions(generatedRequest.rawRequest.Method, inputUrl, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)), &options)
+
+		// send rawhttp request and get response
+		resp, err = httpclientpool.SendRawRequest(generatedRequest.original.rawhttpClient, &httpclientpool.RawHttpRequestOpts{
+			Method:  generatedRequest.rawRequest.Method,
+			URL:     inputUrl,
+			Path:    generatedRequest.rawRequest.Path,
+			Headers: generators.ExpandMapValues(generatedRequest.rawRequest.Headers),
+			Body:    io.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)),
+			Options: &options,
+		})
 	} else {
 		//** For Normal requests **//
 		hostname = generatedRequest.request.URL.Host
@@ -630,17 +762,34 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 			if errSignature := request.handleSignature(generatedRequest); errSignature != nil {
 				return errSignature
 			}
-
 			httpclient := request.httpClient
+
+			// this will be assigned/updated if this specific request has a custom configuration
+			var modifiedConfig *httpclientpool.Configuration
+
+			// check for cookie related configuration
 			if input.CookieJar != nil {
 				connConfiguration := request.connConfiguration
 				connConfiguration.Connection.SetCookieJar(input.CookieJar)
-				client, err := httpclientpool.Get(request.options.Options, connConfiguration)
+				modifiedConfig = connConfiguration
+			}
+			// check for request updatedTimeout annotation
+			updatedTimeout, ok := generatedRequest.request.Context().Value(httpclientpool.WithCustomTimeout{}).(httpclientpool.WithCustomTimeout)
+			if ok {
+				if modifiedConfig == nil {
+					modifiedConfig = request.connConfiguration
+				}
+				modifiedConfig.ResponseHeaderTimeout = updatedTimeout.Timeout
+			}
+
+			if modifiedConfig != nil {
+				client, err := httpclientpool.Get(request.options.Options, modifiedConfig)
 				if err != nil {
 					return errors.Wrap(err, "could not get http client")
 				}
 				httpclient = client
 			}
+
 			resp, err = httpclient.Do(generatedRequest.request)
 		}
 	}
@@ -981,7 +1130,25 @@ func (request *Request) pruneSignatureInternalValues(maps ...map[string]interfac
 
 func (request *Request) newContext(input *contextargs.Context) context.Context {
 	if input.MetaInput.CustomIP != "" {
-		return context.WithValue(context.Background(), fastdialer.IP, input.MetaInput.CustomIP)
+		return context.WithValue(input.Context(), fastdialer.IP, input.MetaInput.CustomIP)
 	}
-	return context.Background()
+	return input.Context()
+}
+
+// markUnresponsiveAddress checks if the error is a unreponsive host error and marks it
+func (request *Request) markUnresponsiveAddress(input *contextargs.Context, err error) {
+	if err == nil {
+		return
+	}
+	if request.options.HostErrorsCache != nil {
+		request.options.HostErrorsCache.MarkFailed(input, err)
+	}
+}
+
+// isUnresponsiveAddress checks if the error is a unreponsive based on its execution history
+func (request *Request) isUnresponsiveAddress(input *contextargs.Context) bool {
+	if request.options.HostErrorsCache != nil {
+		return request.options.HostErrorsCache.Check(input)
+	}
+	return false
 }

pkg/protocols/http/request_annotations.go

@@ -9,7 +9,10 @@ import (
 	"time"
 
 	"github.com/projectdiscovery/fastdialer/fastdialer"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
+	"github.com/projectdiscovery/nuclei/v3/pkg/types/nucleierr"
 	"github.com/projectdiscovery/retryablehttp-go"
+	"github.com/projectdiscovery/utils/errkit"
 	iputil "github.com/projectdiscovery/utils/ip"
 	stringsutil "github.com/projectdiscovery/utils/strings"
 )
@@ -26,6 +29,15 @@ var (
 	reTimeoutAnnotation = regexp.MustCompile(`(?m)^@timeout:\s*(.+)\s*$`)
 	// @once sets the request to be executed only once for a specific URL
 	reOnceAnnotation = regexp.MustCompile(`(?m)^@once\s*$`)
+
+	// ErrTimeoutAnnotationDeadline is the error returned when a specific amount of time was exceeded for a request
+	// which was alloted using @timeout annotation this usually means that vulnerability was not found
+	// in rare case it could also happen due to network congestion
+	// the assigned class is TemplateLogic since this in almost every case means that server is not vulnerable
+	ErrTimeoutAnnotationDeadline = errkit.New("timeout annotation deadline exceeded").SetKind(nucleierr.ErrTemplateLogic).Build()
+	// ErrRequestTimeoutDeadline is the error returned when a specific amount of time was exceeded for a request
+	// this happens when the request execution exceeds alloted time
+	ErrRequestTimeoutDeadline = errkit.New("request timeout deadline exceeded when notimeout is set").SetKind(errkit.ErrKindDeadline).Build()
 )
 
 type flowMark int
@@ -119,12 +131,16 @@ func (r *Request) parseAnnotations(rawRequest string, request *retryablehttp.Req
 			value := strings.TrimSpace(duration[1])
 			if parsed, err := time.ParseDuration(value); err == nil {
 				//nolint:govet // cancelled automatically by withTimeout
-				ctx, overrides.cancelFunc = context.WithTimeout(context.Background(), parsed)
+				// global timeout is overridden by annotation by replacing context
+				ctx, overrides.cancelFunc = context.WithTimeoutCause(context.TODO(), parsed, ErrTimeoutAnnotationDeadline)
+				// add timeout value to context
+				ctx = context.WithValue(ctx, httpclientpool.WithCustomTimeout{}, httpclientpool.WithCustomTimeout{Timeout: parsed})
 				request = request.Clone(ctx)
 			}
 		} else {
 			//nolint:govet // cancelled automatically by withTimeout
-			ctx, overrides.cancelFunc = context.WithTimeout(context.Background(), time.Duration(r.options.Options.Timeout)*time.Second)
+			// global timeout is overridden by annotation by replacing context
+			ctx, overrides.cancelFunc = context.WithTimeoutCause(context.TODO(), httpclientpool.GetHttpTimeout(r.options.Options), ErrRequestTimeoutDeadline)
 			request = request.Clone(ctx)
 		}
 	}

pkg/protocols/http/request_fuzz.go

@@ -6,7 +6,6 @@ package http
 //		-> request.executeGeneratedFuzzingRequest [execute final generated fuzzing request and get result]
 
 import (
-	"context"
 	"fmt"
 	"net/http"
 	"strings"
@@ -24,6 +23,7 @@ import (
 	protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	"github.com/projectdiscovery/retryablehttp-go"
+	"github.com/projectdiscovery/useragent"
 	urlutil "github.com/projectdiscovery/utils/url"
 )
 
@@ -89,6 +89,9 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous
 	if err != nil {
 		return errors.Wrap(err, "fuzz: could not build request from url")
 	}
+	userAgent := useragent.PickRandom()
+	baseRequest.Header.Set("User-Agent", userAgent.Raw)
+
 	// execute with one value first to checks its applicability
 	err = request.executeAllFuzzingRules(inputx, previous, baseRequest, callback)
 	if err != nil {
@@ -110,14 +113,26 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous
 func (request *Request) executeAllFuzzingRules(input *contextargs.Context, values map[string]interface{}, baseRequest *retryablehttp.Request, callback protocols.OutputEventCallback) error {
 	applicable := false
 	for _, rule := range request.Fuzzing {
+		select {
+		case <-input.Context().Done():
+			return input.Context().Err()
+		default:
+		}
+
 		err := rule.Execute(&fuzz.ExecuteRuleInput{
 			Input: input,
 			Callback: func(gr fuzz.GeneratedRequest) bool {
+				select {
+				case <-input.Context().Done():
+					return false
+				default:
+				}
+
 				// TODO: replace this after scanContext Refactor
 				return request.executeGeneratedFuzzingRequest(gr, input, callback)
 			},
 			Values:      values,
-			BaseRequest: baseRequest.Clone(context.TODO()),
+			BaseRequest: baseRequest.Clone(input.Context()),
 		})
 		if err == nil {
 			applicable = true
@@ -142,7 +157,7 @@ func (request *Request) executeAllFuzzingRules(input *contextargs.Context, value
 func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest, input *contextargs.Context, callback protocols.OutputEventCallback) bool {
 	hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators)
 	hasInteractMarkers := len(gr.InteractURLs) > 0
-	if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input.MetaInput.Input) {
+	if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input) {
 		return false
 	}
 	request.options.RateLimitTake()
@@ -154,6 +169,13 @@ func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest,
 	}
 	var gotMatches bool
 	requestErr := request.executeRequest(input, req, gr.DynamicValues, hasInteractMatchers, func(event *output.InternalWrappedEvent) {
+		for _, result := range event.Results {
+			result.IsFuzzingResult = true
+			result.FuzzingMethod = gr.Request.Method
+			result.FuzzingParameter = gr.Parameter
+			result.FuzzingPosition = gr.Component.Name()
+		}
+
 		if hasInteractMarkers && hasInteractMatchers && request.options.Interactsh != nil {
 			requestData := &interactsh.RequestData{
 				MakeResultFunc: request.MakeResultEvent,
@@ -178,7 +200,7 @@ func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest,
 	}
 	if requestErr != nil {
 		if request.options.HostErrorsCache != nil {
-			request.options.HostErrorsCache.MarkFailed(input.MetaInput.Input, requestErr)
+			request.options.HostErrorsCache.MarkFailed(input, requestErr)
 		}
 		gologger.Verbose().Msgf("[%s] Error occurred in request: %s\n", request.options.TemplateID, requestErr)
 	}

pkg/protocols/http/request_test.go

@@ -1,6 +1,7 @@
 package http
 
 import (
+	"context"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
@@ -83,7 +84,7 @@ Disallow: /c`))
 	t.Run("test", func(t *testing.T) {
 		metadata := make(output.InternalEvent)
 		previous := make(output.InternalEvent)
-		ctxArgs := contextargs.NewWithInput(ts.URL)
+		ctxArgs := contextargs.NewWithInput(context.Background(), ts.URL)
 		err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
 			if event.OperatorsResult != nil && event.OperatorsResult.Matched {
 				matchCount++
@@ -159,7 +160,7 @@ func TestDisableTE(t *testing.T) {
 	t.Run("test", func(t *testing.T) {
 		metadata := make(output.InternalEvent)
 		previous := make(output.InternalEvent)
-		ctxArgs := contextargs.NewWithInput(ts.URL)
+		ctxArgs := contextargs.NewWithInput(context.Background(), ts.URL)
 		err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
 			if event.OperatorsResult != nil && event.OperatorsResult.Matched {
 				matchCount++
@@ -172,7 +173,7 @@ func TestDisableTE(t *testing.T) {
 	t.Run("test2", func(t *testing.T) {
 		metadata := make(output.InternalEvent)
 		previous := make(output.InternalEvent)
-		ctxArgs := contextargs.NewWithInput(ts.URL)
+		ctxArgs := contextargs.NewWithInput(context.Background(), ts.URL)
 		err := request2.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
 			if event.OperatorsResult != nil && event.OperatorsResult.Matched {
 				matchCount++
@@ -242,7 +243,7 @@ func TestReqURLPattern(t *testing.T) {
 	t.Run("test", func(t *testing.T) {
 		metadata := make(output.InternalEvent)
 		previous := make(output.InternalEvent)
-		ctxArgs := contextargs.NewWithInput(ts.URL)
+		ctxArgs := contextargs.NewWithInput(context.Background(), ts.URL)
 		err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
 			if event.OperatorsResult != nil && event.OperatorsResult.Matched {
 				matchCount++

pkg/protocols/javascript/js.go

@@ -32,6 +32,7 @@ import (
 	protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
 	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	"github.com/projectdiscovery/utils/errkit"
 	errorutil "github.com/projectdiscovery/utils/errors"
 	iputil "github.com/projectdiscovery/utils/ip"
 	syncutil "github.com/projectdiscovery/utils/sync"
@@ -154,6 +155,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
 		opts := &compiler.ExecuteOptions{
 			Timeout: request.Timeout,
 			Source:  &request.Init,
+			Context: context.Background(),
 		}
 		// register 'export' function to export variables from init code
 		// these are saved in args and are available in pre-condition and request code
@@ -343,7 +345,7 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, dynamicV
 		argsCopy.TemplateCtx = templateCtx.GetAll()
 
 		result, err := request.options.JsCompiler.ExecuteWithOptions(request.preConditionCompiled, argsCopy,
-			&compiler.ExecuteOptions{Timeout: request.Timeout, Source: &request.PreCondition})
+			&compiler.ExecuteOptions{Timeout: request.Timeout, Source: &request.PreCondition, Context: target.Context()})
 		if err != nil {
 			return errorutil.NewWithTag(request.TemplateID, "could not execute pre-condition: %s", err)
 		}
@@ -359,7 +361,7 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, dynamicV
 	}
 
 	if request.generator != nil && request.Threads > 1 {
-		request.executeRequestParallel(context.Background(), hostPort, hostname, input, payloadValues, callback)
+		request.executeRequestParallel(target.Context(), hostPort, hostname, input, payloadValues, callback)
 		return nil
 	}
 
@@ -373,6 +375,12 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, dynamicV
 				return nil
 			}
 
+			select {
+			case <-input.Context().Done():
+				return input.Context().Err()
+			default:
+			}
+
 			if err := request.executeRequestWithPayloads(hostPort, input, hostname, value, payloadValues, func(result *output.InternalWrappedEvent) {
 				if result.OperatorsResult != nil && result.OperatorsResult.Matched {
 					gotMatches = true
@@ -380,11 +388,10 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, dynamicV
 				}
 				callback(result)
 			}, requestOptions); err != nil {
-				_ = err
-				// Review: should we log error here?
-				// it is technically not error as it is expected to fail
-				// gologger.Warning().Msgf("Could not execute request: %s\n", err)
-				// do not return even if error occured
+				if errkit.IsNetworkPermanentErr(err) {
+					// gologger.Verbose().Msgf("Could not execute request: %s\n", err)
+					return err
+				}
 			}
 			// If this was a match, and we want to stop at first match, skip all further requests.
 			shouldStopAtFirstMatch := request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch
@@ -401,8 +408,8 @@ func (request *Request) executeRequestParallel(ctxParent context.Context, hostPo
 	if threads == 0 {
 		threads = 1
 	}
-	ctx, cancel := context.WithCancel(ctxParent)
-	defer cancel()
+	ctx, cancel := context.WithCancelCause(ctxParent)
+	defer cancel(nil)
 	requestOptions := request.options
 	gotmatches := &atomic.Bool{}
 
@@ -419,9 +426,17 @@ func (request *Request) executeRequestParallel(ctxParent context.Context, hostPo
 				break
 			}
 
+			select {
+			case <-input.Context().Done():
+				return
+			default:
+			}
+
 			// resize check point - nop if there are no changes
 			if shouldFollowGlobal && sg.Size != request.options.Options.PayloadConcurrency {
-				sg.Resize(request.options.Options.PayloadConcurrency)
+				if err := sg.Resize(ctxParent, request.options.Options.PayloadConcurrency); err != nil {
+					gologger.Warning().Msgf("Could not resize workpool: %s\n", err)
+				}
 			}
 
 			sg.Add()
@@ -438,16 +453,15 @@ func (request *Request) executeRequestParallel(ctxParent context.Context, hostPo
 					}
 					callback(result)
 				}, requestOptions); err != nil {
-					_ = err
-					// Review: should we log error here?
-					// it is technically not error as it is expected to fail
-					// gologger.Warning().Msgf("Could not execute request: %s\n", err)
-					// do not return even if error occured
+					if errkit.IsNetworkPermanentErr(err) {
+						cancel(err)
+						return
+					}
 				}
 				// If this was a match, and we want to stop at first match, skip all further requests.
 
 				if shouldStopAtFirstMatch && gotmatches.Load() {
-					cancel()
+					cancel(nil)
 					return
 				}
 			}()
@@ -486,13 +500,12 @@ func (request *Request) executeRequestWithPayloads(hostPort string, input *conte
 	}
 
 	results, err := request.options.JsCompiler.ExecuteWithOptions(request.scriptCompiled, argsCopy,
-		&compiler.ExecuteOptions{Timeout: request.Timeout, Source: &request.Code})
+		&compiler.ExecuteOptions{Timeout: request.Timeout, Source: &request.Code, Context: input.Context()})
 	if err != nil {
 		// shouldn't fail even if it returned error instead create a failure event
 		results = compiler.ExecuteResult{"success": false, "error": err.Error()}
 	}
 	request.options.Progress.IncrementRequests()
-
 	requestOptions.Output.Request(requestOptions.TemplateID, hostPort, request.Type().String(), err)
 	gologger.Verbose().Msgf("[%s] Sent Javascript request to %s", request.options.TemplateID, hostPort)
 

pkg/protocols/network/request.go

@@ -1,7 +1,6 @@
 package network
 
 import (
-	"context"
 	"encoding/hex"
 	"fmt"
 	"net"
@@ -36,12 +35,6 @@ import (
 	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
-var (
-	// TODO: make this configurable
-	// DefaultReadTimeout is the default read timeout for network requests
-	DefaultReadTimeout = time.Duration(5) * time.Second
-)
-
 var _ protocols.Request = &Request{}
 
 // Type returns the type of the protocol request
@@ -70,7 +63,7 @@ func (request *Request) getOpenPorts(target *contextargs.Context) ([]string, err
 			errs = append(errs, err)
 			continue
 		}
-		conn, err := protocolstate.Dialer.Dial(context.TODO(), "tcp", addr)
+		conn, err := protocolstate.Dialer.Dial(target.Context(), "tcp", addr)
 		if err != nil {
 			errs = append(errs, err)
 			continue
@@ -125,6 +118,10 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, metadata
 func (request *Request) executeOnTarget(input *contextargs.Context, visited mapsutil.Map[string, struct{}], metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
 	var address string
 	var err error
+	if request.isUnresponsiveAddress(input) {
+		// skip on unresponsive address no need to continue
+		return nil
+	}
 
 	if request.SelfContained {
 		address = ""
@@ -145,6 +142,12 @@ func (request *Request) executeOnTarget(input *contextargs.Context, visited maps
 	variables = generators.MergeMaps(variablesMap, variables, request.options.Constants)
 
 	for _, kv := range request.addresses {
+		select {
+		case <-input.Context().Done():
+			return input.Context().Err()
+		default:
+		}
+
 		actualAddress := replacer.Replace(kv.address, variables)
 
 		if visited.Has(actualAddress) && !request.options.Options.DisableClustering {
@@ -173,6 +176,8 @@ func (request *Request) executeAddress(variables map[string]interface{}, actualA
 		request.options.Progress.IncrementFailedRequestsBy(1)
 		return err
 	}
+	updatedTarget := input.Clone()
+	updatedTarget.MetaInput.Input = actualAddress
 
 	// if request threads matches global payload concurrency we follow it
 	shouldFollowGlobal := request.Threads == request.options.Options.PayloadConcurrency
@@ -192,15 +197,33 @@ func (request *Request) executeAddress(variables map[string]interface{}, actualA
 				break
 			}
 
+			select {
+			case <-input.Context().Done():
+				return input.Context().Err()
+			default:
+			}
+
 			// resize check point - nop if there are no changes
 			if shouldFollowGlobal && swg.Size != request.options.Options.PayloadConcurrency {
-				swg.Resize(request.options.Options.PayloadConcurrency)
+				if err := swg.Resize(input.Context(), request.options.Options.PayloadConcurrency); err != nil {
+					m.Lock()
+					multiErr = multierr.Append(multiErr, err)
+					m.Unlock()
+				}
+			}
+			if request.isUnresponsiveAddress(updatedTarget) {
+				// skip on unresponsive address no need to continue
+				return nil
 			}
 
 			value = generators.MergeMaps(value, payloads)
 			swg.Add()
 			go func(vars map[string]interface{}) {
 				defer swg.Done()
+				if request.isUnresponsiveAddress(updatedTarget) {
+					// skip on unresponsive address no need to continue
+					return
+				}
 				if err := request.executeRequestWithPayloads(variables, actualAddress, address, input, shouldUseTLS, vars, previous, callback); err != nil {
 					m.Lock()
 					multiErr = multierr.Append(multiErr, err)
@@ -230,13 +253,22 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
 	if host, _, err := net.SplitHostPort(actualAddress); err == nil {
 		hostname = host
 	}
+	updatedTarget := input.Clone()
+	updatedTarget.MetaInput.Input = actualAddress
+
+	if request.isUnresponsiveAddress(updatedTarget) {
+		// skip on unresponsive address no need to continue
+		return nil
+	}
 
 	if shouldUseTLS {
-		conn, err = request.dialer.DialTLS(context.Background(), "tcp", actualAddress)
+		conn, err = request.dialer.DialTLS(input.Context(), "tcp", actualAddress)
 	} else {
-		conn, err = request.dialer.Dial(context.Background(), "tcp", actualAddress)
+		conn, err = request.dialer.Dial(input.Context(), "tcp", actualAddress)
 	}
 	if err != nil {
+		// adds it to unresponsive address list if applicable
+		request.markUnresponsiveAddress(updatedTarget, err)
 		request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err)
 		request.options.Progress.IncrementFailedRequestsBy(1)
 		return errors.Wrap(err, "could not connect to server")
@@ -295,7 +327,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
 		}
 
 		if input.Read > 0 {
-			buffer, err := ConnReadNWithTimeout(conn, int64(input.Read), DefaultReadTimeout)
+			buffer, err := ConnReadNWithTimeout(conn, int64(input.Read), request.options.Options.ResponseReadTimeout)
 			if err != nil {
 				return errorutil.NewWithErr(err).Msgf("could not read response from connection")
 			}
@@ -345,7 +377,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
 		bufferSize = -1
 	}
 
-	final, err := ConnReadNWithTimeout(conn, int64(bufferSize), DefaultReadTimeout)
+	final, err := ConnReadNWithTimeout(conn, int64(bufferSize), request.options.Options.ResponseReadTimeout)
 	if err != nil {
 		request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err)
 		gologger.Verbose().Msgf("could not read more data from %s: %s", actualAddress, err)
@@ -446,9 +478,6 @@ func getAddress(toTest string) (string, error) {
 }
 
 func ConnReadNWithTimeout(conn net.Conn, n int64, timeout time.Duration) ([]byte, error) {
-	if timeout == 0 {
-		timeout = DefaultReadTimeout
-	}
 	if n == -1 {
 		// if n is -1 then read all available data from connection
 		return reader.ConnReadNWithTimeout(conn, -1, timeout)
@@ -468,3 +497,21 @@ func ConnReadNWithTimeout(conn net.Conn, n int64, timeout time.Duration) ([]byte
 	}
 	return b[:count], nil
 }
+
+// markUnresponsiveAddress checks if the error is a unreponsive host error and marks it
+func (request *Request) markUnresponsiveAddress(input *contextargs.Context, err error) {
+	if err == nil {
+		return
+	}
+	if request.options.HostErrorsCache != nil {
+		request.options.HostErrorsCache.MarkFailed(input, err)
+	}
+}
+
+// isUnresponsiveAddress checks if the error is a unreponsive based on its execution history
+func (request *Request) isUnresponsiveAddress(input *contextargs.Context) bool {
+	if request.options.HostErrorsCache != nil {
+		return request.options.HostErrorsCache.Check(input)
+	}
+	return false
+}

pkg/protocols/network/request_test.go

@@ -1,6 +1,7 @@
 package network
 
 import (
+	"context"
 	"encoding/hex"
 	"fmt"
 	"net/http"
@@ -65,7 +66,7 @@ func TestNetworkExecuteWithResults(t *testing.T) {
 	t.Run("domain-valid", func(t *testing.T) {
 		metadata := make(output.InternalEvent)
 		previous := make(output.InternalEvent)
-		ctxArgs := contextargs.NewWithInput(parsed.Host)
+		ctxArgs := contextargs.NewWithInput(context.Background(), parsed.Host)
 		err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
 			finalEvent = event
 		})
@@ -81,7 +82,7 @@ func TestNetworkExecuteWithResults(t *testing.T) {
 	t.Run("invalid-port-override", func(t *testing.T) {
 		metadata := make(output.InternalEvent)
 		previous := make(output.InternalEvent)
-		ctxArgs := contextargs.NewWithInput("127.0.0.1:11211")
+		ctxArgs := contextargs.NewWithInput(context.Background(), "127.0.0.1:11211")
 		err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
 			finalEvent = event
 		})
@@ -95,7 +96,7 @@ func TestNetworkExecuteWithResults(t *testing.T) {
 	t.Run("hex-to-string", func(t *testing.T) {
 		metadata := make(output.InternalEvent)
 		previous := make(output.InternalEvent)
-		ctxArgs := contextargs.NewWithInput(parsed.Host)
+		ctxArgs := contextargs.NewWithInput(context.Background(), parsed.Host)
 		err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
 			finalEvent = event
 		})

pkg/protocols/protocols.go

@@ -1,6 +1,7 @@
 package protocols
 
 import (
+	"context"
 	"encoding/base64"
 	"sync/atomic"
 
@@ -173,7 +174,7 @@ func (e *ExecutorOptions) GetTemplateCtx(input *contextargs.MetaInput) *contexta
 	templateCtx, ok := e.templateCtxStore.Get(scanId)
 	if !ok {
 		// if template context does not exist create new and add it to store and return it
-		templateCtx = contextargs.New()
+		templateCtx = contextargs.New(context.Background())
 		templateCtx.MetaInput = input
 		_ = e.templateCtxStore.Set(scanId, templateCtx)
 	}

pkg/protocols/ssl/ssl_test.go

@@ -1,6 +1,7 @@
 package ssl
 
 import (
+	"context"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -28,7 +29,7 @@ func TestSSLProtocol(t *testing.T) {
 	require.Nil(t, err, "could not compile ssl request")
 
 	var gotEvent output.InternalEvent
-	ctxArgs := contextargs.NewWithInput("scanme.sh:443")
+	ctxArgs := contextargs.NewWithInput(context.Background(), "scanme.sh:443")
 	err = request.ExecuteWithResults(ctxArgs, nil, nil, func(event *output.InternalWrappedEvent) {
 		gotEvent = event.InternalEvent
 	})

pkg/protocols/utils/variables.go

@@ -24,6 +24,7 @@ func init() {
 		Host:     "Host",
 		Port:     "Port",
 		Path:     "Path",
+		Query:    "Query",
 		File:     "File",
 		Scheme:   "Scheme",
 		Input:    "Input",
@@ -44,6 +45,7 @@ const (
 	Host
 	Port
 	Path
+	Query
 	File
 	Scheme
 	Input
@@ -162,6 +164,12 @@ func generateVariables(inputURL *urlutil.URL, removeTrailingSlash bool) map[stri
 			knownVariables[v] = port
 		case Path:
 			knownVariables[v] = requestPath
+		case Query:
+			if queryParams := urlutil.GetParams(parsed.URL.Query()); len(queryParams) > 0 {
+				knownVariables[v] = "?" + queryParams.Encode()
+			} else {
+				knownVariables[v] = ""
+			}
 		case File:
 			knownVariables[v] = base
 		case Scheme:

pkg/protocols/utils/variables_test.go

@@ -1,6 +1,7 @@
 package utils
 
 import (
+	"context"
 	"reflect"
 	"testing"
 
@@ -49,7 +50,7 @@ func TestHTTPVariables(t *testing.T) {
 	require.Equal(t, values["Hostname"], "foobar.com", "incorrect hostname")
 
 	baseURL = "http://scanme.sh"
-	ctxArgs := contextargs.NewWithInput(baseURL)
+	ctxArgs := contextargs.NewWithInput(context.Background(), baseURL)
 	ctxArgs.MetaInput.CustomIP = "1.2.3.4"
 	values = GenerateVariablesWithContextArgs(ctxArgs, true)
 

pkg/protocols/websocket/websocket.go

@@ -1,7 +1,6 @@
 package websocket
 
 import (
-	"context"
 	"crypto/tls"
 	"fmt"
 	"io"
@@ -228,7 +227,7 @@ func (request *Request) executeRequestWithPayloads(target *contextargs.Context,
 	parsedAddress.Path = path.Join(parsedAddress.Path, parsed.Path)
 	addressToDial = parsedAddress.String()
 
-	conn, readBuffer, _, err := websocketDialer.Dial(context.Background(), addressToDial)
+	conn, readBuffer, _, err := websocketDialer.Dial(target.Context(), addressToDial)
 	if err != nil {
 		requestOptions.Output.Request(requestOptions.TemplateID, input, request.Type().String(), err)
 		requestOptions.Progress.IncrementFailedRequestsBy(1)

pkg/reporting/reporting.go

@@ -276,6 +276,7 @@ func (c *ReportingClient) CreateIssue(event *output.ResultEvent) error {
 			if tracker.ShouldFilter(event) {
 				continue
 			}
+
 			trackerName := tracker.Name()
 			stats, statsOk := c.stats[trackerName]
 

pkg/reporting/trackers/gitea/gitea.go

@@ -140,7 +140,7 @@ func (i *Integration) CloseIssue(event *output.ResultEvent) error {
 // ShouldFilter determines if an issue should be logged to this tracker
 func (i *Integration) ShouldFilter(event *output.ResultEvent) bool {
 	if i.options.AllowList != nil && i.options.AllowList.GetMatch(event) {
-		return true
+		return false
 	}
 
 	if i.options.DenyList != nil && i.options.DenyList.GetMatch(event) {

pkg/reporting/trackers/github/github.go

@@ -176,7 +176,7 @@ func (i *Integration) Name() string {
 // ShouldFilter determines if an issue should be logged to this tracker
 func (i *Integration) ShouldFilter(event *output.ResultEvent) bool {
 	if i.options.AllowList != nil && i.options.AllowList.GetMatch(event) {
-		return true
+		return false
 	}
 
 	if i.options.DenyList != nil && i.options.DenyList.GetMatch(event) {

pkg/reporting/trackers/gitlab/gitlab.go

@@ -165,7 +165,7 @@ func (i *Integration) CloseIssue(event *output.ResultEvent) error {
 // ShouldFilter determines if an issue should be logged to this tracker
 func (i *Integration) ShouldFilter(event *output.ResultEvent) bool {
 	if i.options.AllowList != nil && i.options.AllowList.GetMatch(event) {
-		return true
+		return false
 	}
 
 	if i.options.DenyList != nil && i.options.DenyList.GetMatch(event) {

pkg/reporting/trackers/jira/jira.go

@@ -315,7 +315,7 @@ func (i *Integration) FindExistingIssue(event *output.ResultEvent) (jira.Issue,
 // ShouldFilter determines if an issue should be logged to this tracker
 func (i *Integration) ShouldFilter(event *output.ResultEvent) bool {
 	if i.options.AllowList != nil && i.options.AllowList.GetMatch(event) {
-		return true
+		return false
 	}
 
 	if i.options.DenyList != nil && i.options.DenyList.GetMatch(event) {