Merge remote-tracking branch 'origin'

* add default template severity and error when validating

* ignore workflows when validating severity

* add scan name support in pdcp result upload

* scan upload: fix missing name query param

* make profile-loader integration tests generic

* add scan-id validation

* ignore invalid scan id's

.github/workflows/build-test.yml

@@ -42,6 +42,7 @@ jobs:
       - name: Test
         env:
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          PDCP_API_KEY: "${{ secrets.PDCP_API_KEY }}"
         run: go test ./...
 
       - name: Integration Tests
@@ -49,6 +50,7 @@ jobs:
         env:
           GH_ACTION: true
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          PDCP_API_KEY: "${{ secrets.PDCP_API_KEY }}"
         run: |
           chmod +x run.sh
           bash run.sh ${{ matrix.os }}

.github/workflows/lint-test.yml

@@ -21,7 +21,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v3.6.0
+        uses: golangci/golangci-lint-action@v4.0.0
         with:
           version: latest
           args: --timeout 5m

README.md

@@ -137,6 +137,7 @@ TEMPLATES:
    -nss, -no-strict-syntax                disable strict syntax check on templates
    -td, -template-display                 displays the templates content
    -tl                                    list all available templates
+   -tgl                                   list all available tags
    -sign                                  signs the templates with the private key defined in NUCLEI_SIGNATURE_PRIVATE_KEY env variable
    -code                                  enable loading code protocol-based templates
    -dut, -disable-unsigned-templates      disable running unsigned templates or templates with mismatched signature

cmd/integration-test/fuzz.go

@@ -18,6 +18,7 @@ const (
 
 var fuzzingTestCases = []TestCaseInfo{
 	{Path: "fuzz/fuzz-mode.yaml", TestCase: &fuzzModeOverride{}},
+	{Path: "fuzz/fuzz-multi-mode.yaml", TestCase: &fuzzMultipleMode{}},
 	{Path: "fuzz/fuzz-type.yaml", TestCase: &fuzzTypeOverride{}},
 	{Path: "fuzz/fuzz-query.yaml", TestCase: &httpFuzzQuery{}},
 	{Path: "fuzz/fuzz-headless.yaml", TestCase: &HeadlessFuzzingQuery{}},
@@ -174,3 +175,29 @@ func (h *HeadlessFuzzingQuery) Execute(filePath string) error {
 	}
 	return expectResultsCount(got, 2)
 }
+
+type fuzzMultipleMode struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *fuzzMultipleMode) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		xClientId := r.Header.Get("X-Client-Id")
+		xSecretId := r.Header.Get("X-Secret-Id")
+		if xClientId != "nuclei-v3" || xSecretId != "nuclei-v3" {
+			w.WriteHeader(http.StatusUnauthorized)
+			return
+		}
+		w.Header().Set("Content-Type", "text/html")
+		resp := fmt.Sprintf("<html><body><h1>This is multi-mode fuzzing test: %v <h1></body></html>", xClientId)
+		fmt.Fprint(w, resp)
+	})
+	ts := httptest.NewTLSServer(router)
+	defer ts.Close()
+
+	got, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL+"?url=https://scanme.sh", debug, "-jsonl", "-fuzz")
+	if err != nil {
+		return err
+	}
+	return expectResultsCount(got, 1)
+}

cmd/integration-test/integration-test.go

@@ -37,6 +37,7 @@ var (
 		"dns":             dnsTestCases,
 		"workflow":        workflowTestcases,
 		"loader":          loaderTestcases,
+		"profile-loader":  profileLoaderTestcases,
 		"websocket":       websocketTestCases,
 		"headless":        headlessTestcases,
 		"whois":           whoisTestCases,

cmd/integration-test/library.go

@@ -128,7 +128,7 @@ func executeNucleiAsLibrary(templatePath, templateURL string) ([]string, error)
 	}
 	store.Load()
 
-	_ = engine.Execute(store.Templates(), provider.NewSimpleInputProviderWithUrls(templateURL))
+	_ = engine.Execute(context.Background(), store.Templates(), provider.NewSimpleInputProviderWithUrls(templateURL))
 	engine.WorkPool().Wait() // Wait for the scan to finish
 
 	return results, nil

cmd/integration-test/profile-loader.go

@@ -0,0 +1,54 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/testutils"
+	errorutil "github.com/projectdiscovery/utils/errors"
+)
+
+var profileLoaderTestcases = []TestCaseInfo{
+	{Path: "profile-loader/load-with-filename", TestCase: &profileLoaderByRelFile{}},
+	{Path: "profile-loader/load-with-id", TestCase: &profileLoaderById{}},
+	{Path: "profile-loader/basic.yml", TestCase: &customProfileLoader{}},
+}
+
+type profileLoaderByRelFile struct{}
+
+func (h *profileLoaderByRelFile) Execute(testName string) error {
+	results, err := testutils.RunNucleiWithArgsAndGetResults(false, "-tl", "-tp", "cloud.yml")
+	if err != nil {
+		return errorutil.NewWithErr(err).Msgf("failed to load template with id")
+	}
+	if len(results) <= 10 {
+		return fmt.Errorf("incorrect result: expected more results than %d, got %v", 10, len(results))
+	}
+	return nil
+}
+
+type profileLoaderById struct{}
+
+func (h *profileLoaderById) Execute(testName string) error {
+	results, err := testutils.RunNucleiWithArgsAndGetResults(false, "-tl", "-tp", "cloud")
+	if err != nil {
+		return errorutil.NewWithErr(err).Msgf("failed to load template with id")
+	}
+	if len(results) <= 10 {
+		return fmt.Errorf("incorrect result: expected more results than %d, got %v", 10, len(results))
+	}
+	return nil
+}
+
+// this profile with load kevs
+type customProfileLoader struct{}
+
+func (h *customProfileLoader) Execute(filepath string) error {
+	results, err := testutils.RunNucleiWithArgsAndGetResults(false, "-tl", "-tp", filepath)
+	if err != nil {
+		return errorutil.NewWithErr(err).Msgf("failed to load template with id")
+	}
+	if len(results) < 1 {
+		return fmt.Errorf("incorrect result: expected more results than %d, got %v", 1, len(results))
+	}
+	return nil
+}

cmd/nuclei/main.go

@@ -42,9 +42,10 @@ import (
 )
 
 var (
-	cfgFile    string
-	memProfile string // optional profile file path
-	options    = &types.Options{}
+	cfgFile         string
+	templateProfile string
+	memProfile      string // optional profile file path
+	options         = &types.Options{}
 )
 
 func main() {
@@ -199,7 +200,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 	*/
 
 	flagSet.CreateGroup("input", "Target",
-		flagSet.StringSliceVarP(&options.Targets, "target", "u", nil, "target URLs/hosts to scan", goflags.StringSliceOptions),
+		flagSet.StringSliceVarP(&options.Targets, "target", "u", nil, "target URLs/hosts to scan", goflags.CommaSeparatedStringSliceOptions),
 		flagSet.StringVarP(&options.TargetsFilePath, "list", "l", "", "path to file containing a list of target URLs/hosts to scan (one per line)"),
 		flagSet.StringSliceVarP(&options.ExcludeTargets, "exclude-hosts", "eh", nil, "hosts to exclude to scan from the input list (ip, cidr, hostname)", goflags.FileCommaSeparatedStringSliceOptions),
 		flagSet.StringVar(&options.Resume, "resume", "", "resume scan using resume.cfg (clustering will be disabled)"),
@@ -225,6 +226,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.BoolVarP(&options.NoStrictSyntax, "no-strict-syntax", "nss", false, "disable strict syntax check on templates"),
 		flagSet.BoolVarP(&options.TemplateDisplay, "template-display", "td", false, "displays the templates content"),
 		flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"),
+		flagSet.BoolVar(&options.TagList, "tgl", false, "list all available tags"),
 		flagSet.StringSliceVarConfigOnly(&options.RemoteTemplateDomainList, "remote-template-domain", []string{"cloud.projectdiscovery.io"}, "allowed domain list to load remote templates from"),
 		flagSet.BoolVar(&options.SignTemplates, "sign", false, "signs the templates with the private key defined in NUCLEI_SIGNATURE_PRIVATE_KEY env variable"),
 		flagSet.BoolVar(&options.EnableCodeTemplates, "code", false, "enable loading code protocol-based templates"),
@@ -270,6 +272,8 @@ on extensive configurability, massive extensibility and ease of use.`)
 
 	flagSet.CreateGroup("configs", "Configurations",
 		flagSet.StringVar(&cfgFile, "config", "", "path to the nuclei configuration file"),
+		flagSet.StringVarP(&templateProfile, "profile", "tp", "", "template profile config file to run"),
+		flagSet.BoolVarP(&options.ListTemplateProfiles, "profile-list", "tpl", false, "list community template profiles"),
 		flagSet.BoolVarP(&options.FollowRedirects, "follow-redirects", "fr", false, "enable following redirects for http templates"),
 		flagSet.BoolVarP(&options.FollowHostRedirects, "follow-host-redirects", "fhr", false, "follow redirects on the same host"),
 		flagSet.IntVarP(&options.MaxRedirects, "max-redirects", "mr", 10, "max number of redirects to follow for http templates"),
@@ -296,10 +300,12 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.StringVarP(&options.Interface, "interface", "i", "", "network interface to use for network scan"),
 		flagSet.StringVarP(&options.AttackType, "attack-type", "at", "", "type of payload combinations to perform (batteringram,pitchfork,clusterbomb)"),
 		flagSet.StringVarP(&options.SourceIP, "source-ip", "sip", "", "source ip address to use for network scan"),
-		flagSet.IntVarP(&options.ResponseReadSize, "response-size-read", "rsr", 10*1024*1024, "max response size to read in bytes"),
+		flagSet.IntVarP(&options.ResponseReadSize, "response-size-read", "rsr", 0, "max response size to read in bytes"),
 		flagSet.IntVarP(&options.ResponseSaveSize, "response-size-save", "rss", 1*1024*1024, "max response size to read in bytes"),
+		flagSet.DurationVarP(&options.ResponseReadTimeout, "response-read-timeout", "rrt", time.Duration(5*time.Second), "response read timeout in seconds"),
 		flagSet.CallbackVar(resetCallback, "reset", "reset removes all nuclei configuration and data files (including nuclei-templates)"),
 		flagSet.BoolVarP(&options.TlsImpersonate, "tls-impersonate", "tlsi", false, "enable experimental client hello (ja3) tls randomization"),
+		flagSet.StringVarP(&options.HttpApiEndpoint, "http-api-endpoint", "hae", "", "experimental http api endpoint"),
 	)
 
 	flagSet.CreateGroup("interactsh", "interactsh",
@@ -407,7 +413,8 @@ on extensive configurability, massive extensibility and ease of use.`)
 	flagSet.CreateGroup("cloud", "Cloud",
 		flagSet.DynamicVar(&pdcpauth, "auth", "true", "configure projectdiscovery cloud (pdcp) api key"),
 		flagSet.BoolVarP(&options.EnableCloudUpload, "cloud-upload", "cup", false, "upload scan results to pdcp dashboard"),
-		flagSet.StringVarP(&options.ScanID, "scan-id", "sid", "", "upload scan results to given scan id"),
+		flagSet.StringVarP(&options.ScanID, "scan-id", "sid", "", "upload scan results to existing scan id (optional)"),
+		flagSet.StringVarP(&options.ScanName, "scan-name", "sname", "", "scan name to set (optional)"),
 	)
 
 	flagSet.CreateGroup("Authentication", "Authentication",
@@ -495,6 +502,34 @@ Additional documentation is available at: https://docs.nuclei.sh/getting-started
 		config.DefaultConfig.SetTemplatesDir(options.NewTemplatesDirectory)
 	}
 
+	defaultProfilesPath := filepath.Join(config.DefaultConfig.GetTemplateDir(), "profiles")
+	if templateProfile != "" {
+		if filepath.Ext(templateProfile) == "" {
+			if tp := findProfilePathById(templateProfile, defaultProfilesPath); tp != "" {
+				templateProfile = tp
+			} else {
+				gologger.Fatal().Msgf("'%s' is not a profile-id or profile path", templateProfile)
+			}
+		}
+		if !filepath.IsAbs(templateProfile) {
+			if filepath.Dir(templateProfile) == "profiles" {
+				defaultProfilesPath = filepath.Join(config.DefaultConfig.GetTemplateDir())
+			}
+			currentDir, err := os.Getwd()
+			if err == nil && fileutil.FileExists(filepath.Join(currentDir, templateProfile)) {
+				templateProfile = filepath.Join(currentDir, templateProfile)
+			} else {
+				templateProfile = filepath.Join(defaultProfilesPath, templateProfile)
+			}
+		}
+		if !fileutil.FileExists(templateProfile) {
+			gologger.Fatal().Msgf("given template profile file '%s' does not exist", templateProfile)
+		}
+		if err := flagSet.MergeConfigFile(templateProfile); err != nil {
+			gologger.Fatal().Msgf("Could not read template profile: %s\n", err)
+		}
+	}
+
 	if len(options.SecretsFile) > 0 {
 		for _, secretFile := range options.SecretsFile {
 			if !fileutil.FileExists(secretFile) {
@@ -620,6 +655,27 @@ Note: Make sure you have backup of your custom nuclei-templates before proceedin
 	os.Exit(0)
 }
 
+func findProfilePathById(profileId, templatesDir string) string {
+	var profilePath string
+	err := filepath.WalkDir(templatesDir, func(iterItem string, d fs.DirEntry, err error) error {
+		ext := filepath.Ext(iterItem)
+		isYaml := ext == extensions.YAML || ext == extensions.YML
+		if err != nil || d.IsDir() || !isYaml {
+			// skip non yaml files
+			return nil
+		}
+		if strings.TrimSuffix(filepath.Base(iterItem), ext) == profileId {
+			profilePath = iterItem
+			return fmt.Errorf("FOUND")
+		}
+		return nil
+	})
+	if err != nil && err.Error() != "FOUND" {
+		gologger.Error().Msgf("%s\n", err)
+	}
+	return profilePath
+}
+
 func init() {
 	// print stacktrace of errors in debug mode
 	if strings.EqualFold(os.Getenv("DEBUG"), "true") {

go.mod

@@ -20,12 +20,12 @@ require (
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.0.20
-	github.com/projectdiscovery/fastdialer v0.0.67
+	github.com/projectdiscovery/fastdialer v0.0.69
 	github.com/projectdiscovery/hmap v0.0.41
 	github.com/projectdiscovery/interactsh v1.1.9
-	github.com/projectdiscovery/rawhttp v0.1.45
+	github.com/projectdiscovery/rawhttp v0.1.47
 	github.com/projectdiscovery/retryabledns v1.0.58
-	github.com/projectdiscovery/retryablehttp-go v1.0.55
+	github.com/projectdiscovery/retryablehttp-go v1.0.58
 	github.com/projectdiscovery/yamldoc-go v1.0.4
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.5.0
@@ -81,24 +81,25 @@ require (
 	github.com/projectdiscovery/dsl v0.0.52
 	github.com/projectdiscovery/fasttemplate v0.0.2
 	github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
-	github.com/projectdiscovery/goflags v0.1.48
+	github.com/projectdiscovery/goflags v0.1.50
 	github.com/projectdiscovery/gologger v1.1.12
 	github.com/projectdiscovery/gostruct v0.0.2
 	github.com/projectdiscovery/gozero v0.0.2
 	github.com/projectdiscovery/httpx v1.6.0
-	github.com/projectdiscovery/mapcidr v1.1.16
+	github.com/projectdiscovery/mapcidr v1.1.34
 	github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5
-	github.com/projectdiscovery/ratelimit v0.0.35
+	github.com/projectdiscovery/ratelimit v0.0.39
 	github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917
 	github.com/projectdiscovery/sarif v0.0.1
 	github.com/projectdiscovery/tlsx v1.1.6
 	github.com/projectdiscovery/uncover v1.0.7
-	github.com/projectdiscovery/useragent v0.0.40
-	github.com/projectdiscovery/utils v0.0.88
-	github.com/projectdiscovery/wappalyzergo v0.0.116
+	github.com/projectdiscovery/useragent v0.0.49
+	github.com/projectdiscovery/utils v0.0.92
+	github.com/projectdiscovery/wappalyzergo v0.0.120
 	github.com/redis/go-redis/v9 v9.1.0
 	github.com/seh-msft/burpxml v1.0.1
 	github.com/stretchr/testify v1.9.0
+	github.com/tarunKoyalwar/goleak v0.0.0-20240426214851-746d64600adc
 	github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706
 	golang.org/x/term v0.19.0
 	gopkg.in/yaml.v3 v3.0.1
@@ -142,8 +143,6 @@ require (
 	github.com/docker/cli v24.0.5+incompatible // indirect
 	github.com/docker/docker v24.0.9+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
-	github.com/eapache/channels v1.1.0 // indirect
-	github.com/eapache/queue v1.1.0 // indirect
 	github.com/fatih/color v1.15.0 // indirect
 	github.com/free5gc/util v1.0.5-0.20230511064842-2e120956883b // indirect
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
@@ -179,6 +178,7 @@ require (
 	github.com/klauspost/compress v1.17.6 // indirect
 	github.com/klauspost/pgzip v1.2.6 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/logrusorgru/aurora/v4 v4.0.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/mackerelio/go-osstat v0.2.4 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
@@ -232,6 +232,7 @@ require (
 	github.com/yuin/goldmark-emoji v1.0.1 // indirect
 	github.com/zcalusic/sysinfo v1.0.2 // indirect
 	github.com/zeebo/blake3 v0.2.3 // indirect
+	go.uber.org/goleak v1.3.0 // indirect
 	golang.org/x/arch v0.3.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
 	gopkg.in/djherbis/times.v1 v1.3.0 // indirect
@@ -349,5 +350,3 @@ require (
 
 // https://go.dev/ref/mod#go-mod-file-retract
 retract v3.2.0 // retract due to broken js protocol issue
-
-replace github.com/go-echarts/go-echarts/v2 => github.com/tarunKoyalwar/go-echarts/v2 v2.1.1

go.sum

@@ -296,11 +296,8 @@ github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj6
 github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
-github.com/eapache/channels v1.1.0 h1:F1taHcn7/F0i8DYqKXJnyhJcVpp2kgFcNePxXtnyu4k=
-github.com/eapache/channels v1.1.0/go.mod h1:jMm2qB5Ubtg9zLd+inMZd2/NUvXgzmWXsDaLyQIGfH0=
 github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
-github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
 github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
@@ -317,6 +314,8 @@ github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
 github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
 github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
+github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
 github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
 github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
@@ -671,6 +670,8 @@ github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-b
 github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
 github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
 github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
+github.com/logrusorgru/aurora/v4 v4.0.0 h1:sRjfPpun/63iADiSvGGjgA1cAYegEWMPCJdUpJYn9JA=
+github.com/logrusorgru/aurora/v4 v4.0.0/go.mod h1:lP0iIa2nrnT/qoFXcOZSrZQpJ1o6n2CUf/hyHi2Q4ZQ=
 github.com/lor00x/goldap v0.0.0-20180618054307-a546dffdd1a3 h1:wIONC+HMNRqmWBjuMxhatuSzHaljStc4gjDeKycxy0A=
 github.com/lor00x/goldap v0.0.0-20180618054307-a546dffdd1a3/go.mod h1:37YR9jabpiIxsb8X9VCIx8qFOjTDIIrIHHODa8C4gz0=
 github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
@@ -836,16 +837,16 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo
 github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4=
 github.com/projectdiscovery/dsl v0.0.52 h1:jvIvF+qN8+MbI1MHtWJJKfWqAZQlCExL3ob7SddQbZE=
 github.com/projectdiscovery/dsl v0.0.52/go.mod h1:xfcHwhy2HSaeGgh+1wqzOoCGm2XTdh5JzjBRBVHEMvI=
-github.com/projectdiscovery/fastdialer v0.0.67 h1:NvBpZUiLr9Ne9N+Lvi6FFiNNLWuhk5Bc1H+oE9J8C1E=
-github.com/projectdiscovery/fastdialer v0.0.67/go.mod h1:GhSAKnojJN8N9K0JNjLmwLCmEDsQ5cBAStqSCm/tm84=
+github.com/projectdiscovery/fastdialer v0.0.69 h1:BfFQTyTB1hrw9sWCw4CjQfbmlpvnJCPZEmKtxcwJGbU=
+github.com/projectdiscovery/fastdialer v0.0.69/go.mod h1:RXrx7M2T3V3rMZ2h0X2/SsY93+RhgF/LmFa1E0MI3L8=
 github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
 github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw=
 github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q=
 github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE=
 github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG906Drtbpz4DwU5mhGIeOhRcktDH4cGQitGUMAsg=
 github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY=
-github.com/projectdiscovery/goflags v0.1.48 h1:iJgq+YuQOEm0V7BqlV/0wx7GaI2rmGiFzPkvL/4sCj0=
-github.com/projectdiscovery/goflags v0.1.48/go.mod h1:C9Qqyivehk5iYCf3VAC3+85y60Qbde1ZBMqe5wHhcVw=
+github.com/projectdiscovery/goflags v0.1.50 h1:vhpM3mDTnXTktypUuVKWxgRpuBFJE+wzOr56rnOl4Bc=
+github.com/projectdiscovery/goflags v0.1.50/go.mod h1:3A8u7Q7e59S7f6QlJmMvKMXD/Tp+bfWre64gUpt6a6Q=
 github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A=
 github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw=
 github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M=
@@ -862,22 +863,22 @@ github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb h1:M
 github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb/go.mod h1:vmgC0DTFCfoCLp0RAfsfYTZZan0QMVs+cmTbH6blfjk=
 github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE=
 github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI=
-github.com/projectdiscovery/mapcidr v1.1.16 h1:rjj1w5D6hbTsUQXYClLcGdfBEy9bryclgi70t0vBggo=
-github.com/projectdiscovery/mapcidr v1.1.16/go.mod h1:rGqpBhStdwOQ2uS62QM9qPsybwMwIhT7CTd2bxoHs8Q=
+github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQtls1wmXN/kM=
+github.com/projectdiscovery/mapcidr v1.1.34/go.mod h1:1+1R6OkKSAKtWDXE9RvxXtXPoajXTYX0eiEdkqlhQqQ=
 github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw1pfT6bg35NiN7yd1XKtJap5Nk6lMwQ0RNi8=
 github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc=
 github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7x7gHm028TJ5XwI8=
 github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI=
-github.com/projectdiscovery/ratelimit v0.0.35 h1:epEzFATOcXZ4tssV4Hax5Op9lrbUnQMEGMV5PoUpTKc=
-github.com/projectdiscovery/ratelimit v0.0.35/go.mod h1:mPqa8UpV5I7eAN5/ZcsjLiXMhjtVvZRrHtpBRsTPuyA=
-github.com/projectdiscovery/rawhttp v0.1.45 h1:5jssUybhRz2ljI3bW4kUO9MTvZ37ArG8pISYtgqzwBg=
-github.com/projectdiscovery/rawhttp v0.1.45/go.mod h1:oLWuZkyKi4rddPoLEN3TeAqyLvFNfr3dxmzV2kwpPvs=
+github.com/projectdiscovery/ratelimit v0.0.39 h1:gMpkkuRSzsxNOxPOiD0aYTLQTOiqM/I0WZSZ7kmL+hc=
+github.com/projectdiscovery/ratelimit v0.0.39/go.mod h1:vR+q959CbyXCV/pD4bl4aGP2/R91BOKv1SyiHDOqeL0=
+github.com/projectdiscovery/rawhttp v0.1.47 h1:bjhZBK+7iuvcu0QTRJjdS4wP747+33Sd5gfgc4V9AWU=
+github.com/projectdiscovery/rawhttp v0.1.47/go.mod h1:2XA7xODWEGoHpEB8qzgqsV8yVL3cg5G63ZaT9ALwU4g=
 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk=
 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg=
 github.com/projectdiscovery/retryabledns v1.0.58 h1:ut1FSB9+GZ6zQIlKJFLqIz2RZs81EmkbsHTuIrWfYLE=
 github.com/projectdiscovery/retryabledns v1.0.58/go.mod h1:RobmKoNBgngAVE4H9REQtaLP1pa4TCyypHy1MWHT1mY=
-github.com/projectdiscovery/retryablehttp-go v1.0.55 h1:ADgugnl9jKkNXn5m/Zd8TGPq1P7GplYlqUNKm/qTmls=
-github.com/projectdiscovery/retryablehttp-go v1.0.55/go.mod h1:Kpvh4ruFPOEPYaYxgbFmlvBJr4lJKqpcbGvx1j0r/Ng=
+github.com/projectdiscovery/retryablehttp-go v1.0.58 h1:i5BlSJGgNnoTULyqLSe3d39o/XShxK4oEvx0e/gb9N4=
+github.com/projectdiscovery/retryablehttp-go v1.0.58/go.mod h1:bbok7sSEplSwZOY91UlLdVilhavYos1RaCJLJx761V0=
 github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us=
 github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ=
 github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA=
@@ -886,12 +887,12 @@ github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1Uzc
 github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo=
 github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7siFy9sj0A=
 github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE=
-github.com/projectdiscovery/useragent v0.0.40 h1:1LUhReSGPkhqsM5n40OOC9dIoNqMGs1dyGFJcOmg2Fo=
-github.com/projectdiscovery/useragent v0.0.40/go.mod h1:EvK1x3s948Gtqb/XOahXcauyejCL/rSgy5d1IAvsKT4=
-github.com/projectdiscovery/utils v0.0.88 h1:oYfCXM+8VHNLyH/H6cOibkuDUwHUAOBAMRNPFX6NPrs=
-github.com/projectdiscovery/utils v0.0.88/go.mod h1:lAWzFdGXtJRPKdhUu1Z46d8B8JbASTk1Z69WY6H/3kA=
-github.com/projectdiscovery/wappalyzergo v0.0.116 h1:xy+mBpwbYo/0PSzmJOQ/RXHomEh0D3nDBcbCxsW69m8=
-github.com/projectdiscovery/wappalyzergo v0.0.116/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8=
+github.com/projectdiscovery/useragent v0.0.49 h1:wQc9i+Xy+mUMJ45Ralv1JsQImRWqEOEvpYUe6MchScg=
+github.com/projectdiscovery/useragent v0.0.49/go.mod h1:jQz6X/usiXrPYE6B/1uVKuzIrBJXgw9hLC9eeNy38+0=
+github.com/projectdiscovery/utils v0.0.92 h1:lGCmjUJhzoNX4FQZWpp80058pRlD0/dYxLJOSs07EqY=
+github.com/projectdiscovery/utils v0.0.92/go.mod h1:d5uvD5qcRiK3qxZbBy9eatCqrCSuj9SObL04w/WgXSg=
+github.com/projectdiscovery/wappalyzergo v0.0.120 h1:dphOXnaT3rryo9h9fgbxnAVhtQ1uq61yyQZMYyHz960=
+github.com/projectdiscovery/wappalyzergo v0.0.120/go.mod h1:qW0PP+UBMcdQBBnwk+X6YYFs6huKNvn2BOVs4vQPru0=
 github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=
 github.com/projectdiscovery/yamldoc-go v1.0.4/go.mod h1:8PIPRcUD55UbtQdcfFR1hpIGRWG0P7alClXNGt1TBik=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
@@ -1015,8 +1016,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
-github.com/tarunKoyalwar/go-echarts/v2 v2.1.1 h1:5fsXGPmK+i18J8cDgxy7AJkiXWBARpVTb0Gbv+bAzPo=
-github.com/tarunKoyalwar/go-echarts/v2 v2.1.1/go.mod h1:VEeyPT5Odx/UHeuxtIAHGu2+87MWGA5OBaZ120NFi/w=
+github.com/tarunKoyalwar/goleak v0.0.0-20240426214851-746d64600adc h1:/5P5I7oDqdLee8W9Moof0xSD8tT1qEVzhObSI9CqHkg=
+github.com/tarunKoyalwar/goleak v0.0.0-20240426214851-746d64600adc/go.mod h1:uQdBQGrE1fZ2EyOs0pLcCDd1bBV4rSThieuIIGhXZ50=
 github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI=
 github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8=
 github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI=
@@ -1151,8 +1152,8 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
-go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
-go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=

integration_tests/fuzz/fuzz-multi-mode.yaml

@@ -0,0 +1,27 @@
+id: fuzz-multi-mode-test
+
+info:
+  name: multi-mode fuzzing test
+  author: pdteam
+  severity: info
+
+http:
+  - payloads:
+      inject:
+        - nuclei-v1
+        - nuclei-v2
+        - nuclei-v3
+
+    fuzzing:
+      - part: header
+        type: replace
+        mode: multiple
+        fuzz:
+          X-Client-Id: "{{inject}}"
+          X-Secret-Id: "{{inject}}"
+          
+    matchers-condition: or
+    matchers:
+      - type: word
+        words:
+          - "nuclei-v3"

integration_tests/profile-loader/basic.yml

@@ -0,0 +1,2 @@
+tags:
+  - kev

internal/httpapi/apiendpoint.go

@@ -0,0 +1,112 @@
+package httpapi
+
+import (
+	"encoding/json"
+	"net/http"
+	"time"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/js/compiler"
+	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+)
+
+type Concurrency struct {
+	BulkSize              int    `json:"bulk_size"`
+	Threads               int    `json:"threads"`
+	RateLimit             int    `json:"rate_limit"`
+	RateLimitDuration     string `json:"rate_limit_duration"`
+	PayloadConcurrency    int    `json:"payload_concurrency"`
+	ProbeConcurrency      int    `json:"probe_concurrency"`
+	JavascriptConcurrency int    `json:"javascript_concurrency"`
+}
+
+// Server represents the HTTP server that handles the concurrency settings endpoints.
+type Server struct {
+	addr   string
+	config *types.Options
+}
+
+// New creates a new instance of Server.
+func New(addr string, config *types.Options) *Server {
+	return &Server{
+		addr:   addr,
+		config: config,
+	}
+}
+
+// Start initializes the server and its routes, then starts listening on the specified address.
+func (s *Server) Start() error {
+	http.HandleFunc("/api/concurrency", s.handleConcurrency)
+	if err := http.ListenAndServe(s.addr, nil); err != nil {
+		return err
+	}
+	return nil
+}
+
+// handleConcurrency routes the request based on its method to the appropriate handler.
+func (s *Server) handleConcurrency(w http.ResponseWriter, r *http.Request) {
+	switch r.Method {
+	case http.MethodGet:
+		s.getSettings(w, r)
+	case http.MethodPut:
+		s.updateSettings(w, r)
+	default:
+		http.Error(w, "Unsupported HTTP method", http.StatusMethodNotAllowed)
+	}
+}
+
+// GetSettings handles GET requests and returns the current concurrency settings
+func (s *Server) getSettings(w http.ResponseWriter, _ *http.Request) {
+	concurrencySettings := Concurrency{
+		BulkSize:              s.config.BulkSize,
+		Threads:               s.config.TemplateThreads,
+		RateLimit:             s.config.RateLimit,
+		RateLimitDuration:     s.config.RateLimitDuration.String(),
+		PayloadConcurrency:    s.config.PayloadConcurrency,
+		ProbeConcurrency:      s.config.ProbeConcurrency,
+		JavascriptConcurrency: compiler.PoolingJsVmConcurrency,
+	}
+	w.Header().Set("Content-Type", "application/json")
+	if err := json.NewEncoder(w).Encode(concurrencySettings); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
+// UpdateSettings handles PUT requests to update the concurrency settings
+func (s *Server) updateSettings(w http.ResponseWriter, r *http.Request) {
+	var newSettings Concurrency
+	if err := json.NewDecoder(r.Body).Decode(&newSettings); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	if newSettings.RateLimitDuration != "" {
+		if duration, err := time.ParseDuration(newSettings.RateLimitDuration); err == nil {
+			s.config.RateLimitDuration = duration
+		} else {
+			http.Error(w, "Invalid duration format", http.StatusBadRequest)
+			return
+		}
+	}
+	if newSettings.BulkSize > 0 {
+		s.config.BulkSize = newSettings.BulkSize
+	}
+	if newSettings.Threads > 0 {
+		s.config.TemplateThreads = newSettings.Threads
+	}
+	if newSettings.RateLimit > 0 {
+		s.config.RateLimit = newSettings.RateLimit
+	}
+	if newSettings.PayloadConcurrency > 0 {
+		s.config.PayloadConcurrency = newSettings.PayloadConcurrency
+	}
+	if newSettings.ProbeConcurrency > 0 {
+		s.config.ProbeConcurrency = newSettings.ProbeConcurrency
+	}
+	if newSettings.JavascriptConcurrency > 0 {
+		compiler.PoolingJsVmConcurrency = newSettings.JavascriptConcurrency
+		s.config.JsConcurrency = newSettings.JavascriptConcurrency // no-op on speed change
+	}
+
+	w.WriteHeader(http.StatusOK)
+}

internal/pdcp/writer.go

@@ -9,6 +9,7 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"regexp"
 	"sync/atomic"
 	"time"
 
@@ -27,9 +28,13 @@ const (
 	appendEndpoint = "/v1/scans/%s/import"
 	flushTimer     = time.Duration(1) * time.Minute
 	MaxChunkSize   = 1024 * 1024 * 4 // 4 MB
+	xidRe          = `^[a-z0-9]{20}$`
 )
 
-var _ output.Writer = &UploadWriter{}
+var (
+	xidRegex               = regexp.MustCompile(xidRe)
+	_        output.Writer = &UploadWriter{}
+)
 
 // UploadWriter is a writer that uploads its output to pdcp
 // server to enable web dashboard and more
@@ -41,6 +46,7 @@ type UploadWriter struct {
 	cancel    context.CancelFunc
 	done      chan struct{}
 	scanID    string
+	scanName  string
 	counter   atomic.Int32
 }
 
@@ -86,8 +92,17 @@ func NewUploadWriter(ctx context.Context, creds *pdcpauth.PDCPCredentials) (*Upl
 }
 
 // SetScanID sets the scan id for the upload writer
-func (u *UploadWriter) SetScanID(id string) {
+func (u *UploadWriter) SetScanID(id string) error {
+	if !xidRegex.MatchString(id) {
+		return fmt.Errorf("invalid scan id provided")
+	}
 	u.scanID = id
+	return nil
+}
+
+// SetScanName sets the scan name for the upload writer
+func (u *UploadWriter) SetScanName(name string) {
+	u.scanName = name
 }
 
 func (u *UploadWriter) autoCommit(ctx context.Context, r *io.PipeReader) {
@@ -220,7 +235,13 @@ func (u *UploadWriter) getRequest(bin []byte) (*retryablehttp.Request, error) {
 		return nil, errorutil.NewWithErr(err).Msgf("could not create cloud upload request")
 	}
 	// add pdtm meta params
-	req.URL.RawQuery = updateutils.GetpdtmParams(config.Version)
+	req.URL.Params.Merge(updateutils.GetpdtmParams(config.Version))
+	// if it is upload endpoint also include name if it exists
+	if u.scanName != "" && req.URL.Path == uploadEndpoint {
+		req.URL.Params.Add("name", u.scanName)
+	}
+	req.URL.Update()
+
 	req.Header.Set(pdcpauth.ApiKeyHeaderName, u.creds.APIKey)
 	req.Header.Set("Content-Type", "application/octet-stream")
 	req.Header.Set("Accept", "application/json")

internal/runner/inputs.go

@@ -1,6 +1,7 @@
 package runner
 
 import (
+	"context"
 	"sync/atomic"
 	"time"
 
@@ -16,8 +17,6 @@ import (
 	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
-var GlobalProbeBulkSize = 50
-
 // initializeTemplatesHTTPInput initializes the http form of input
 // for any loaded http templates if input is in non-standard format.
 func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
@@ -31,11 +30,6 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 	}
 	gologger.Info().Msgf("Running httpx on input host")
 
-	var bulkSize = GlobalProbeBulkSize
-	if r.options.BulkSize > GlobalProbeBulkSize {
-		bulkSize = r.options.BulkSize
-	}
-
 	httpxOptions := httpx.DefaultOptions
 	httpxOptions.RetryMax = r.options.Retries
 	httpxOptions.Timeout = time.Duration(r.options.Timeout) * time.Second
@@ -45,10 +39,8 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 		return nil, errors.Wrap(err, "could not create httpx client")
 	}
 
-	shouldFollowGlobalProbeBulkSize := bulkSize == GlobalProbeBulkSize
-
 	// Probe the non-standard URLs and store them in cache
-	swg, err := syncutil.New(syncutil.WithSize(bulkSize))
+	swg, err := syncutil.New(syncutil.WithSize(r.options.BulkSize))
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create adaptive group")
 	}
@@ -58,8 +50,10 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) {
 			return true
 		}
 
-		if shouldFollowGlobalProbeBulkSize && swg.Size != GlobalProbeBulkSize {
-			swg.Resize(GlobalProbeBulkSize)
+		if r.options.ProbeConcurrency > 0 && swg.Size != r.options.ProbeConcurrency {
+			if err := swg.Resize(context.Background(), r.options.ProbeConcurrency); err != nil {
+				gologger.Error().Msgf("Could not resize workpool: %s\n", err)
+			}
 		}
 
 		swg.Add()

internal/runner/lazy.go

@@ -1,6 +1,7 @@
 package runner
 
 import (
+	"context"
 	"fmt"
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/authprovider/authx"
@@ -71,7 +72,8 @@ func GetLazyAuthFetchCallback(opts *AuthLazyFetchOptions) authx.LazyFetchSecret
 		tmpl := tmpls[0]
 		// add args to tmpl here
 		vars := map[string]interface{}{}
-		ctx := scan.NewScanContext(contextargs.NewWithInput(d.Input))
+		mainCtx := context.Background()
+		ctx := scan.NewScanContext(mainCtx, contextargs.NewWithInput(mainCtx, d.Input))
 		for _, v := range d.Variables {
 			vars[v.Key] = v.Value
 			ctx.Input.Add(v.Key, v.Value)

internal/runner/options.go

@@ -3,6 +3,7 @@ package runner
 import (
 	"bufio"
 	"fmt"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -25,6 +26,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/jsonl"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/sarif"
+	"github.com/projectdiscovery/nuclei/v3/pkg/templates/extensions"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml"
 	fileutil "github.com/projectdiscovery/utils/file"
@@ -74,6 +76,31 @@ func ParseOptions(options *types.Options) {
 		}
 		os.Exit(0)
 	}
+
+	defaultProfilesPath := filepath.Join(config.DefaultConfig.GetTemplateDir(), "profiles")
+	if options.ListTemplateProfiles {
+		gologger.Print().Msgf(
+			"\nListing available %v nuclei template profiles for %v",
+			config.DefaultConfig.TemplateVersion,
+			config.DefaultConfig.TemplatesDirectory,
+		)
+		templatesRootDir := config.DefaultConfig.GetTemplateDir()
+		err := filepath.WalkDir(defaultProfilesPath, func(iterItem string, d fs.DirEntry, err error) error {
+			ext := filepath.Ext(iterItem)
+			isYaml := ext == extensions.YAML || ext == extensions.YML
+			if err != nil || d.IsDir() || !isYaml {
+				return nil
+			}
+			if profileRelPath, err := filepath.Rel(templatesRootDir, iterItem); err == nil {
+				gologger.Print().Msgf("%s (%s)\n", profileRelPath, strings.TrimSuffix(filepath.Base(iterItem), ext))
+			}
+			return nil
+		})
+		if err != nil {
+			gologger.Error().Msgf("%s\n", err)
+		}
+		os.Exit(0)
+	}
 	if options.StoreResponseDir != DefaultDumpTrafficOutputFolder && !options.StoreResponse {
 		gologger.Debug().Msgf("Store response directory specified, enabling \"store-resp\" flag automatically\n")
 		options.StoreResponse = true

internal/runner/runner.go

@@ -32,6 +32,7 @@ import (
 
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/internal/colorizer"
+	"github.com/projectdiscovery/nuclei/v3/internal/httpapi"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk"
@@ -52,6 +53,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/uncover"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/excludematchers"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
+	httpProtocol "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
@@ -88,8 +90,9 @@ type Runner struct {
 	pdcpUploadErrMsg string
 	inputProvider    provider.InputProvider
 	//general purpose temporary directory
-	tmpDir string
-	parser parser.Parser
+	tmpDir          string
+	parser          parser.Parser
+	httpApiEndpoint *httpapi.Server
 }
 
 const pprofServerAddress = "127.0.0.1:8086"
@@ -221,6 +224,17 @@ func New(options *types.Options) (*Runner, error) {
 		}()
 	}
 
+	if options.HttpApiEndpoint != "" {
+		apiServer := httpapi.New(options.HttpApiEndpoint, options)
+		gologger.Info().Msgf("Listening api endpoint on: %s", options.HttpApiEndpoint)
+		runner.httpApiEndpoint = apiServer
+		go func() {
+			if err := apiServer.Start(); err != nil {
+				gologger.Error().Msgf("Failed to start API server: %s", err)
+			}
+		}()
+	}
+
 	if (len(options.Templates) == 0 || !options.NewTemplates || (options.TargetsFilePath == "" && !options.Stdin && len(options.Targets) == 0)) && options.UpdateTemplates {
 		os.Exit(0)
 	}
@@ -400,7 +414,11 @@ func (r *Runner) setupPDCPUpload(writer output.Writer) output.Writer {
 		return writer
 	}
 	if r.options.ScanID != "" {
-		uploadWriter.SetScanID(r.options.ScanID)
+		// ignore and use empty scan id if invalid
+		_ = uploadWriter.SetScanID(r.options.ScanID)
+	}
+	if r.options.ScanName != "" {
+		uploadWriter.SetScanName(r.options.ScanName)
 	}
 	return output.NewMultiWriter(writer, uploadWriter)
 }
@@ -496,6 +514,23 @@ func (r *Runner) RunEnumeration() error {
 		return errors.Wrap(err, "Could not create loader.")
 	}
 
+	// list all templates or tags as specified by user.
+	// This uses a separate parser to reduce time taken as
+	// normally nuclei does a lot of compilation and stuff
+	// for templates, which we don't want for these simp
+	if r.options.TemplateList || r.options.TemplateDisplay || r.options.TagList {
+		if err := store.LoadTemplatesOnlyMetadata(); err != nil {
+			return err
+		}
+
+		if r.options.TagList {
+			r.listAvailableStoreTags(store)
+		} else {
+			r.listAvailableStoreTemplates(store)
+		}
+		os.Exit(0)
+	}
+
 	if r.options.Validate {
 		if err := store.ValidateTemplates(); err != nil {
 			return err
@@ -526,12 +561,6 @@ func (r *Runner) RunEnumeration() error {
 			_ = r.inputProvider.SetWithExclusions(host)
 		}
 	}
-	// list all templates
-	if r.options.TemplateList || r.options.TemplateDisplay {
-		r.listAvailableStoreTemplates(store)
-		os.Exit(0)
-	}
-
 	// display execution info like version , templates used etc
 	r.displayExecutionInfo(store)
 
@@ -655,7 +684,7 @@ func (r *Runner) executeTemplatesInput(store *loader.Store, engine *core.Engine)
 	if r.inputProvider == nil {
 		return nil, errors.New("no input provider found")
 	}
-	results := engine.ExecuteScanWithOpts(finalTemplates, r.inputProvider, r.options.DisableClustering)
+	results := engine.ExecuteScanWithOpts(context.Background(), finalTemplates, r.inputProvider, r.options.DisableClustering)
 	return results, nil
 }
 
@@ -684,6 +713,7 @@ func (r *Runner) displayExecutionInfo(store *loader.Store) {
 	} else {
 		stats.DisplayAsWarning(templates.SkippedCodeTmplTamperedStats)
 	}
+	stats.DisplayAsWarning(httpProtocol.SetThreadToCountZero)
 	stats.ForceDisplayWarning(templates.SkippedUnsignedStats)
 	stats.ForceDisplayWarning(templates.SkippedRequestSignatureStats)
 

internal/runner/templates.go

@@ -3,9 +3,11 @@ package runner
 import (
 	"bytes"
 	"path/filepath"
+	"sort"
 	"strings"
 
 	"github.com/alecthomas/chroma/quick"
+	jsoniter "github.com/json-iterator/go"
 	"github.com/logrusorgru/aurora"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
@@ -71,6 +73,40 @@ func (r *Runner) listAvailableStoreTemplates(store *loader.Store) {
 	}
 }
 
+func (r *Runner) listAvailableStoreTags(store *loader.Store) {
+	gologger.Print().Msgf(
+		"\nListing available %v nuclei tags for %v",
+		config.DefaultConfig.TemplateVersion,
+		config.DefaultConfig.TemplatesDirectory,
+	)
+	tagsMap := make(map[string]int)
+	for _, tpl := range store.Templates() {
+		for _, tag := range tpl.Info.Tags.ToSlice() {
+			tagsMap[tag]++
+		}
+	}
+	type kv struct {
+		Key   string `json:"tag"`
+		Value int    `json:"count"`
+	}
+	var tagsList []kv
+	for k, v := range tagsMap {
+		tagsList = append(tagsList, kv{k, v})
+	}
+	sort.Slice(tagsList, func(i, j int) bool {
+		return tagsList[i].Value > tagsList[j].Value
+	})
+
+	for _, tag := range tagsList {
+		if r.options.JSONL {
+			marshalled, _ := jsoniter.Marshal(tag)
+			gologger.Silent().Msgf("%s\n", string(marshalled))
+		} else {
+			gologger.Silent().Msgf("%s (%d)\n", tag.Key, tag.Value)
+		}
+	}
+}
+
 func (r *Runner) highlightTemplate(body *[]byte) ([]byte, error) {
 	var buf bytes.Buffer
 	// YAML lexer, true color terminal formatter and monokai style

lib/config.go

@@ -10,6 +10,7 @@ import (
 	"github.com/projectdiscovery/ratelimit"
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/authprovider"
+	"github.com/projectdiscovery/nuclei/v3/pkg/catalog"
 	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"github.com/projectdiscovery/nuclei/v3/pkg/progress"
@@ -393,7 +394,7 @@ func EnablePassiveMode() NucleiSDKOptions {
 	}
 }
 
-// WithAuthOptions allows setting a custom authprovider implementation
+// WithAuthProvider allows setting a custom authprovider implementation
 func WithAuthProvider(provider authprovider.AuthProvider) NucleiSDKOptions {
 	return func(e *NucleiEngine) error {
 		e.authprovider = provider
@@ -425,3 +426,11 @@ func SignedTemplatesOnly() NucleiSDKOptions {
 		return nil
 	}
 }
+
+// WithCatalog uses a supplied catalog
+func WithCatalog(cat catalog.Catalog) NucleiSDKOptions {
+	return func(e *NucleiEngine) error {
+		e.catalog = cat
+		return nil
+	}
+}

lib/multi.go

@@ -58,6 +58,22 @@ func createEphemeralObjects(base *NucleiEngine, opts *types.Options) (*unsafeOpt
 	return u, nil
 }
 
+// closeEphemeralObjects closes all resources used by ephemeral nuclei objects/instances/types
+func closeEphemeralObjects(u *unsafeOptions) {
+	if u.executerOpts.RateLimiter != nil {
+		u.executerOpts.RateLimiter.Stop()
+	}
+	// dereference all objects that were inherited from base nuclei engine
+	// since these are meant to be closed globally by base nuclei engine
+	u.executerOpts.Output = nil
+	u.executerOpts.IssuesClient = nil
+	u.executerOpts.Interactsh = nil
+	u.executerOpts.HostErrorsCache = nil
+	u.executerOpts.Progress = nil
+	u.executerOpts.Catalog = nil
+	u.executerOpts.Parser = nil
+}
+
 // ThreadSafeNucleiEngine is a tweaked version of nuclei.Engine whose methods are thread-safe
 // and can be used concurrently. Non-thread-safe methods start with Global prefix
 type ThreadSafeNucleiEngine struct {
@@ -95,11 +111,11 @@ func (e *ThreadSafeNucleiEngine) GlobalResultCallback(callback func(event *outpu
 	e.eng.resultCallbacks = []func(*output.ResultEvent){callback}
 }
 
-// ExecuteWithCallback executes templates on targets and calls callback on each result(only if results are found)
+// ExecuteNucleiWithOptsCtx executes templates on targets and calls callback on each result(only if results are found)
 // This method can be called concurrently and it will use some global resources but can be runned parallelly
 // by invoking this method with different options and targets
 // Note: Not all options are thread-safe. this method will throw error if you try to use non-thread-safe options
-func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOpts(targets []string, opts ...NucleiSDKOptions) error {
+func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOptsCtx(ctx context.Context, targets []string, opts ...NucleiSDKOptions) error {
 	baseOpts := *e.eng.opts
 	tmpEngine := &NucleiEngine{opts: &baseOpts, mode: threadSafe}
 	for _, option := range opts {
@@ -107,11 +123,14 @@ func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOpts(targets []string, opts ..
 			return err
 		}
 	}
+	defer tmpEngine.Close()
 	// create ephemeral nuclei objects/instances/types using base nuclei engine
 	unsafeOpts, err := createEphemeralObjects(e.eng, tmpEngine.opts)
 	if err != nil {
 		return err
 	}
+	// cleanup and stop all resources
+	defer closeEphemeralObjects(unsafeOpts)
 
 	// load templates
 	workflowLoader, err := workflow.NewLoader(&unsafeOpts.executerOpts)
@@ -138,12 +157,18 @@ func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOpts(targets []string, opts ..
 	engine := core.New(tmpEngine.opts)
 	engine.SetExecuterOptions(unsafeOpts.executerOpts)
 
-	_ = engine.ExecuteScanWithOpts(store.Templates(), inputProvider, false)
+	_ = engine.ExecuteScanWithOpts(context.Background(), store.Templates(), inputProvider, false)
 
 	engine.WorkPool().Wait()
 	return nil
 }
 
+// ExecuteNucleiWithOpts is same as ExecuteNucleiWithOptsCtx but with default context
+// This is a placeholder and will be deprecated in future major release
+func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOpts(targets []string, opts ...NucleiSDKOptions) error {
+	return e.ExecuteNucleiWithOptsCtx(context.Background(), targets, opts...)
+}
+
 // Close all resources used by nuclei engine
 func (e *ThreadSafeNucleiEngine) Close() {
 	e.eng.Close()

lib/sdk.go

@@ -3,10 +3,11 @@ package nuclei
 import (
 	"bufio"
 	"bytes"
+	"context"
 	"io"
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/authprovider"
-	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk"
+	"github.com/projectdiscovery/nuclei/v3/pkg/catalog"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
 	"github.com/projectdiscovery/nuclei/v3/pkg/core"
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/provider"
@@ -17,6 +18,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
 	"github.com/projectdiscovery/nuclei/v3/pkg/reporting"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
@@ -63,7 +65,7 @@ type NucleiEngine struct {
 
 	// unexported core fields
 	interactshClient *interactsh.Client
-	catalog          *disk.DiskCatalog
+	catalog          catalog.Catalog
 	rateLimiter      *ratelimit.Limiter
 	store            *loader.Store
 	httpxClient      providerTypes.InputLivenessProbe
@@ -183,15 +185,43 @@ func (e *NucleiEngine) SignTemplate(tmplSigner *signer.TemplateSigner, data []by
 
 // Close all resources used by nuclei engine
 func (e *NucleiEngine) Close() {
-	e.interactshClient.Close()
-	e.rc.Close()
-	e.customWriter.Close()
-	e.hostErrCache.Close()
-	e.executerOpts.RateLimiter.Stop()
+	if e.interactshClient != nil {
+		e.interactshClient.Close()
+	}
+	if e.rc != nil {
+		e.rc.Close()
+	}
+	if e.customWriter != nil {
+		e.customWriter.Close()
+	}
+	if e.customProgress != nil {
+		e.customProgress.Stop()
+	}
+	if e.hostErrCache != nil {
+		e.hostErrCache.Close()
+	}
+	if e.executerOpts.RateLimiter != nil {
+		e.executerOpts.RateLimiter.Stop()
+	}
+	if e.rateLimiter != nil {
+		e.rateLimiter.Stop()
+	}
+	// close global shared resources
+	protocolstate.Close()
+	if e.inputProvider != nil {
+		e.inputProvider.Close()
+	}
+	if e.browserInstance != nil {
+		e.browserInstance.Close()
+	}
+	if e.httpxClient != nil {
+		_ = e.httpxClient.Close()
+	}
 }
 
-// ExecuteWithCallback executes templates on targets and calls callback on each result(only if results are found)
-func (e *NucleiEngine) ExecuteWithCallback(callback ...func(event *output.ResultEvent)) error {
+// ExecuteCallbackWithCtx executes templates on targets and calls callback on each result(only if results are found)
+// enable matcher-status option if you expect this callback to be called for all results regardless if it matched or not
+func (e *NucleiEngine) ExecuteCallbackWithCtx(ctx context.Context, callback ...func(event *output.ResultEvent)) error {
 	if !e.templatesLoaded {
 		_ = e.LoadAllTemplates()
 	}
@@ -210,15 +240,23 @@ func (e *NucleiEngine) ExecuteWithCallback(callback ...func(event *output.Result
 	}
 	e.resultCallbacks = append(e.resultCallbacks, filtered...)
 
-	_ = e.engine.ExecuteScanWithOpts(e.store.Templates(), e.inputProvider, false)
+	_ = e.engine.ExecuteScanWithOpts(context.Background(), e.store.Templates(), e.inputProvider, false)
 	defer e.engine.WorkPool().Wait()
 	return nil
 }
 
+// ExecuteWithCallback is same as ExecuteCallbackWithCtx but with default context
+// Note this is deprecated and will be removed in future major release
+func (e *NucleiEngine) ExecuteWithCallback(callback ...func(event *output.ResultEvent)) error {
+	return e.ExecuteCallbackWithCtx(context.Background(), callback...)
+}
+
+// Options return nuclei Type Options
 func (e *NucleiEngine) Options() *types.Options {
 	return e.opts
 }
 
+// Engine returns core Executer of nuclei
 func (e *NucleiEngine) Engine() *core.Engine {
 	return e.engine
 }

lib/sdk_private.go

@@ -35,6 +35,8 @@ import (
 	"github.com/projectdiscovery/ratelimit"
 )
 
+var sharedInit sync.Once = sync.Once{}
+
 // applyRequiredDefaults to options
 func (e *NucleiEngine) applyRequiredDefaults() {
 	mockoutput := testutils.NewMockOutputWriter(e.opts.OmitTemplate)
@@ -116,8 +118,11 @@ func (e *NucleiEngine) init() error {
 
 	e.parser = templates.NewParser()
 
-	_ = protocolstate.Init(e.opts)
-	_ = protocolinit.Init(e.opts)
+	sharedInit.Do(func() {
+		_ = protocolstate.Init(e.opts)
+		_ = protocolinit.Init(e.opts)
+	})
+
 	e.applyRequiredDefaults()
 	var err error
 
@@ -146,7 +151,9 @@ func (e *NucleiEngine) init() error {
 		return err
 	}
 
-	e.catalog = disk.NewCatalog(config.DefaultConfig.TemplatesDirectory)
+	if e.catalog == nil {
+		e.catalog = disk.NewCatalog(config.DefaultConfig.TemplatesDirectory)
+	}
 
 	e.executerOpts = protocols.ExecutorOptions{
 		Output:          e.customWriter,

lib/sdk_test.go

@@ -1,60 +0,0 @@
-package nuclei_test
-
-import (
-	"testing"
-
-	nuclei "github.com/projectdiscovery/nuclei/v3/lib"
-	"github.com/stretchr/testify/require"
-)
-
-func TestSimpleNuclei(t *testing.T) {
-	ne, err := nuclei.NewNucleiEngine(
-		nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}),
-		nuclei.EnableStatsWithOpts(nuclei.StatsOptions{JSON: true}),
-	)
-	require.Nil(t, err)
-	ne.LoadTargets([]string{"scanme.sh"}, false) // probe non http/https target is set to false here
-	// when callback is nil it nuclei will print JSON output to stdout
-	err = ne.ExecuteWithCallback(nil)
-	require.Nil(t, err)
-	defer ne.Close()
-}
-
-func TestSimpleNucleiRemote(t *testing.T) {
-	ne, err := nuclei.NewNucleiEngine(
-		nuclei.WithTemplatesOrWorkflows(
-			nuclei.TemplateSources{
-				RemoteTemplates: []string{"https://cloud.projectdiscovery.io/public/nameserver-fingerprint.yaml"},
-			},
-		),
-	)
-	require.Nil(t, err)
-	ne.LoadTargets([]string{"scanme.sh"}, false) // probe non http/https target is set to false here
-	err = ne.LoadAllTemplates()
-	require.Nil(t, err, "could not load templates")
-	// when callback is nil it nuclei will print JSON output to stdout
-	err = ne.ExecuteWithCallback(nil)
-	require.Nil(t, err)
-	defer ne.Close()
-}
-
-func TestThreadSafeNuclei(t *testing.T) {
-	// create nuclei engine with options
-	ne, err := nuclei.NewThreadSafeNucleiEngine()
-	require.Nil(t, err)
-
-	// scan 1 = run dns templates on scanme.sh
-	t.Run("scanme.sh", func(t *testing.T) {
-		err = ne.ExecuteNucleiWithOpts([]string{"scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}))
-		require.Nil(t, err)
-	})
-
-	// scan 2 = run dns templates on honey.scanme.sh
-	t.Run("honey.scanme.sh", func(t *testing.T) {
-		err = ne.ExecuteNucleiWithOpts([]string{"honey.scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}))
-		require.Nil(t, err)
-	})
-
-	// wait for all scans to finish
-	defer ne.Close()
-}

lib/tests/sdk_test.go

@@ -0,0 +1,132 @@
+package sdk_test
+
+import (
+	"os"
+	"os/exec"
+	"testing"
+	"time"
+
+	nuclei "github.com/projectdiscovery/nuclei/v3/lib"
+	"github.com/projectdiscovery/utils/env"
+	"github.com/stretchr/testify/require"
+	"github.com/tarunKoyalwar/goleak"
+)
+
+var knownLeaks = []goleak.Option{
+	// prettyify the output and generate dependency graph and more details instead of just stack output
+	goleak.Pretty(),
+	// net/http transport maintains idle connections which are closed with cooldown
+	// hence they don't count as leaks
+	goleak.IgnoreAnyFunction("net/http.(*http2ClientConn).readLoop"),
+}
+
+func TestSimpleNuclei(t *testing.T) {
+	fn := func() {
+		defer func() {
+			// resources like leveldb have a delay to commit in-memory resources
+			// to disk, typically 1-2 seconds, so we wait for 2 seconds
+			time.Sleep(2 * time.Second)
+			goleak.VerifyNone(t, knownLeaks...)
+		}()
+		ne, err := nuclei.NewNucleiEngine(
+			nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}),
+			nuclei.EnableStatsWithOpts(nuclei.StatsOptions{JSON: true}),
+		)
+		require.Nil(t, err)
+		ne.LoadTargets([]string{"scanme.sh"}, false) // probe non http/https target is set to false here
+		// when callback is nil it nuclei will print JSON output to stdout
+		err = ne.ExecuteWithCallback(nil)
+		require.Nil(t, err)
+		defer ne.Close()
+	}
+
+	// this is shared test so needs to be run as seperate process
+	if env.GetEnvOrDefault("TestSimpleNuclei", false) {
+		// run as new process
+		cmd := exec.Command(os.Args[0], "-test.run=TestSimpleNuclei")
+		cmd.Env = append(os.Environ(), "TestSimpleNuclei=true")
+		out, err := cmd.CombinedOutput()
+		if err != nil {
+			t.Fatalf("process ran with error %s, output: %s", err, out)
+		}
+	} else {
+		fn()
+	}
+}
+
+func TestSimpleNucleiRemote(t *testing.T) {
+	fn := func() {
+		defer func() {
+			// resources like leveldb have a delay to commit in-memory resources
+			// to disk, typically 1-2 seconds, so we wait for 2 seconds
+			time.Sleep(2 * time.Second)
+			goleak.VerifyNone(t, knownLeaks...)
+		}()
+		ne, err := nuclei.NewNucleiEngine(
+			nuclei.WithTemplatesOrWorkflows(
+				nuclei.TemplateSources{
+					RemoteTemplates: []string{"https://cloud.projectdiscovery.io/public/nameserver-fingerprint.yaml"},
+				},
+			),
+		)
+		require.Nil(t, err)
+		ne.LoadTargets([]string{"scanme.sh"}, false) // probe non http/https target is set to false here
+		err = ne.LoadAllTemplates()
+		require.Nil(t, err, "could not load templates")
+		// when callback is nil it nuclei will print JSON output to stdout
+		err = ne.ExecuteWithCallback(nil)
+		require.Nil(t, err)
+		defer ne.Close()
+	}
+	// this is shared test so needs to be run as seperate process
+	if env.GetEnvOrDefault("TestSimpleNucleiRemote", false) {
+		cmd := exec.Command(os.Args[0], "-test.run=TestSimpleNucleiRemote")
+		cmd.Env = append(os.Environ(), "TestSimpleNucleiRemote=true")
+		out, err := cmd.CombinedOutput()
+		if err != nil {
+			t.Fatalf("process ran with error %s, output: %s", err, out)
+		}
+	} else {
+		fn()
+	}
+}
+
+func TestThreadSafeNuclei(t *testing.T) {
+	fn := func() {
+		defer func() {
+			// resources like leveldb have a delay to commit in-memory resources
+			// to disk, typically 1-2 seconds, so we wait for 2 seconds
+			time.Sleep(2 * time.Second)
+			goleak.VerifyNone(t, knownLeaks...)
+		}()
+		// create nuclei engine with options
+		ne, err := nuclei.NewThreadSafeNucleiEngine()
+		require.Nil(t, err)
+
+		// scan 1 = run dns templates on scanme.sh
+		t.Run("scanme.sh", func(t *testing.T) {
+			err = ne.ExecuteNucleiWithOpts([]string{"scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}))
+			require.Nil(t, err)
+		})
+
+		// scan 2 = run dns templates on honey.scanme.sh
+		t.Run("honey.scanme.sh", func(t *testing.T) {
+			err = ne.ExecuteNucleiWithOpts([]string{"honey.scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}))
+			require.Nil(t, err)
+		})
+
+		// wait for all scans to finish
+		defer ne.Close()
+	}
+
+	if env.GetEnvOrDefault("TestThreadSafeNuclei", false) {
+		cmd := exec.Command(os.Args[0], "-test.run=TestThreadSafeNuclei")
+		cmd.Env = append(os.Environ(), "TestThreadSafeNuclei=true")
+		out, err := cmd.CombinedOutput()
+		if err != nil {
+			t.Fatalf("process ran with error %s, output: %s", err, out)
+		}
+	} else {
+		fn()
+	}
+}

pkg/authprovider/authx/file.go

@@ -219,7 +219,7 @@ func GetAuthDataFromFile(file string) (*Authx, error) {
 	return GetAuthDataFromJSON(bin)
 }
 
-// GetTemplateIDsFromSecretFile reads the template IDs from the secret file
+// GetTemplatePathsFromSecretFile reads the template IDs from the secret file
 func GetTemplatePathsFromSecretFile(file string) ([]string, error) {
 	auth, err := GetAuthDataFromFile(file)
 	if err != nil {

pkg/catalog/config/constants.go

@@ -31,7 +31,7 @@ const (
 	CLIConfigFileName               = "config.yaml"
 	ReportingConfigFilename         = "reporting-config.yaml"
 	// Version is the current version of nuclei
-	Version = `v3.2.4`
+	Version = `v3.2.7`
 	// Directory Names of custom templates
 	CustomS3TemplatesDirName     = "s3"
 	CustomGitHubTemplatesDirName = "github"

pkg/catalog/config/nucleiconfig.go

@@ -166,12 +166,12 @@ func (c *Config) GetTemplateIndexFilePath() string {
 	return filepath.Join(c.TemplatesDirectory, NucleiTemplatesIndexFileName)
 }
 
-// GetTemplatesConfigFilePath returns checksum file path of nuclei templates
+// GetChecksumFilePath returns checksum file path of nuclei templates
 func (c *Config) GetChecksumFilePath() string {
 	return filepath.Join(c.TemplatesDirectory, NucleiTemplatesCheckSumFileName)
 }
 
-// GetCLIOptsConfigFilePath returns the nuclei cli config file path
+// GetFlagsConfigFilePath returns the nuclei cli config file path
 func (c *Config) GetFlagsConfigFilePath() string {
 	return filepath.Join(c.configDir, CLIConfigFileName)
 }

pkg/catalog/loader/loader.go

@@ -255,6 +255,46 @@ func init() {
 	templateIDPathMap = make(map[string]string)
 }
 
+// LoadTemplatesOnlyMetadata loads only the metadata of the templates
+func (store *Store) LoadTemplatesOnlyMetadata() error {
+	templatePaths, errs := store.config.Catalog.GetTemplatesPath(store.finalTemplates)
+	store.logErroredTemplates(errs)
+
+	filteredTemplatePaths := store.pathFilter.Match(templatePaths)
+
+	validPaths := make(map[string]struct{})
+	for templatePath := range filteredTemplatePaths {
+		loaded, err := store.config.ExecutorOptions.Parser.LoadTemplate(templatePath, store.tagFilter, nil, store.config.Catalog)
+		if loaded || store.pathFilter.MatchIncluded(templatePath) {
+			validPaths[templatePath] = struct{}{}
+		}
+		if err != nil {
+			if strings.Contains(err.Error(), templates.ErrExcluded.Error()) {
+				stats.Increment(templates.TemplatesExcludedStats)
+				if config.DefaultConfig.LogAllEvents {
+					gologger.Print().Msgf("[%v] %v\n", aurora.Yellow("WRN").String(), err.Error())
+				}
+				continue
+			}
+			gologger.Warning().Msg(err.Error())
+		}
+	}
+	parserItem, ok := store.config.ExecutorOptions.Parser.(*templates.Parser)
+	if !ok {
+		return errors.New("invalid parser")
+	}
+	templatesCache := parserItem.Cache()
+
+	for templatePath := range validPaths {
+		template, _, _ := templatesCache.Has(templatePath)
+		if template != nil {
+			template.Path = templatePath
+			store.templates = append(store.templates, template)
+		}
+	}
+	return nil
+}
+
 // ValidateTemplates takes a list of templates and validates them
 // erroring out on discovering any faulty templates.
 func (store *Store) ValidateTemplates() error {

pkg/core/execute_options.go

@@ -1,6 +1,7 @@
 package core
 
 import (
+	"context"
 	"sync"
 	"sync/atomic"
 
@@ -20,18 +21,18 @@ import (
 //
 // All the execution logic for the templates/workflows happens in this part
 // of the engine.
-func (e *Engine) Execute(templates []*templates.Template, target provider.InputProvider) *atomic.Bool {
-	return e.ExecuteScanWithOpts(templates, target, false)
+func (e *Engine) Execute(ctx context.Context, templates []*templates.Template, target provider.InputProvider) *atomic.Bool {
+	return e.ExecuteScanWithOpts(ctx, templates, target, false)
 }
 
 // ExecuteWithResults a list of templates with results
-func (e *Engine) ExecuteWithResults(templatesList []*templates.Template, target provider.InputProvider, callback func(*output.ResultEvent)) *atomic.Bool {
+func (e *Engine) ExecuteWithResults(ctx context.Context, templatesList []*templates.Template, target provider.InputProvider, callback func(*output.ResultEvent)) *atomic.Bool {
 	e.Callback = callback
-	return e.ExecuteScanWithOpts(templatesList, target, false)
+	return e.ExecuteScanWithOpts(ctx, templatesList, target, false)
 }
 
 // ExecuteScanWithOpts executes scan with given scanStrategy
-func (e *Engine) ExecuteScanWithOpts(templatesList []*templates.Template, target provider.InputProvider, noCluster bool) *atomic.Bool {
+func (e *Engine) ExecuteScanWithOpts(ctx context.Context, templatesList []*templates.Template, target provider.InputProvider, noCluster bool) *atomic.Bool {
 	results := &atomic.Bool{}
 	selfcontainedWg := &sync.WaitGroup{}
 
@@ -83,14 +84,14 @@ func (e *Engine) ExecuteScanWithOpts(templatesList []*templates.Template, target
 	}
 
 	// Execute All SelfContained in parallel
-	e.executeAllSelfContained(selfContained, results, selfcontainedWg)
+	e.executeAllSelfContained(ctx, selfContained, results, selfcontainedWg)
 
 	strategyResult := &atomic.Bool{}
 	switch e.options.ScanStrategy {
 	case scanstrategy.TemplateSpray.String():
-		strategyResult = e.executeTemplateSpray(filtered, target)
+		strategyResult = e.executeTemplateSpray(ctx, filtered, target)
 	case scanstrategy.HostSpray.String():
-		strategyResult = e.executeHostSpray(filtered, target)
+		strategyResult = e.executeHostSpray(ctx, filtered, target)
 	}
 
 	results.CompareAndSwap(false, strategyResult.Load())
@@ -100,7 +101,7 @@ func (e *Engine) ExecuteScanWithOpts(templatesList []*templates.Template, target
 }
 
 // executeTemplateSpray executes scan using template spray strategy where targets are iterated over each template
-func (e *Engine) executeTemplateSpray(templatesList []*templates.Template, target provider.InputProvider) *atomic.Bool {
+func (e *Engine) executeTemplateSpray(ctx context.Context, templatesList []*templates.Template, target provider.InputProvider) *atomic.Bool {
 	results := &atomic.Bool{}
 
 	// wp is workpool that contains different waitgroups for
@@ -108,6 +109,12 @@ func (e *Engine) executeTemplateSpray(templatesList []*templates.Template, targe
 	wp := e.GetWorkPool()
 
 	for _, template := range templatesList {
+		select {
+		case <-ctx.Done():
+			return results
+		default:
+		}
+
 		// resize check point - nop if there are no changes
 		wp.RefreshWithConfig(e.GetWorkPoolConfig())
 
@@ -125,7 +132,7 @@ func (e *Engine) executeTemplateSpray(templatesList []*templates.Template, targe
 			// All other request types are executed here
 			// Note: executeTemplateWithTargets creates goroutines and blocks
 			// given template is executed on all targets
-			e.executeTemplateWithTargets(tpl, target, results)
+			e.executeTemplateWithTargets(ctx, tpl, target, results)
 		}(template)
 	}
 	wp.Wait()
@@ -133,15 +140,21 @@ func (e *Engine) executeTemplateSpray(templatesList []*templates.Template, targe
 }
 
 // executeHostSpray executes scan using host spray strategy where templates are iterated over each target
-func (e *Engine) executeHostSpray(templatesList []*templates.Template, target provider.InputProvider) *atomic.Bool {
+func (e *Engine) executeHostSpray(ctx context.Context, templatesList []*templates.Template, target provider.InputProvider) *atomic.Bool {
 	results := &atomic.Bool{}
 	wp, _ := syncutil.New(syncutil.WithSize(e.options.BulkSize + e.options.HeadlessBulkSize))
 
 	target.Iterate(func(value *contextargs.MetaInput) bool {
+		select {
+		case <-ctx.Done():
+			return false
+		default:
+		}
+
 		wp.Add()
 		go func(targetval *contextargs.MetaInput) {
 			defer wp.Done()
-			e.executeTemplatesOnTarget(templatesList, targetval, results)
+			e.executeTemplatesOnTarget(ctx, templatesList, targetval, results)
 		}(value)
 		return true
 	})

pkg/core/executors.go

@@ -1,6 +1,7 @@
 package core
 
 import (
+	"context"
 	"sync"
 	"sync/atomic"
 
@@ -17,14 +18,14 @@ import (
 // Executors are low level executors that deals with template execution on a target
 
 // executeAllSelfContained executes all self contained templates that do not use `target`
-func (e *Engine) executeAllSelfContained(alltemplates []*templates.Template, results *atomic.Bool, sg *sync.WaitGroup) {
+func (e *Engine) executeAllSelfContained(ctx context.Context, alltemplates []*templates.Template, results *atomic.Bool, sg *sync.WaitGroup) {
 	for _, v := range alltemplates {
 		sg.Add(1)
 		go func(template *templates.Template) {
 			defer sg.Done()
 			var err error
 			var match bool
-			ctx := scan.NewScanContext(contextargs.New())
+			ctx := scan.NewScanContext(ctx, contextargs.New(ctx))
 			if e.Callback != nil {
 				if results, err := template.Executer.ExecuteWithResults(ctx); err != nil {
 					for _, result := range results {
@@ -45,7 +46,7 @@ func (e *Engine) executeAllSelfContained(alltemplates []*templates.Template, res
 }
 
 // executeTemplateWithTarget executes a given template on x targets (with a internal targetpool(i.e concurrency))
-func (e *Engine) executeTemplateWithTargets(template *templates.Template, target provider.InputProvider, results *atomic.Bool) {
+func (e *Engine) executeTemplateWithTargets(ctx context.Context, template *templates.Template, target provider.InputProvider, results *atomic.Bool) {
 	// this is target pool i.e max target to execute
 	wg := e.workPool.InputPool(template.Type())
 
@@ -77,6 +78,12 @@ func (e *Engine) executeTemplateWithTargets(template *templates.Template, target
 	}
 
 	target.Iterate(func(scannedValue *contextargs.MetaInput) bool {
+		select {
+		case <-ctx.Done():
+			return false // exit
+		default:
+		}
+
 		// Best effort to track the host progression
 		// skips indexes lower than the minimum in-flight at interruption time
 		var skip bool
@@ -114,9 +121,9 @@ func (e *Engine) executeTemplateWithTargets(template *templates.Template, target
 
 			var match bool
 			var err error
-			ctxArgs := contextargs.New()
+			ctxArgs := contextargs.New(ctx)
 			ctxArgs.MetaInput = value
-			ctx := scan.NewScanContext(ctxArgs)
+			ctx := scan.NewScanContext(ctx, ctxArgs)
 			switch template.Type() {
 			case types.WorkflowProtocol:
 				match = e.executeWorkflow(ctx, template.CompiledWorkflow)
@@ -149,7 +156,7 @@ func (e *Engine) executeTemplateWithTargets(template *templates.Template, target
 }
 
 // executeTemplatesOnTarget execute given templates on given single target
-func (e *Engine) executeTemplatesOnTarget(alltemplates []*templates.Template, target *contextargs.MetaInput, results *atomic.Bool) {
+func (e *Engine) executeTemplatesOnTarget(ctx context.Context, alltemplates []*templates.Template, target *contextargs.MetaInput, results *atomic.Bool) {
 	// all templates are executed on single target
 
 	// wp is workpool that contains different waitgroups for
@@ -158,6 +165,12 @@ func (e *Engine) executeTemplatesOnTarget(alltemplates []*templates.Template, ta
 	wp := e.GetWorkPool()
 
 	for _, tpl := range alltemplates {
+		select {
+		case <-ctx.Done():
+			return
+		default:
+		}
+
 		// resize check point - nop if there are no changes
 		wp.RefreshWithConfig(e.GetWorkPoolConfig())
 
@@ -173,9 +186,9 @@ func (e *Engine) executeTemplatesOnTarget(alltemplates []*templates.Template, ta
 
 			var match bool
 			var err error
-			ctxArgs := contextargs.New()
+			ctxArgs := contextargs.New(ctx)
 			ctxArgs.MetaInput = value
-			ctx := scan.NewScanContext(ctxArgs)
+			ctx := scan.NewScanContext(ctx, ctxArgs)
 			switch template.Type() {
 			case types.WorkflowProtocol:
 				match = e.executeWorkflow(ctx, template.CompiledWorkflow)
@@ -199,52 +212,3 @@ func (e *Engine) executeTemplatesOnTarget(alltemplates []*templates.Template, ta
 	}
 	wp.Wait()
 }
-
-type ChildExecuter struct {
-	e *Engine
-
-	results *atomic.Bool
-}
-
-// Close closes the executer returning bool results
-func (e *ChildExecuter) Close() *atomic.Bool {
-	e.e.workPool.Wait()
-	return e.results
-}
-
-// Execute executes a template and URLs
-func (e *ChildExecuter) Execute(template *templates.Template, value *contextargs.MetaInput) {
-	templateType := template.Type()
-
-	// resize check point - nop if there are no changes
-	e.e.WorkPool().RefreshWithConfig(e.e.GetWorkPoolConfig())
-
-	var wg *syncutil.AdaptiveWaitGroup
-	if templateType == types.HeadlessProtocol {
-		wg = e.e.workPool.Headless
-	} else {
-		wg = e.e.workPool.Default
-	}
-
-	wg.Add()
-	go func(tpl *templates.Template) {
-		defer wg.Done()
-
-		ctxArgs := contextargs.New()
-		ctxArgs.MetaInput = value
-		ctx := scan.NewScanContext(ctxArgs)
-		match, err := template.Executer.Execute(ctx)
-		if err != nil {
-			gologger.Warning().Msgf("[%s] Could not execute step: %s\n", e.e.executerOpts.Colorizer.BrightBlue(template.ID), err)
-		}
-		e.results.CompareAndSwap(false, match)
-	}(template)
-}
-
-// ExecuteWithOpts executes with the full options
-func (e *Engine) ChildExecuter() *ChildExecuter {
-	return &ChildExecuter{
-		e:       e,
-		results: &atomic.Bool{},
-	}
-}

pkg/core/workflow_execute.go

@@ -21,7 +21,7 @@ func (e *Engine) executeWorkflow(ctx *scan.ScanContext, w *workflows.Workflow) b
 
 	// at this point we should be at the start root execution of a workflow tree, hence we create global shared instances
 	workflowCookieJar, _ := cookiejar.New(nil)
-	ctxArgs := contextargs.New()
+	ctxArgs := contextargs.New(ctx.Context())
 	ctxArgs.MetaInput = ctx.Input.MetaInput
 	ctxArgs.CookieJar = workflowCookieJar
 
@@ -139,7 +139,7 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan
 							defer swg.Done()
 
 							// create a new context with the same input but with unset callbacks
-							subCtx := scan.NewScanContext(ctx.Input)
+							subCtx := scan.NewScanContext(ctx.Context(), ctx.Input)
 							if err := e.runWorkflowStep(subtemplate, subCtx, results, swg, w); err != nil {
 								gologger.Warning().Msgf(workflowStepExecutionError, subtemplate.Template, err)
 							}
@@ -165,7 +165,7 @@ func (e *Engine) runWorkflowStep(template *workflows.WorkflowTemplate, ctx *scan
 
 			go func(template *workflows.WorkflowTemplate) {
 				// create a new context with the same input but with unset callbacks
-				subCtx := scan.NewScanContext(ctx.Input)
+				subCtx := scan.NewScanContext(ctx.Context(), ctx.Input)
 				if err := e.runWorkflowStep(template, subCtx, results, swg, w); err != nil {
 					gologger.Warning().Msgf(workflowStepExecutionError, template.Template, err)
 				}

pkg/core/workflow_execute_test.go

@@ -1,6 +1,7 @@
 package core
 
 import (
+	"context"
 	"testing"
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/stringslice"
@@ -25,8 +26,8 @@ func TestWorkflowsSimple(t *testing.T) {
 	}}
 
 	engine := &Engine{}
-	input := contextargs.NewWithInput("https://test.com")
-	ctx := scan.NewScanContext(input)
+	input := contextargs.NewWithInput(context.Background(), "https://test.com")
+	ctx := scan.NewScanContext(context.Background(), input)
 	matched := engine.executeWorkflow(ctx, workflow)
 	require.True(t, matched, "could not get correct match value")
 }
@@ -49,8 +50,8 @@ func TestWorkflowsSimpleMultiple(t *testing.T) {
 	}}
 
 	engine := &Engine{}
-	input := contextargs.NewWithInput("https://test.com")
-	ctx := scan.NewScanContext(input)
+	input := contextargs.NewWithInput(context.Background(), "https://test.com")
+	ctx := scan.NewScanContext(context.Background(), input)
 	matched := engine.executeWorkflow(ctx, workflow)
 	require.True(t, matched, "could not get correct match value")
 
@@ -77,8 +78,8 @@ func TestWorkflowsSubtemplates(t *testing.T) {
 	}}
 
 	engine := &Engine{}
-	input := contextargs.NewWithInput("https://test.com")
-	ctx := scan.NewScanContext(input)
+	input := contextargs.NewWithInput(context.Background(), "https://test.com")
+	ctx := scan.NewScanContext(context.Background(), input)
 	matched := engine.executeWorkflow(ctx, workflow)
 	require.True(t, matched, "could not get correct match value")
 
@@ -103,8 +104,8 @@ func TestWorkflowsSubtemplatesNoMatch(t *testing.T) {
 	}}
 
 	engine := &Engine{}
-	input := contextargs.NewWithInput("https://test.com")
-	ctx := scan.NewScanContext(input)
+	input := contextargs.NewWithInput(context.Background(), "https://test.com")
+	ctx := scan.NewScanContext(context.Background(), input)
 	matched := engine.executeWorkflow(ctx, workflow)
 	require.False(t, matched, "could not get correct match value")
 
@@ -134,8 +135,8 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) {
 	}}
 
 	engine := &Engine{}
-	input := contextargs.NewWithInput("https://test.com")
-	ctx := scan.NewScanContext(input)
+	input := contextargs.NewWithInput(context.Background(), "https://test.com")
+	ctx := scan.NewScanContext(context.Background(), input)
 	matched := engine.executeWorkflow(ctx, workflow)
 	require.True(t, matched, "could not get correct match value")
 
@@ -165,8 +166,8 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) {
 	}}
 
 	engine := &Engine{}
-	input := contextargs.NewWithInput("https://test.com")
-	ctx := scan.NewScanContext(input)
+	input := contextargs.NewWithInput(context.Background(), "https://test.com")
+	ctx := scan.NewScanContext(context.Background(), input)
 	matched := engine.executeWorkflow(ctx, workflow)
 	require.False(t, matched, "could not get correct match value")
 

pkg/core/workpool.go

@@ -1,6 +1,9 @@
 package core
 
 import (
+	"context"
+
+	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	syncutil "github.com/projectdiscovery/utils/sync"
 )
@@ -71,14 +74,18 @@ func (w *WorkPool) RefreshWithConfig(config WorkPoolConfig) {
 	if w.config.HeadlessInputConcurrency != config.HeadlessInputConcurrency {
 		w.config.HeadlessInputConcurrency = config.HeadlessInputConcurrency
 	}
-	w.Refresh()
+	w.Refresh(context.Background())
 }
 
-func (w *WorkPool) Refresh() {
+func (w *WorkPool) Refresh(ctx context.Context) {
 	if w.Default.Size != w.config.TypeConcurrency {
-		w.Default.Resize(w.config.TypeConcurrency)
+		if err := w.Default.Resize(ctx, w.config.TypeConcurrency); err != nil {
+			gologger.Warning().Msgf("Could not resize workpool: %s\n", err)
+		}
 	}
 	if w.Headless.Size != w.config.HeadlessTypeConcurrency {
-		w.Headless.Resize(w.config.HeadlessTypeConcurrency)
+		if err := w.Headless.Resize(ctx, w.config.HeadlessTypeConcurrency); err != nil {
+			gologger.Warning().Msgf("Could not resize workpool: %s\n", err)
+		}
 	}
 }

pkg/fuzz/component/headers.go

@@ -83,6 +83,9 @@ func (q *Header) Delete(key string) error {
 func (q *Header) Rebuild() (*retryablehttp.Request, error) {
 	cloned := q.req.Clone(context.Background())
 	q.value.parsed.Iterate(func(key string, value any) bool {
+		if strings.TrimSpace(key) == "" {
+			return true
+		}
 		if strings.EqualFold(key, "Host") {
 			return true
 		}

pkg/fuzz/execute.go

@@ -57,6 +57,8 @@ type GeneratedRequest struct {
 	DynamicValues map[string]interface{}
 	// Component is the component for the request
 	Component component.Component
+	// Parameter being fuzzed
+	Parameter string
 }
 
 // Execute executes a fuzzing rule accepting a callback on which
@@ -211,11 +213,19 @@ func (rule *Rule) executeRuleValues(input *ExecuteRuleInput, ruleComponent compo
 		})
 		// if mode is multiple now build and execute it
 		if rule.modeType == multipleModeType {
+			rule.Fuzz.KV.Iterate(func(key, value string) bool {
+				var evaluated string
+				evaluated, input.InteractURLs = rule.executeEvaluate(input, key, "", value, input.InteractURLs)
+				if err := ruleComponent.SetValue(key, evaluated); err != nil {
+					return true
+				}
+				return true
+			})
 			req, err := ruleComponent.Rebuild()
 			if err != nil {
 				return err
 			}
-			if gotErr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent); gotErr != nil {
+			if gotErr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, ""); gotErr != nil {
 				return gotErr
 			}
 		}

pkg/fuzz/parts.go

@@ -68,7 +68,7 @@ func (rule *Rule) executePartComponentOnValues(input *ExecuteRuleInput, payloadS
 				return err
 			}
 
-			if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent); qerr != nil {
+			if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, key); qerr != nil {
 				return qerr
 			}
 			// fmt.Printf("executed with value: %s\n", evaluated)
@@ -90,7 +90,7 @@ func (rule *Rule) executePartComponentOnValues(input *ExecuteRuleInput, payloadS
 		if err != nil {
 			return err
 		}
-		if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent); qerr != nil {
+		if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, ""); qerr != nil {
 			err = qerr
 			return err
 		}
@@ -125,7 +125,7 @@ func (rule *Rule) executePartComponentOnKV(input *ExecuteRuleInput, payload Valu
 				return err
 			}
 
-			if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent); qerr != nil {
+			if qerr := rule.execWithInput(input, req, input.InteractURLs, ruleComponent, key); qerr != nil {
 				return err
 			}
 
@@ -144,12 +144,13 @@ func (rule *Rule) executePartComponentOnKV(input *ExecuteRuleInput, payload Valu
 }
 
 // execWithInput executes a rule with input via callback
-func (rule *Rule) execWithInput(input *ExecuteRuleInput, httpReq *retryablehttp.Request, interactURLs []string, component component.Component) error {
+func (rule *Rule) execWithInput(input *ExecuteRuleInput, httpReq *retryablehttp.Request, interactURLs []string, component component.Component, parameter string) error {
 	request := GeneratedRequest{
 		Request:       httpReq,
 		InteractURLs:  interactURLs,
 		DynamicValues: input.Values,
 		Component:     component,
+		Parameter:     parameter,
 	}
 	if !input.Callback(request) {
 		return types.ErrNoMoreRequests

pkg/input/formats/json/json.go

@@ -28,9 +28,10 @@ var _ formats.Format = &JSONFormat{}
 type proxifyRequest struct {
 	URL     string `json:"url"`
 	Request struct {
-		Header map[string]string `json:"header"`
-		Body   string            `json:"body"`
-		Raw    string            `json:"raw"`
+		Header   map[string]string `json:"header"`
+		Body     string            `json:"body"`
+		Raw      string            `json:"raw"`
+		Endpoint string            `json:"endpoint"`
 	} `json:"request"`
 }
 
@@ -63,6 +64,9 @@ func (j *JSONFormat) Parse(input string, resultsCb formats.ParseReqRespCallback)
 			return errors.Wrap(err, "could not decode json file")
 		}
 
+		if request.URL == "" && request.Request.Endpoint != "" {
+			request.URL = request.Request.Endpoint
+		}
 		rawRequest, err := types.ParseRawRequestWithURL(request.Request.Raw, request.URL)
 		if err != nil {
 			gologger.Warning().Msgf("jsonl: Could not parse raw request %s: %s\n", request.URL, err)

pkg/input/formats/openapi/generator.go

@@ -179,6 +179,10 @@ func generateRequestsFromOp(opts *generateReqOptions) error {
 	for _, parameter := range reqParams {
 		value := parameter.Value
 
+		if value.Schema == nil || value.Schema.Value == nil {
+			continue
+		}
+
 		// paramValue or default value to use
 		var paramValue interface{}
 

pkg/input/provider/list/hmap_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils/expand"
+	"github.com/projectdiscovery/utils/auth/pdcp"
 	"github.com/stretchr/testify/require"
 )
 
@@ -68,7 +69,7 @@ func (m *mockDnsHandler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) {
 		msg.Authoritative = true
 		domain := msg.Question[0].Name
 		msg.Answer = append(msg.Answer, &dns.AAAA{
-			Hdr:  dns.RR_Header{Name: domain, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60},
+			Hdr:  dns.RR_Header{Name: domain, Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 60},
 			AAAA: net.ParseIP("2400:6180:0:d0::91:1001"),
 		})
 	}
@@ -153,6 +154,13 @@ func Test_scanallips_normalizeStoreInputValue(t *testing.T) {
 }
 
 func Test_expandASNInputValue(t *testing.T) {
+	// skip this test if pdcp keys are not present
+	h := pdcp.PDCPCredHandler{}
+	creds, err := h.GetCreds()
+	if err != nil || creds == nil || creds.APIKey == "" {
+		t.Logf("Skipping asnmap test as pdcp keys are not present")
+		t.SkipNow()
+	}
 	tests := []struct {
 		asn                string
 		expectedOutputFile string

pkg/input/types/http.go

@@ -12,6 +12,7 @@ import (
 	"sync"
 
 	"github.com/projectdiscovery/retryablehttp-go"
+	"github.com/projectdiscovery/useragent"
 	"github.com/projectdiscovery/utils/conversion"
 	mapsutil "github.com/projectdiscovery/utils/maps"
 	urlutil "github.com/projectdiscovery/utils/url"
@@ -73,6 +74,10 @@ func (rr *RequestResponse) BuildRequest() (*retryablehttp.Request, error) {
 			req.Header.Add(k, v)
 			return true
 		})
+		if req.Header.Get("User-Agent") == "" {
+			userAgent := useragent.PickRandom()
+			req.Header.Set("User-Agent", userAgent.Raw)
+		}
 		rr.req = req
 	})
 	return rr.req, rr.reqErr

pkg/input/types/probe.go

@@ -4,4 +4,6 @@ package types
 type InputLivenessProbe interface {
 	// ProbeURL probes the scheme for a URL. first HTTPS is tried
 	ProbeURL(input string) (string, error)
+	// Close closes the liveness probe
+	Close() error
 }

pkg/installer/versioncheck.go

@@ -83,6 +83,10 @@ func UpdateIgnoreFile() error {
 }
 
 func doVersionCheck(isSDK bool) error {
+	// we use global retryablehttp client so its not immeditely gc'd if any references are held
+	// and according our config we have idle connections which are shown as leaked by goleak in tests
+	// i.e we close all idle connections after our use and it doesn't affect any other part of the code
+	defer retryableHttpClient.HTTPClient.CloseIdleConnections()
 	resp, err := retryableHttpClient.Get(pdtmNucleiVersionEndpoint + "?" + getpdtmParams(isSDK))
 	if err != nil {
 		return err

pkg/js/compiler/compiler.go

@@ -37,6 +37,8 @@ type ExecuteOptions struct {
 	// Source is original source of the script
 	Source *string
 
+	Context context.Context
+
 	// Manually exported objects
 	exports map[string]interface{}
 }
@@ -77,13 +79,13 @@ func (c *Compiler) Execute(code string, args *ExecuteArgs) (ExecuteResult, error
 	if err != nil {
 		return nil, err
 	}
-	return c.ExecuteWithOptions(p, args, &ExecuteOptions{})
+	return c.ExecuteWithOptions(p, args, &ExecuteOptions{Context: context.Background()})
 }
 
 // ExecuteWithOptions executes a script with the provided options.
 func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs, opts *ExecuteOptions) (ExecuteResult, error) {
 	if opts == nil {
-		opts = &ExecuteOptions{}
+		opts = &ExecuteOptions{Context: context.Background()}
 	}
 	if args == nil {
 		args = NewExecuteArgs()
@@ -105,7 +107,7 @@ func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs,
 	}
 
 	// execute with context and timeout
-	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(opts.Timeout)*time.Second)
+	ctx, cancel := context.WithTimeout(opts.Context, time.Duration(opts.Timeout)*time.Second)
 	defer cancel()
 	// execute the script
 	results, err := contextutil.ExecFuncWithTwoReturns(ctx, func() (val goja.Value, err error) {

pkg/js/compiler/non-pool.go

@@ -8,7 +8,7 @@ import (
 )
 
 var (
-	ephemeraljsc, _ = syncutil.New(syncutil.WithSize(NonPoolingVMConcurrency))
+	ephemeraljsc    *syncutil.AdaptiveWaitGroup
 	lazyFixedSgInit = sync.OnceFunc(func() {
 		ephemeraljsc, _ = syncutil.New(syncutil.WithSize(NonPoolingVMConcurrency))
 	})

pkg/js/compiler/pool.go

@@ -2,6 +2,7 @@ package compiler
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"fmt"
 	"reflect"
@@ -55,10 +56,12 @@ var (
 	lazySgInit = sync.OnceFunc(func() {
 		pooljsc, _ = syncutil.New(syncutil.WithSize(PoolingJsVmConcurrency))
 	})
-	sgResizeCheck = func() {
+	sgResizeCheck = func(ctx context.Context) {
 		// resize check point
 		if pooljsc.Size != PoolingJsVmConcurrency {
-			pooljsc.Resize(PoolingJsVmConcurrency)
+			if err := pooljsc.Resize(ctx, PoolingJsVmConcurrency); err != nil {
+				gologger.Warning().Msgf("Could not resize workpool: %s\n", err)
+			}
 		}
 	}
 )
@@ -122,7 +125,7 @@ func executeWithPoolingProgram(p *goja.Program, args *ExecuteArgs, opts *Execute
 	// its unknown (most likely cannot be done) to limit max js runtimes at a moment without making it static
 	// unlike sync.Pool which reacts to GC and its purposes is to reuse objects rather than creating new ones
 	lazySgInit()
-	sgResizeCheck()
+	sgResizeCheck(opts.Context)
 
 	pooljsc.Add()
 	defer pooljsc.Done()

pkg/model/model_test.go

@@ -80,7 +80,7 @@ func TestUnmarshal(t *testing.T) {
 	templateName := "Test Template"
 	authors := []string{"forgedhallpass", "ice3man"}
 	tags := []string{"cve", "misc"}
-	references := []string{"http://test.com", "http://domain.com"}
+	references := []string{"http://test.com", "http://Domain.com"}
 
 	dynamicKey1 := "customDynamicKey1"
 	dynamicKey2 := "customDynamicKey2"
@@ -109,7 +109,7 @@ func TestUnmarshal(t *testing.T) {
   author: ` + strings.Join(authors, ", ") + `
   tags: ` + strings.Join(tags, ", ") + `
   severity: critical
-  reference: ` + strings.Join(references, ", ") + `
+  reference: ` + strings.Join(references, ",") + `
   metadata:
      ` + dynamicKey1 + `: ` + dynamicKeysMap[dynamicKey1].(string) + `
      ` + dynamicKey2 + `: ` + dynamicKeysMap[dynamicKey2].(string) + `

pkg/model/types/severity/severities.go

@@ -26,6 +26,14 @@ func (severities *Severities) Set(values string) error {
 	return nil
 }
 
+func (severities Severities) MarshalYAML() (interface{}, error) {
+	var stringSeverities = make([]string, 0, len(severities))
+	for _, severity := range severities {
+		stringSeverities = append(stringSeverities, severity.String())
+	}
+	return stringSeverities, nil
+}
+
 func (severities *Severities) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	var stringSliceValue stringslice.StringSlice
 	if err := unmarshal(&stringSliceValue); err != nil {

pkg/model/types/severity/severity_test.go

@@ -74,3 +74,12 @@ func TestMarshalJSON(t *testing.T) {
 	}
 	require.Equal(t, "[\"low\",\"medium\"]", string(data), "could not marshal json")
 }
+
+func TestSeveritiesMarshalYaml(t *testing.T) {
+	unmarshalled := Severities{Low, Medium}
+	marshalled, err := yaml.Marshal(unmarshalled)
+	if err != nil {
+		panic(err)
+	}
+	require.Equal(t, "- low\n- medium\n", string(marshalled), "could not marshal yaml")
+}

pkg/model/types/stringslice/stringslice_raw.go

@@ -12,6 +12,15 @@ func (rawStringSlice *RawStringSlice) Normalize(value string) string {
 	return value
 }
 
+func (rawStringSlice *RawStringSlice) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	marshalledSlice, err := marshalStringToSlice(unmarshal)
+	if err != nil {
+		return err
+	}
+	rawStringSlice.Value = marshalledSlice
+	return nil
+}
+
 func (rawStringSlice RawStringSlice) JSONSchemaAlias() any {
 	return StringOrSlice("")
 }

pkg/output/format_screen.go

@@ -106,5 +106,21 @@ func (w *StandardWriter) formatScreen(output *ResultEvent) []byte {
 		}
 		builder.WriteString("]")
 	}
+
+	// If it is a fuzzing output, enrich with additional
+	// metadata for the match.
+	if output.IsFuzzingResult {
+		if output.FuzzingParameter != "" {
+			builder.WriteString(" [")
+			builder.WriteString(output.FuzzingPosition)
+			builder.WriteRune(':')
+			builder.WriteString(w.aurora.BrightMagenta(output.FuzzingParameter).String())
+			builder.WriteString("]")
+		}
+
+		builder.WriteString(" [")
+		builder.WriteString(output.FuzzingMethod)
+		builder.WriteString("]")
+	}
 	return builder.Bytes()
 }

pkg/output/output.go

@@ -175,6 +175,14 @@ type ResultEvent struct {
 	// must be enabled by setting protocols.ExecuterOptions.ExportReqURLPattern to true
 	ReqURLPattern string `json:"req_url_pattern,omitempty"`
 
+	// Fields related to HTTP Fuzzing functionality of nuclei.
+	// The output contains additional fields when the result is
+	// for a fuzzing template.
+	IsFuzzingResult  bool   `json:"is_fuzzing_result,omitempty"`
+	FuzzingMethod    string `json:"fuzzing_method,omitempty"`
+	FuzzingParameter string `json:"fuzzing_parameter,omitempty"`
+	FuzzingPosition  string `json:"fuzzing_position,omitempty"`
+
 	FileToIndexPosition map[string]int `json:"-"`
 	Error               string         `json:"error,omitempty"`
 }

pkg/protocols/code/code.go

@@ -199,6 +199,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 				Source:   &request.PreCondition,
 				Callback: registerPreConditionFunctions,
 				Cleanup:  cleanUpPreConditionFunctions,
+				Context:  input.Context(),
 			})
 		if err != nil {
 			return errorutil.NewWithTag(request.TemplateID, "could not execute pre-condition: %s", err)

pkg/protocols/code/code_test.go

@@ -3,6 +3,7 @@
 package code
 
 import (
+	"context"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -31,7 +32,7 @@ func TestCodeProtocol(t *testing.T) {
 	require.Nil(t, err, "could not compile code request")
 
 	var gotEvent output.InternalEvent
-	ctxArgs := contextargs.NewWithInput("")
+	ctxArgs := contextargs.NewWithInput(context.Background(), "")
 	err = request.ExecuteWithResults(ctxArgs, nil, nil, func(event *output.InternalWrappedEvent) {
 		gotEvent = event.InternalEvent
 	})

pkg/protocols/common/automaticscan/automaticscan.go

@@ -1,6 +1,7 @@
 package automaticscan
 
 import (
+	"context"
 	"io"
 	"net/http"
 	"os"
@@ -55,7 +56,6 @@ type Service struct {
 	engine             *core.Engine
 	target             provider.InputProvider
 	wappalyzer         *wappalyzer.Wappalyze
-	childExecuter      *core.ChildExecuter
 	httpclient         *retryablehttp.Client
 	templateDirs       []string // root Template Directories
 	technologyMappings map[string]string
@@ -94,7 +94,6 @@ func New(opts Options) (*Service, error) {
 		return nil, err
 	}
 
-	childExecuter := opts.Engine.ChildExecuter()
 	httpclient, err := httpclientpool.Get(opts.ExecuterOpts.Options, &httpclientpool.Configuration{
 		Connection: &httpclientpool.ConnectionConfiguration{
 			DisableKeepAlive: httputil.ShouldDisableKeepAlive(opts.ExecuterOpts.Options),
@@ -110,7 +109,6 @@ func New(opts Options) (*Service, error) {
 		target:             opts.Target,
 		wappalyzer:         wappalyzer,
 		templateDirs:       templateDirs, // fix this
-		childExecuter:      childExecuter,
 		httpclient:         httpclient,
 		technologyMappings: mappingData,
 		techTemplates:      techDetectTemplates,
@@ -189,7 +187,7 @@ func (s *Service) executeAutomaticScanOnTarget(input *contextargs.MetaInput) {
 	execOptions.Progress = &testutils.MockProgressClient{} // stats are not supported yet due to centralized logic and cannot be reinitialized
 	eng.SetExecuterOptions(execOptions)
 
-	tmp := eng.ExecuteScanWithOpts(finalTemplates, provider.NewSimpleInputProviderWithUrls(input.Input), true)
+	tmp := eng.ExecuteScanWithOpts(context.Background(), finalTemplates, provider.NewSimpleInputProviderWithUrls(input.Input), true)
 	s.hasResults.Store(tmp.Load())
 }
 
@@ -244,7 +242,9 @@ func (s *Service) getTagsUsingWappalyzer(input *contextargs.MetaInput) []string
 
 // getTagsUsingDetectionTemplates returns tags using detection templates
 func (s *Service) getTagsUsingDetectionTemplates(input *contextargs.MetaInput) ([]string, int) {
-	ctxArgs := contextargs.NewWithInput(input.Input)
+	ctx := context.Background()
+
+	ctxArgs := contextargs.NewWithInput(ctx, input.Input)
 
 	// execute tech detection templates on target
 	tags := map[string]struct{}{}
@@ -256,7 +256,7 @@ func (s *Service) getTagsUsingDetectionTemplates(input *contextargs.MetaInput) (
 		sg.Add()
 		go func(template *templates.Template) {
 			defer sg.Done()
-			ctx := scan.NewScanContext(ctxArgs)
+			ctx := scan.NewScanContext(ctx, ctxArgs)
 			ctx.OnResult = func(event *output.InternalWrappedEvent) {
 				if event == nil {
 					return

pkg/protocols/common/contextargs/contextargs.go

@@ -1,6 +1,7 @@
 package contextargs
 
 import (
+	"context"
 	"net/http/cookiejar"
 	"strings"
 	"sync/atomic"
@@ -19,6 +20,8 @@ var (
 
 // Context implements a shared context struct to share information across multiple templates within a workflow
 type Context struct {
+	ctx context.Context
+
 	// Meta is the target for the executor
 	MetaInput *MetaInput
 
@@ -30,17 +33,18 @@ type Context struct {
 }
 
 // Create a new contextargs instance
-func New() *Context {
-	return NewWithInput("")
+func New(ctx context.Context) *Context {
+	return NewWithInput(ctx, "")
 }
 
 // Create a new contextargs instance with input string
-func NewWithInput(input string) *Context {
+func NewWithInput(ctx context.Context, input string) *Context {
 	jar, err := cookiejar.New(nil)
 	if err != nil {
 		gologger.Error().Msgf("contextargs: could not create cookie jar: %s\n", err)
 	}
 	return &Context{
+		ctx:       ctx,
 		MetaInput: &MetaInput{Input: input},
 		CookieJar: jar,
 		args: &mapsutil.SyncLockMap[string, interface{}]{
@@ -50,6 +54,11 @@ func NewWithInput(input string) *Context {
 	}
 }
 
+// Context returns the context of the current contextargs
+func (ctx *Context) Context() context.Context {
+	return ctx.ctx
+}
+
 // Set the specific key-value pair
 func (ctx *Context) Set(key string, value interface{}) {
 	_ = ctx.args.Set(key, value)
@@ -158,6 +167,7 @@ func (ctx *Context) HasArgs() bool {
 
 func (ctx *Context) Clone() *Context {
 	newCtx := &Context{
+		ctx:       ctx.ctx,
 		MetaInput: ctx.MetaInput.Clone(),
 		args:      ctx.args.Clone(),
 		CookieJar: ctx.CookieJar,

pkg/protocols/common/hosterrorscache/hosterrorscache.go

@@ -124,7 +124,7 @@ func (c *Cache) MarkFailed(value string, err error) {
 	_ = c.failedTargets.Set(finalValue, existingCacheItemValue)
 }
 
-var reCheckError = regexp.MustCompile(`(no address found for host|Client\.Timeout exceeded while awaiting headers|could not resolve host|connection refused|connection reset by peer)`)
+var reCheckError = regexp.MustCompile(`(no address found for host|Client\.Timeout exceeded while awaiting headers|could not resolve host|connection refused|connection reset by peer|i/o timeout|could not connect to any address found for host|timeout awaiting response headers)`)
 
 // checkError checks if an error represents a type that should be
 // added to the host skipping table.

pkg/protocols/common/protocolstate/memguardian.go

@@ -1,6 +1,7 @@
 package protocolstate
 
 import (
+	"context"
 	"sync"
 	"time"
 
@@ -14,20 +15,27 @@ var (
 	MaxThreadsOnLowMemory          = env.GetEnvOrDefault("MEMGUARDIAN_THREADS", 0)
 	MaxBytesBufferAllocOnLowMemory = env.GetEnvOrDefault("MEMGUARDIAN_ALLOC", 0)
 	memTimer                       *time.Ticker
+	cancelFunc                     context.CancelFunc
 )
 
-func StartActiveMemGuardian() {
+func StartActiveMemGuardian(ctx context.Context) {
 	if memguardian.DefaultMemGuardian == nil {
 		return
 	}
 
-	memTimer := time.NewTicker(memguardian.DefaultInterval)
+	memTimer = time.NewTicker(memguardian.DefaultInterval)
+	ctx, cancelFunc = context.WithCancel(ctx)
 	go func() {
-		for range memTimer.C {
-			if IsLowOnMemory() {
-				_ = GlobalGuardBytesBufferAlloc()
-			} else {
-				GlobalRestoreBytesBufferAlloc()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-memTimer.C:
+				if IsLowOnMemory() {
+					_ = GlobalGuardBytesBufferAlloc()
+				} else {
+					GlobalRestoreBytesBufferAlloc()
+				}
 			}
 		}
 	}()
@@ -38,7 +46,10 @@ func StopActiveMemGuardian() {
 		return
 	}
 
-	memTimer.Stop()
+	if memTimer != nil {
+		memTimer.Stop()
+		cancelFunc()
+	}
 }
 
 func IsLowOnMemory() bool {

pkg/protocols/common/protocolstate/state.go

@@ -148,7 +148,7 @@ func Init(options *types.Options) error {
 		return Dialer.Dial(ctx, "tcp", addr)
 	})
 
-	StartActiveMemGuardian()
+	StartActiveMemGuardian(context.Background())
 
 	return nil
 }

pkg/protocols/common/uncover/uncover.go

@@ -11,6 +11,7 @@ import (
 	"github.com/projectdiscovery/uncover"
 	"github.com/projectdiscovery/uncover/sources"
 	mapsutil "github.com/projectdiscovery/utils/maps"
+	sliceutil "github.com/projectdiscovery/utils/slice"
 	stringsutil "github.com/projectdiscovery/utils/strings"
 )
 
@@ -84,9 +85,21 @@ func GetUncoverTargetsFromMetadata(ctx context.Context, templates []*templates.T
 			if queriesMap[engine] == nil {
 				queriesMap[engine] = []string{}
 			}
-			queriesMap[engine] = append(queriesMap[engine], fmt.Sprint(v))
+			switch v := v.(type) {
+			case []interface{}:
+				qs := queriesMap[engine]
+				for _, vv := range v {
+					qs = append(qs, fmt.Sprint(vv))
+				}
+				queriesMap[engine] = qs
+			default:
+				queriesMap[engine] = append(queriesMap[engine], fmt.Sprint(v))
+			}
 		}
 	}
+	for engine, queries := range queriesMap {
+		queriesMap[engine] = sliceutil.Dedupe(queries)
+	}
 	keys := mapsutil.GetKeys(queriesMap)
 	gologger.Info().Msgf("Running uncover queries from template against: %s", strings.Join(keys, ","))
 	result := make(chan string, runtime.NumCPU())
@@ -97,8 +110,11 @@ func GetUncoverTargetsFromMetadata(ctx context.Context, templates []*templates.T
 		// TODO: add support for map[engine]queries in uncover
 		// Note below implementation is intentionally sequential to avoid burning all the API keys
 		counter := 0
-
+	outerLoop:
 		for eng, queries := range queriesMap {
+			if opts.Limit > 0 && counter >= opts.Limit {
+				break
+			}
 			// create new uncover options for each engine
 			uncoverOpts := &uncover.Options{
 				Agents:        []string{eng},
@@ -114,18 +130,20 @@ func GetUncoverTargetsFromMetadata(ctx context.Context, templates []*templates.T
 				gologger.Error().Msgf("Could not get targets using %v engine from uncover: %s", eng, err)
 				return
 			}
+
+		innerLoop:
 			for {
 				select {
 				case <-ctx.Done():
 					return
 				case res, ok := <-ch:
 					if !ok {
-						return
+						continue outerLoop
 					}
 					result <- res
 					counter++
 					if opts.Limit > 0 && counter >= opts.Limit {
-						return
+						break innerLoop
 					}
 				}
 			}

pkg/protocols/dns/request.go

@@ -80,9 +80,17 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
 				break
 			}
 
+			select {
+			case <-input.Context().Done():
+				return input.Context().Err()
+			default:
+			}
+
 			// resize check point - nop if there are no changes
 			if shouldFollowGlobal && swg.Size != request.options.Options.PayloadConcurrency {
-				swg.Resize(request.options.Options.PayloadConcurrency)
+				if err := swg.Resize(input.Context(), request.options.Options.PayloadConcurrency); err != nil {
+					return err
+				}
 			}
 
 			value = generators.MergeMaps(vars, value)

pkg/protocols/dns/request_test.go

@@ -1,6 +1,7 @@
 package dns
 
 import (
+	"context"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -37,7 +38,7 @@ func TestDNSExecuteWithResults(t *testing.T) {
 				Name:  "test",
 				Part:  "raw",
 				Type:  matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
-				Words: []string{"93.184.216.34"},
+				Words: []string{"93.184.215.14"},
 			}},
 			Extractors: []*extractors.Extractor{{
 				Part:  "raw",
@@ -54,7 +55,7 @@ func TestDNSExecuteWithResults(t *testing.T) {
 	t.Run("domain-valid", func(t *testing.T) {
 		metadata := make(output.InternalEvent)
 		previous := make(output.InternalEvent)
-		ctxArgs := contextargs.NewWithInput("example.com")
+		ctxArgs := contextargs.NewWithInput(context.Background(), "example.com")
 		err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
 			finalEvent = event
 		})
@@ -64,13 +65,13 @@ func TestDNSExecuteWithResults(t *testing.T) {
 	require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results")
 	require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results")
 	require.Equal(t, 1, len(finalEvent.Results[0].ExtractedResults), "could not get correct number of extracted results")
-	require.Equal(t, "93.184.216.34", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
+	require.Equal(t, "93.184.215.14", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
 	finalEvent = nil
 
 	t.Run("url-to-domain", func(t *testing.T) {
 		metadata := make(output.InternalEvent)
 		previous := make(output.InternalEvent)
-		err := request.ExecuteWithResults(contextargs.NewWithInput("https://example.com"), metadata, previous, func(event *output.InternalWrappedEvent) {
+		err := request.ExecuteWithResults(contextargs.NewWithInput(context.Background(), "https://example.com"), metadata, previous, func(event *output.InternalWrappedEvent) {
 			finalEvent = event
 		})
 		require.Nil(t, err, "could not execute dns request")
@@ -79,6 +80,6 @@ func TestDNSExecuteWithResults(t *testing.T) {
 	require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results")
 	require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results")
 	require.Equal(t, 1, len(finalEvent.Results[0].ExtractedResults), "could not get correct number of extracted results")
-	require.Equal(t, "93.184.216.34", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
+	require.Equal(t, "93.184.215.14", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
 	finalEvent = nil
 }

pkg/protocols/file/request_test.go

@@ -1,6 +1,7 @@
 package file
 
 import (
+	"context"
 	"os"
 	"path/filepath"
 	"testing"
@@ -67,7 +68,7 @@ func TestFileExecuteWithResults(t *testing.T) {
 	t.Run("valid", func(t *testing.T) {
 		metadata := make(output.InternalEvent)
 		previous := make(output.InternalEvent)
-		ctxArgs := contextargs.NewWithInput(tempDir)
+		ctxArgs := contextargs.NewWithInput(context.Background(), tempDir)
 		err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
 			finalEvent = event
 		})

pkg/protocols/headless/engine/page_actions_test.go

@@ -1,6 +1,7 @@
 package engine
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"math/rand"
@@ -595,7 +596,7 @@ func testHeadless(t *testing.T, actions []*Action, timeout time.Duration, handle
 	ts := httptest.NewServer(http.HandlerFunc(handler))
 	defer ts.Close()
 
-	input := contextargs.NewWithInput(ts.URL)
+	input := contextargs.NewWithInput(context.Background(), ts.URL)
 	input.CookieJar, err = cookiejar.New(nil)
 	require.Nil(t, err)
 
@@ -674,7 +675,7 @@ func TestBlockedHeadlessURLS(t *testing.T) {
 			{ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}},
 		}
 
-		data, page, err := instance.Run(contextargs.NewWithInput(ts.URL), actions, nil, &Options{Timeout: 20 * time.Second, Options: opts}) // allow file access in test
+		data, page, err := instance.Run(contextargs.NewWithInput(context.Background(), ts.URL), actions, nil, &Options{Timeout: 20 * time.Second, Options: opts}) // allow file access in test
 		require.Error(t, err, "expected error for url %s got %v", testcase, data)
 		require.True(t, stringsutil.ContainsAny(err.Error(), "net::ERR_ACCESS_DENIED", "failed to parse url", "Cannot navigate to invalid URL", "net::ERR_ABORTED", "net::ERR_INVALID_URL"), "found different error %v for testcases %v", err, testcase)
 		require.Len(t, data, 0, "expected no data for url %s got %v", testcase, data)

pkg/protocols/headless/request.go

@@ -44,7 +44,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata,
 		if err != nil {
 			return err
 		}
-		input = contextargs.NewWithInput(url)
+		input = contextargs.NewWithInput(input.Context(), url)
 	}
 
 	if request.options.Browser.UserAgent() == "" {

pkg/protocols/http/build_request.go

@@ -277,7 +277,7 @@ func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, data st
 			return nil, fmt.Errorf("malformed request supplied")
 		}
 
-		if err := expressions.ContainsUnresolvedVariables(parts[1]); err != nil {
+		if err := expressions.ContainsUnresolvedVariables(parts[1]); err != nil && !r.request.SkipVariablesCheck {
 			return nil, ErrUnresolvedVars.Msgf(parts[1])
 		}
 
@@ -296,7 +296,7 @@ func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, data st
 		}
 		return r.generateRawRequest(ctx, data, parsed, values, payloads)
 	}
-	if err := expressions.ContainsUnresolvedVariables(data); err != nil {
+	if err := expressions.ContainsUnresolvedVariables(data); err != nil && !r.request.SkipVariablesCheck {
 		// early exit: if there are any unresolved variables in `path` after evaluation
 		// then return early since this will definitely fail
 		return nil, ErrUnresolvedVars.Msgf(data)

pkg/protocols/http/build_request_test.go

@@ -40,7 +40,7 @@ func TestMakeRequestFromModal(t *testing.T) {
 
 	generator := request.newGenerator(false)
 	inputData, payloads, _ := generator.nextValue()
-	req, err := generator.Make(context.Background(), contextargs.NewWithInput("https://example.com"), inputData, payloads, map[string]interface{}{})
+	req, err := generator.Make(context.Background(), contextargs.NewWithInput(context.Background(), "https://example.com"), inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	if req.request.URL == nil {
 		t.Fatalf("url is nil in generator make")
@@ -70,13 +70,13 @@ func TestMakeRequestFromModalTrimSuffixSlash(t *testing.T) {
 
 	generator := request.newGenerator(false)
 	inputData, payloads, _ := generator.nextValue()
-	req, err := generator.Make(context.Background(), contextargs.NewWithInput("https://example.com/test.php"), inputData, payloads, map[string]interface{}{})
+	req, err := generator.Make(context.Background(), contextargs.NewWithInput(context.Background(), "https://example.com/test.php"), inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	require.Equal(t, "https://example.com/test.php?query=example", req.request.URL.String(), "could not get correct request path")
 
 	generator = request.newGenerator(false)
 	inputData, payloads, _ = generator.nextValue()
-	req, err = generator.Make(context.Background(), contextargs.NewWithInput("https://example.com/test/"), inputData, payloads, map[string]interface{}{})
+	req, err = generator.Make(context.Background(), contextargs.NewWithInput(context.Background(), "https://example.com/test/"), inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	require.Equal(t, "https://example.com/test/?query=example", req.request.URL.String(), "could not get correct request path")
 }
@@ -110,13 +110,13 @@ Accept-Encoding: gzip`},
 
 	generator := request.newGenerator(false)
 	inputData, payloads, _ := generator.nextValue()
-	req, err := generator.Make(context.Background(), contextargs.NewWithInput("https://example.com"), inputData, payloads, map[string]interface{}{})
+	req, err := generator.Make(context.Background(), contextargs.NewWithInput(context.Background(), "https://example.com"), inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	authorization := req.request.Header.Get("Authorization")
 	require.Equal(t, "Basic admin:admin", authorization, "could not get correct authorization headers from raw")
 
 	inputData, payloads, _ = generator.nextValue()
-	req, err = generator.Make(context.Background(), contextargs.NewWithInput("https://example.com"), inputData, payloads, map[string]interface{}{})
+	req, err = generator.Make(context.Background(), contextargs.NewWithInput(context.Background(), "https://example.com"), inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	authorization = req.request.Header.Get("Authorization")
 	require.Equal(t, "Basic admin:guest", authorization, "could not get correct authorization headers from raw")
@@ -151,13 +151,13 @@ Accept-Encoding: gzip`},
 
 	generator := request.newGenerator(false)
 	inputData, payloads, _ := generator.nextValue()
-	req, err := generator.Make(context.Background(), contextargs.NewWithInput("https://example.com"), inputData, payloads, map[string]interface{}{})
+	req, err := generator.Make(context.Background(), contextargs.NewWithInput(context.Background(), "https://example.com"), inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	authorization := req.request.Header.Get("Authorization")
 	require.Equal(t, "Basic YWRtaW46YWRtaW4=", authorization, "could not get correct authorization headers from raw")
 
 	inputData, payloads, _ = generator.nextValue()
-	req, err = generator.Make(context.Background(), contextargs.NewWithInput("https://example.com"), inputData, payloads, map[string]interface{}{})
+	req, err = generator.Make(context.Background(), contextargs.NewWithInput(context.Background(), "https://example.com"), inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 	authorization = req.request.Header.Get("Authorization")
 	require.Equal(t, "Basic YWRtaW46Z3Vlc3Q=", authorization, "could not get correct authorization headers from raw")
@@ -195,7 +195,7 @@ func TestMakeRequestFromModelUniqueInteractsh(t *testing.T) {
 	require.Nil(t, err, "could not create interactsh client")
 
 	inputData, payloads, _ := generator.nextValue()
-	got, err := generator.Make(context.Background(), contextargs.NewWithInput("https://example.com"), inputData, payloads, map[string]interface{}{})
+	got, err := generator.Make(context.Background(), contextargs.NewWithInput(context.Background(), "https://example.com"), inputData, payloads, map[string]interface{}{})
 	require.Nil(t, err, "could not make http request")
 
 	// check if all the interactsh markers are replaced with unique urls

pkg/protocols/http/http.go

@@ -9,7 +9,6 @@ import (
 	json "github.com/json-iterator/go"
 	"github.com/pkg/errors"
 
-	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/fuzz"
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators"
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
@@ -19,6 +18,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
 	httputil "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils/http"
+	"github.com/projectdiscovery/nuclei/v3/pkg/utils/stats"
 	"github.com/projectdiscovery/rawhttp"
 	"github.com/projectdiscovery/retryablehttp-go"
 	fileutil "github.com/projectdiscovery/utils/file"
@@ -455,7 +455,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
 			}
 		}
 		if hasNamedInternalExtractor && hasMultipleRequests {
-			gologger.Warning().Label(options.TemplateID).Msgf("Setting thread count to 0 because dynamic extractors are not supported with payloads yet")
+			stats.Increment(SetThreadToCountZero)
 			request.Threads = 0
 		} else {
 			// specifically for http requests high concurrency and and threads will lead to memory exausthion, hence reduce the maximum parallelism
@@ -490,3 +490,11 @@ func (request *Request) Requests() int {
 	}
 	return len(request.Path)
 }
+
+const (
+	SetThreadToCountZero = "set-thread-count-to-zero"
+)
+
+func init() {
+	stats.NewEntry(SetThreadToCountZero, "Setting thread count to 0 for %d templates, dynamic extractors are not supported with payloads yet")
+}

pkg/protocols/http/httpclientpool/clientpool.go

@@ -35,8 +35,18 @@ var (
 	forceMaxRedirects int
 	normalClient      *retryablehttp.Client
 	clientPool        *mapsutil.SyncLockMap[string, *retryablehttp.Client]
+	// ResponseHeaderTimeout is the timeout for response headers
+	// to be read from the server (this prevents infinite hang started by server if any)
+	ResponseHeaderTimeout = time.Duration(5) * time.Second
+	// HttpTimeoutMultiplier is the multiplier for the http timeout
+	HttpTimeoutMultiplier = 3
 )
 
+// GetHttpTimeout returns the http timeout for the client
+func GetHttpTimeout(opts *types.Options) time.Duration {
+	return time.Duration(opts.Timeout*HttpTimeoutMultiplier) * time.Second
+}
+
 // Init initializes the clientpool implementation
 func Init(options *types.Options) error {
 	// Don't create clients if already created in the past.
@@ -139,7 +149,7 @@ func GetRawHTTP(options *types.Options) *rawhttp.Client {
 		} else if Dialer != nil {
 			rawHttpOptions.FastDialer = Dialer
 		}
-		rawHttpOptions.Timeout = time.Duration(options.Timeout) * time.Second
+		rawHttpOptions.Timeout = GetHttpTimeout(options)
 		rawHttpClient = rawhttp.NewClient(rawHttpOptions)
 	}
 	return rawHttpClient
@@ -237,11 +247,12 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
 			}
 			return Dialer.DialTLS(ctx, network, addr)
 		},
-		MaxIdleConns:        maxIdleConns,
-		MaxIdleConnsPerHost: maxIdleConnsPerHost,
-		MaxConnsPerHost:     maxConnsPerHost,
-		TLSClientConfig:     tlsConfig,
-		DisableKeepAlives:   disableKeepAlives,
+		MaxIdleConns:          maxIdleConns,
+		MaxIdleConnsPerHost:   maxIdleConnsPerHost,
+		MaxConnsPerHost:       maxConnsPerHost,
+		TLSClientConfig:       tlsConfig,
+		DisableKeepAlives:     disableKeepAlives,
+		ResponseHeaderTimeout: ResponseHeaderTimeout,
 	}
 
 	if types.ProxyURL != "" {
@@ -288,7 +299,7 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl
 		CheckRedirect: makeCheckRedirectFunc(redirectFlow, maxRedirects),
 	}
 	if !configuration.NoTimeout {
-		httpclient.Timeout = time.Duration(options.Timeout) * time.Second
+		httpclient.Timeout = GetHttpTimeout(options)
 	}
 	client := retryablehttp.NewWithHTTPClient(httpclient, retryableHttpOptions)
 	if jar != nil {

pkg/protocols/http/httputils/spm.go

@@ -5,6 +5,7 @@ import (
 	"sync"
 
 	syncutil "github.com/projectdiscovery/utils/sync"
+	"golang.org/x/exp/maps"
 )
 
 // WorkPoolType is the type of work pool to use
@@ -19,7 +20,7 @@ const (
 
 // StopAtFirstMatchHandler is a handler that executes
 // request and stops on first match
-type StopAtFirstMatchHandler[T any] struct {
+type StopAtFirstMatchHandler[T comparable] struct {
 	once sync.Once
 	// Result Channel
 	ResultChan chan T
@@ -33,12 +34,14 @@ type StopAtFirstMatchHandler[T any] struct {
 	ctx         context.Context
 	cancel      context.CancelFunc
 	internalWg  *sync.WaitGroup
-	results     []T
+	results     map[T]struct{}
+	onResult    func(T)
 	stopEnabled bool
+	maxResults  int
 }
 
 // NewBlockingSPMHandler creates a new stop at first match handler
-func NewBlockingSPMHandler[T any](ctx context.Context, size int, spm bool) *StopAtFirstMatchHandler[T] {
+func NewBlockingSPMHandler[T comparable](ctx context.Context, size int, maxResults int, spm bool) *StopAtFirstMatchHandler[T] {
 	ctx1, cancel := context.WithCancel(ctx)
 
 	awg, _ := syncutil.New(syncutil.WithSize(size))
@@ -51,6 +54,8 @@ func NewBlockingSPMHandler[T any](ctx context.Context, size int, spm bool) *Stop
 		ctx:         ctx1,
 		cancel:      cancel,
 		stopEnabled: spm,
+		results:     make(map[T]struct{}),
+		maxResults:  maxResults,
 	}
 	s.internalWg.Add(1)
 	go s.run(ctx)
@@ -58,7 +63,7 @@ func NewBlockingSPMHandler[T any](ctx context.Context, size int, spm bool) *Stop
 }
 
 // NewNonBlockingSPMHandler creates a new stop at first match handler
-func NewNonBlockingSPMHandler[T any](ctx context.Context, spm bool) *StopAtFirstMatchHandler[T] {
+func NewNonBlockingSPMHandler[T comparable](ctx context.Context, maxResults int, spm bool) *StopAtFirstMatchHandler[T] {
 	ctx1, cancel := context.WithCancel(ctx)
 	s := &StopAtFirstMatchHandler[T]{
 		ResultChan:  make(chan T, 1),
@@ -68,6 +73,8 @@ func NewNonBlockingSPMHandler[T any](ctx context.Context, spm bool) *StopAtFirst
 		ctx:         ctx1,
 		cancel:      cancel,
 		stopEnabled: spm,
+		results:     make(map[T]struct{}),
+		maxResults:  maxResults,
 	}
 	s.internalWg.Add(1)
 	go s.run(ctx)
@@ -82,6 +89,25 @@ func (h *StopAtFirstMatchHandler[T]) Trigger() {
 	}
 }
 
+// Cancel cancels spm context
+func (h *StopAtFirstMatchHandler[T]) Cancel() {
+	h.cancel()
+}
+
+// SetOnResult callback
+// this is not thread safe
+func (h *StopAtFirstMatchHandler[T]) SetOnResultCallback(fn func(T)) {
+	if h.onResult != nil {
+		tmp := h.onResult
+		h.onResult = func(t T) {
+			tmp(t)
+			fn(t)
+		}
+	} else {
+		h.onResult = fn
+	}
+}
+
 // MatchCallback is called when a match is found
 // input fn should be the callback that is intended to be called
 // if stop at first is enabled and other conditions are met
@@ -104,7 +130,14 @@ func (h *StopAtFirstMatchHandler[T]) run(ctx context.Context) {
 			if !ok {
 				return
 			}
-			h.results = append(h.results, val)
+			if h.onResult != nil {
+				h.onResult(val)
+			}
+			if len(h.results) >= h.maxResults {
+				// skip or do not store the result
+				continue
+			}
+			h.results[val] = struct{}{}
 		}
 	}
 }
@@ -114,6 +147,11 @@ func (h *StopAtFirstMatchHandler[T]) Done() <-chan struct{} {
 	return h.ctx.Done()
 }
 
+// Cancelled returns true if the context is cancelled
+func (h *StopAtFirstMatchHandler[T]) Cancelled() bool {
+	return h.ctx.Err() != nil
+}
+
 // FoundFirstMatch returns true if first match was found
 // in stop at first match mode
 func (h *StopAtFirstMatchHandler[T]) FoundFirstMatch() bool {
@@ -143,10 +181,11 @@ func (h *StopAtFirstMatchHandler[T]) Release() {
 	}
 }
 
-func (h *StopAtFirstMatchHandler[T]) Resize(size int) {
+func (h *StopAtFirstMatchHandler[T]) Resize(ctx context.Context, size int) error {
 	if h.sgPool.Size != size {
-		h.sgPool.Resize(size)
+		return h.sgPool.Resize(ctx, size)
 	}
+	return nil
 }
 
 func (h *StopAtFirstMatchHandler[T]) Size() int {
@@ -168,5 +207,5 @@ func (h *StopAtFirstMatchHandler[T]) Wait() {
 
 // CombinedResults returns the combined results
 func (h *StopAtFirstMatchHandler[T]) CombinedResults() []T {
-	return h.results
+	return maps.Keys(h.results)
 }

pkg/protocols/http/request.go

@@ -47,10 +47,13 @@ import (
 
 const (
 	defaultMaxWorkers = 150
+	// max unique errors to store & combine
+	// when executing requests in parallel
+	maxErrorsWhenParallel = 3
 )
 
 var (
-	MaxBodyRead = int64(1 << 22) // 4MB using shift operator
+	MaxBodyRead = int64(10 * 1024 * 1024) // 10MB
 	// ErrMissingVars is error occured when variables are missing
 	ErrMissingVars = errors.New("stop execution due to unresolved variables")
 )
@@ -111,7 +114,7 @@ func (request *Request) executeRaceRequest(input *contextargs.Context, previous
 	}
 
 	shouldStop := (request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch || request.options.StopAtFirstMatch)
-	spmHandler := httputils.NewNonBlockingSPMHandler[error](ctx, shouldStop)
+	spmHandler := httputils.NewNonBlockingSPMHandler[error](ctx, maxErrorsWhenParallel, shouldStop)
 	gotMatches := &atomic.Bool{}
 	// wrappedCallback is a callback that wraps the original callback
 	// to implement stop at first match logic
@@ -132,12 +135,29 @@ func (request *Request) executeRaceRequest(input *contextargs.Context, previous
 		}
 	}
 
+	// look for unresponsive hosts and cancel inflight requests as well
+	spmHandler.SetOnResultCallback(func(err error) {
+		if err == nil {
+			return
+		}
+		// marks thsi host as unresponsive if applicable
+		request.markUnresponsiveHost(input, err)
+		if request.isUnresponsiveHost(input) {
+			// stop all inflight requests
+			spmHandler.Cancel()
+		}
+	})
+
 	for i := 0; i < request.RaceNumberRequests; i++ {
+		if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) {
+			// stop sending more requests condition is met
+			break
+		}
 		spmHandler.Acquire()
 		// execute http request
 		go func(httpRequest *generatedRequest) {
 			defer spmHandler.Release()
-			if spmHandler.FoundFirstMatch() {
+			if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) {
 				// stop sending more requests condition is met
 				return
 			}
@@ -175,7 +195,7 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 	// Stop-at-first-match logic while executing requests
 	// parallely using threads
 	shouldStop := (request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch || request.options.StopAtFirstMatch)
-	spmHandler := httputils.NewBlockingSPMHandler[error](context.Background(), maxWorkers, shouldStop)
+	spmHandler := httputils.NewBlockingSPMHandler[error](context.Background(), maxWorkers, maxErrorsWhenParallel, shouldStop)
 	// wrappedCallback is a callback that wraps the original callback
 	// to implement stop at first match logic
 	wrappedCallback := func(event *output.InternalWrappedEvent) {
@@ -194,6 +214,19 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 		}
 	}
 
+	// look for unresponsive hosts and cancel inflight requests as well
+	spmHandler.SetOnResultCallback(func(err error) {
+		if err == nil {
+			return
+		}
+		// marks thsi host as unresponsive if applicable
+		request.markUnresponsiveHost(input, err)
+		if request.isUnresponsiveHost(input) {
+			// stop all inflight requests
+			spmHandler.Cancel()
+		}
+	})
+
 	// iterate payloads and make requests
 	generator := request.newGenerator(false)
 	for {
@@ -202,9 +235,22 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 			break
 		}
 
+		select {
+		case <-input.Context().Done():
+			return input.Context().Err()
+		default:
+		}
+
 		// resize check point - nop if there are no changes
 		if shouldFollowGlobal && spmHandler.Size() != request.options.Options.PayloadConcurrency {
-			spmHandler.Resize(request.options.Options.PayloadConcurrency)
+			if err := spmHandler.Resize(input.Context(), request.options.Options.PayloadConcurrency); err != nil {
+				return err
+			}
+		}
+
+		// break if stop at first match is found or host is unresponsive
+		if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) {
+			break
 		}
 
 		ctx := request.newContext(input)
@@ -222,19 +268,21 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 		spmHandler.Acquire()
 		go func(httpRequest *generatedRequest) {
 			defer spmHandler.Release()
-			if spmHandler.FoundFirstMatch() {
+			if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) || spmHandler.Cancelled() {
+				return
+			}
+			// putting ratelimiter here prevents any unnecessary waiting if any
+			request.options.RateLimitTake()
+
+			// after ratelimit take, check if we need to stop
+			if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) || spmHandler.Cancelled() {
 				return
 			}
 
 			select {
 			case <-spmHandler.Done():
 				return
-			case spmHandler.ResultChan <- func() error {
-				// putting ratelimiter here prevents any unnecessary waiting if any
-				request.options.RateLimitTake()
-				previous := make(map[string]interface{})
-				return request.executeRequest(input, httpRequest, previous, false, wrappedCallback, 0)
-			}():
+			case spmHandler.ResultChan <- request.executeRequest(input, httpRequest, make(map[string]interface{}), false, wrappedCallback, 0):
 				return
 			}
 		}(generatedHttpRequest)
@@ -276,12 +324,10 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
 		maxWorkers = pipeOptions.MaxPendingRequests
 	}
 
-	// Stop-at-first-match logic while executing requests
-	// parallely using threads
 	// Stop-at-first-match logic while executing requests
 	// parallely using threads
 	shouldStop := (request.options.Options.StopAtFirstMatch || request.StopAtFirstMatch || request.options.StopAtFirstMatch)
-	spmHandler := httputils.NewBlockingSPMHandler[error](context.Background(), maxWorkers, shouldStop)
+	spmHandler := httputils.NewBlockingSPMHandler[error](context.Background(), maxWorkers, maxErrorsWhenParallel, shouldStop)
 	// wrappedCallback is a callback that wraps the original callback
 	// to implement stop at first match logic
 	wrappedCallback := func(event *output.InternalWrappedEvent) {
@@ -300,11 +346,36 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
 		}
 	}
 
+	// look for unresponsive hosts and cancel inflight requests as well
+	spmHandler.SetOnResultCallback(func(err error) {
+		if err == nil {
+			return
+		}
+		// marks thsi host as unresponsive if applicable
+		request.markUnresponsiveHost(input, err)
+		if request.isUnresponsiveHost(input) {
+			// stop all inflight requests
+			spmHandler.Cancel()
+		}
+	})
+
 	for {
 		inputData, payloads, ok := generator.nextValue()
 		if !ok {
 			break
 		}
+
+		select {
+		case <-input.Context().Done():
+			return input.Context().Err()
+		default:
+		}
+
+		if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) || spmHandler.Cancelled() {
+			// skip if first match is found
+			break
+		}
+
 		ctx := request.newContext(input)
 		generatedHttpRequest, err := generator.Make(ctx, input, inputData, payloads, dynamicValues)
 		if err != nil {
@@ -318,11 +389,10 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
 		spmHandler.Acquire()
 		go func(httpRequest *generatedRequest) {
 			defer spmHandler.Release()
-			if spmHandler.FoundFirstMatch() {
+			if spmHandler.FoundFirstMatch() || request.isUnresponsiveHost(input) {
 				// skip if first match is found
 				return
 			}
-
 			select {
 			case <-spmHandler.Done():
 				return
@@ -378,8 +448,9 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 			request.options.RateLimitTake()
 
 			ctx := request.newContext(input)
-			ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Duration(request.options.Options.Timeout)*time.Second)
+			ctxWithTimeout, cancel := context.WithTimeout(ctx, httpclientpool.GetHttpTimeout(request.options.Options))
 			defer cancel()
+
 			generatedHttpRequest, err := generator.Make(ctxWithTimeout, input, data, payloads, dynamicValue)
 			if err != nil {
 				if err == types.ErrNoMoreRequests {
@@ -398,7 +469,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 				input.MetaInput.Input = generatedHttpRequest.URL()
 			}
 			// Check if hosts keep erroring
-			if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(input.MetaInput.ID()) {
+			if request.isUnresponsiveHost(input) {
 				return true, nil
 			}
 			var gotMatches bool
@@ -437,12 +508,13 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 				return true, nil
 			}
 			if execReqErr != nil {
-				if request.options.HostErrorsCache != nil {
-					request.options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err)
-				}
+				// if applicable mark the host as unresponsive
+				request.markUnresponsiveHost(input, execReqErr)
 				requestErr = errorutil.NewWithErr(execReqErr).Msgf("got err while executing %v", generatedHttpRequest.URL())
+				request.options.Progress.IncrementFailedRequestsBy(1)
+			} else {
+				request.options.Progress.IncrementRequests()
 			}
-			request.options.Progress.IncrementRequests()
 
 			// If this was a match, and we want to stop at first match, skip all further requests.
 			shouldStopAtFirstMatch := generatedHttpRequest.original.options.Options.StopAtFirstMatch || generatedHttpRequest.original.options.StopAtFirstMatch || request.StopAtFirstMatch
@@ -456,6 +528,13 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 		if !ok {
 			break
 		}
+
+		select {
+		case <-input.Context().Done():
+			return input.Context().Err()
+		default:
+		}
+
 		var gotErr error
 		var skip bool
 		if len(gotDynamicValues) > 0 {
@@ -482,6 +561,10 @@ const drainReqSize = int64(8 * 1024)
 
 // executeRequest executes the actual generated request and returns error if occurred
 func (request *Request) executeRequest(input *contextargs.Context, generatedRequest *generatedRequest, previousEvent output.InternalEvent, hasInteractMatchers bool, processEvent protocols.OutputEventCallback, requestCount int) (err error) {
+	// Check if hosts keep erroring
+	if request.isUnresponsiveHost(input) {
+		return fmt.Errorf("hostErrorsCache : host %s is unresponsive", input.MetaInput.Input)
+	}
 
 	// wrap one more callback for validation and fixing event
 	callback := func(event *output.InternalWrappedEvent) {
@@ -981,7 +1064,25 @@ func (request *Request) pruneSignatureInternalValues(maps ...map[string]interfac
 
 func (request *Request) newContext(input *contextargs.Context) context.Context {
 	if input.MetaInput.CustomIP != "" {
-		return context.WithValue(context.Background(), fastdialer.IP, input.MetaInput.CustomIP)
+		return context.WithValue(input.Context(), fastdialer.IP, input.MetaInput.CustomIP)
 	}
-	return context.Background()
+	return input.Context()
+}
+
+// markUnresponsiveHost checks if the error is a unreponsive host error and marks it
+func (request *Request) markUnresponsiveHost(input *contextargs.Context, err error) {
+	if err == nil {
+		return
+	}
+	if request.options.HostErrorsCache != nil {
+		request.options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err)
+	}
+}
+
+// isUnresponsiveHost checks if the error is a unreponsive based on its execution history
+func (request *Request) isUnresponsiveHost(input *contextargs.Context) bool {
+	if request.options.HostErrorsCache != nil {
+		return request.options.HostErrorsCache.Check(input.MetaInput.ID())
+	}
+	return false
 }

pkg/protocols/http/request_annotations.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/projectdiscovery/fastdialer/fastdialer"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
 	"github.com/projectdiscovery/retryablehttp-go"
 	iputil "github.com/projectdiscovery/utils/ip"
 	stringsutil "github.com/projectdiscovery/utils/strings"
@@ -124,7 +125,7 @@ func (r *Request) parseAnnotations(rawRequest string, request *retryablehttp.Req
 			}
 		} else {
 			//nolint:govet // cancelled automatically by withTimeout
-			ctx, overrides.cancelFunc = context.WithTimeout(context.Background(), time.Duration(r.options.Options.Timeout)*time.Second)
+			ctx, overrides.cancelFunc = context.WithTimeout(context.Background(), httpclientpool.GetHttpTimeout(r.options.Options))
 			request = request.Clone(ctx)
 		}
 	}

pkg/protocols/http/request_fuzz.go

@@ -24,6 +24,7 @@ import (
 	protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	"github.com/projectdiscovery/retryablehttp-go"
+	"github.com/projectdiscovery/useragent"
 	urlutil "github.com/projectdiscovery/utils/url"
 )
 
@@ -89,6 +90,9 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous
 	if err != nil {
 		return errors.Wrap(err, "fuzz: could not build request from url")
 	}
+	userAgent := useragent.PickRandom()
+	baseRequest.Header.Set("User-Agent", userAgent.Raw)
+
 	// execute with one value first to checks its applicability
 	err = request.executeAllFuzzingRules(inputx, previous, baseRequest, callback)
 	if err != nil {
@@ -110,9 +114,21 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous
 func (request *Request) executeAllFuzzingRules(input *contextargs.Context, values map[string]interface{}, baseRequest *retryablehttp.Request, callback protocols.OutputEventCallback) error {
 	applicable := false
 	for _, rule := range request.Fuzzing {
+		select {
+		case <-input.Context().Done():
+			return input.Context().Err()
+		default:
+		}
+
 		err := rule.Execute(&fuzz.ExecuteRuleInput{
 			Input: input,
 			Callback: func(gr fuzz.GeneratedRequest) bool {
+				select {
+				case <-input.Context().Done():
+					return false
+				default:
+				}
+
 				// TODO: replace this after scanContext Refactor
 				return request.executeGeneratedFuzzingRequest(gr, input, callback)
 			},
@@ -154,6 +170,13 @@ func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest,
 	}
 	var gotMatches bool
 	requestErr := request.executeRequest(input, req, gr.DynamicValues, hasInteractMatchers, func(event *output.InternalWrappedEvent) {
+		for _, result := range event.Results {
+			result.IsFuzzingResult = true
+			result.FuzzingMethod = gr.Request.Method
+			result.FuzzingParameter = gr.Parameter
+			result.FuzzingPosition = gr.Component.Name()
+		}
+
 		if hasInteractMarkers && hasInteractMatchers && request.options.Interactsh != nil {
 			requestData := &interactsh.RequestData{
 				MakeResultFunc: request.MakeResultEvent,

pkg/protocols/http/request_test.go

@@ -1,6 +1,7 @@
 package http
 
 import (
+	"context"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
@@ -83,7 +84,7 @@ Disallow: /c`))
 	t.Run("test", func(t *testing.T) {
 		metadata := make(output.InternalEvent)
 		previous := make(output.InternalEvent)
-		ctxArgs := contextargs.NewWithInput(ts.URL)
+		ctxArgs := contextargs.NewWithInput(context.Background(), ts.URL)
 		err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
 			if event.OperatorsResult != nil && event.OperatorsResult.Matched {
 				matchCount++
@@ -159,7 +160,7 @@ func TestDisableTE(t *testing.T) {
 	t.Run("test", func(t *testing.T) {
 		metadata := make(output.InternalEvent)
 		previous := make(output.InternalEvent)
-		ctxArgs := contextargs.NewWithInput(ts.URL)
+		ctxArgs := contextargs.NewWithInput(context.Background(), ts.URL)
 		err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
 			if event.OperatorsResult != nil && event.OperatorsResult.Matched {
 				matchCount++
@@ -172,7 +173,7 @@ func TestDisableTE(t *testing.T) {
 	t.Run("test2", func(t *testing.T) {
 		metadata := make(output.InternalEvent)
 		previous := make(output.InternalEvent)
-		ctxArgs := contextargs.NewWithInput(ts.URL)
+		ctxArgs := contextargs.NewWithInput(context.Background(), ts.URL)
 		err := request2.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
 			if event.OperatorsResult != nil && event.OperatorsResult.Matched {
 				matchCount++
@@ -242,7 +243,7 @@ func TestReqURLPattern(t *testing.T) {
 	t.Run("test", func(t *testing.T) {
 		metadata := make(output.InternalEvent)
 		previous := make(output.InternalEvent)
-		ctxArgs := contextargs.NewWithInput(ts.URL)
+		ctxArgs := contextargs.NewWithInput(context.Background(), ts.URL)
 		err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
 			if event.OperatorsResult != nil && event.OperatorsResult.Matched {
 				matchCount++

pkg/protocols/javascript/js.go

@@ -154,6 +154,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
 		opts := &compiler.ExecuteOptions{
 			Timeout: request.Timeout,
 			Source:  &request.Init,
+			Context: context.Background(),
 		}
 		// register 'export' function to export variables from init code
 		// these are saved in args and are available in pre-condition and request code
@@ -343,7 +344,7 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, dynamicV
 		argsCopy.TemplateCtx = templateCtx.GetAll()
 
 		result, err := request.options.JsCompiler.ExecuteWithOptions(request.preConditionCompiled, argsCopy,
-			&compiler.ExecuteOptions{Timeout: request.Timeout, Source: &request.PreCondition})
+			&compiler.ExecuteOptions{Timeout: request.Timeout, Source: &request.PreCondition, Context: target.Context()})
 		if err != nil {
 			return errorutil.NewWithTag(request.TemplateID, "could not execute pre-condition: %s", err)
 		}
@@ -373,6 +374,12 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, dynamicV
 				return nil
 			}
 
+			select {
+			case <-input.Context().Done():
+				return input.Context().Err()
+			default:
+			}
+
 			if err := request.executeRequestWithPayloads(hostPort, input, hostname, value, payloadValues, func(result *output.InternalWrappedEvent) {
 				if result.OperatorsResult != nil && result.OperatorsResult.Matched {
 					gotMatches = true
@@ -419,9 +426,17 @@ func (request *Request) executeRequestParallel(ctxParent context.Context, hostPo
 				break
 			}
 
+			select {
+			case <-input.Context().Done():
+				return
+			default:
+			}
+
 			// resize check point - nop if there are no changes
 			if shouldFollowGlobal && sg.Size != request.options.Options.PayloadConcurrency {
-				sg.Resize(request.options.Options.PayloadConcurrency)
+				if err := sg.Resize(ctxParent, request.options.Options.PayloadConcurrency); err != nil {
+					gologger.Warning().Msgf("Could not resize workpool: %s\n", err)
+				}
 			}
 
 			sg.Add()
@@ -486,7 +501,7 @@ func (request *Request) executeRequestWithPayloads(hostPort string, input *conte
 	}
 
 	results, err := request.options.JsCompiler.ExecuteWithOptions(request.scriptCompiled, argsCopy,
-		&compiler.ExecuteOptions{Timeout: request.Timeout, Source: &request.Code})
+		&compiler.ExecuteOptions{Timeout: request.Timeout, Source: &request.Code, Context: input.Context()})
 	if err != nil {
 		// shouldn't fail even if it returned error instead create a failure event
 		results = compiler.ExecuteResult{"success": false, "error": err.Error()}

pkg/protocols/network/request.go

@@ -36,12 +36,6 @@ import (
 	syncutil "github.com/projectdiscovery/utils/sync"
 )
 
-var (
-	// TODO: make this configurable
-	// DefaultReadTimeout is the default read timeout for network requests
-	DefaultReadTimeout = time.Duration(5) * time.Second
-)
-
 var _ protocols.Request = &Request{}
 
 // Type returns the type of the protocol request
@@ -145,6 +139,12 @@ func (request *Request) executeOnTarget(input *contextargs.Context, visited maps
 	variables = generators.MergeMaps(variablesMap, variables, request.options.Constants)
 
 	for _, kv := range request.addresses {
+		select {
+		case <-input.Context().Done():
+			return input.Context().Err()
+		default:
+		}
+
 		actualAddress := replacer.Replace(kv.address, variables)
 
 		if visited.Has(actualAddress) && !request.options.Options.DisableClustering {
@@ -192,9 +192,19 @@ func (request *Request) executeAddress(variables map[string]interface{}, actualA
 				break
 			}
 
+			select {
+			case <-input.Context().Done():
+				return input.Context().Err()
+			default:
+			}
+
 			// resize check point - nop if there are no changes
 			if shouldFollowGlobal && swg.Size != request.options.Options.PayloadConcurrency {
-				swg.Resize(request.options.Options.PayloadConcurrency)
+				if err := swg.Resize(input.Context(), request.options.Options.PayloadConcurrency); err != nil {
+					m.Lock()
+					multiErr = multierr.Append(multiErr, err)
+					m.Unlock()
+				}
 			}
 
 			value = generators.MergeMaps(value, payloads)
@@ -295,7 +305,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
 		}
 
 		if input.Read > 0 {
-			buffer, err := ConnReadNWithTimeout(conn, int64(input.Read), DefaultReadTimeout)
+			buffer, err := ConnReadNWithTimeout(conn, int64(input.Read), request.options.Options.ResponseReadTimeout)
 			if err != nil {
 				return errorutil.NewWithErr(err).Msgf("could not read response from connection")
 			}
@@ -345,7 +355,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
 		bufferSize = -1
 	}
 
-	final, err := ConnReadNWithTimeout(conn, int64(bufferSize), DefaultReadTimeout)
+	final, err := ConnReadNWithTimeout(conn, int64(bufferSize), request.options.Options.ResponseReadTimeout)
 	if err != nil {
 		request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err)
 		gologger.Verbose().Msgf("could not read more data from %s: %s", actualAddress, err)
@@ -446,9 +456,6 @@ func getAddress(toTest string) (string, error) {
 }
 
 func ConnReadNWithTimeout(conn net.Conn, n int64, timeout time.Duration) ([]byte, error) {
-	if timeout == 0 {
-		timeout = DefaultReadTimeout
-	}
 	if n == -1 {
 		// if n is -1 then read all available data from connection
 		return reader.ConnReadNWithTimeout(conn, -1, timeout)

pkg/protocols/network/request_test.go

@@ -1,6 +1,7 @@
 package network
 
 import (
+	"context"
 	"encoding/hex"
 	"fmt"
 	"net/http"
@@ -65,7 +66,7 @@ func TestNetworkExecuteWithResults(t *testing.T) {
 	t.Run("domain-valid", func(t *testing.T) {
 		metadata := make(output.InternalEvent)
 		previous := make(output.InternalEvent)
-		ctxArgs := contextargs.NewWithInput(parsed.Host)
+		ctxArgs := contextargs.NewWithInput(context.Background(), parsed.Host)
 		err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
 			finalEvent = event
 		})
@@ -81,7 +82,7 @@ func TestNetworkExecuteWithResults(t *testing.T) {
 	t.Run("invalid-port-override", func(t *testing.T) {
 		metadata := make(output.InternalEvent)
 		previous := make(output.InternalEvent)
-		ctxArgs := contextargs.NewWithInput("127.0.0.1:11211")
+		ctxArgs := contextargs.NewWithInput(context.Background(), "127.0.0.1:11211")
 		err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
 			finalEvent = event
 		})
@@ -95,7 +96,7 @@ func TestNetworkExecuteWithResults(t *testing.T) {
 	t.Run("hex-to-string", func(t *testing.T) {
 		metadata := make(output.InternalEvent)
 		previous := make(output.InternalEvent)
-		ctxArgs := contextargs.NewWithInput(parsed.Host)
+		ctxArgs := contextargs.NewWithInput(context.Background(), parsed.Host)
 		err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) {
 			finalEvent = event
 		})

pkg/protocols/protocols.go

@@ -1,6 +1,7 @@
 package protocols
 
 import (
+	"context"
 	"encoding/base64"
 	"sync/atomic"
 
@@ -173,7 +174,7 @@ func (e *ExecutorOptions) GetTemplateCtx(input *contextargs.MetaInput) *contexta
 	templateCtx, ok := e.templateCtxStore.Get(scanId)
 	if !ok {
 		// if template context does not exist create new and add it to store and return it
-		templateCtx = contextargs.New()
+		templateCtx = contextargs.New(context.Background())
 		templateCtx.MetaInput = input
 		_ = e.templateCtxStore.Set(scanId, templateCtx)
 	}

pkg/protocols/ssl/ssl_test.go

@@ -1,6 +1,7 @@
 package ssl
 
 import (
+	"context"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -28,7 +29,7 @@ func TestSSLProtocol(t *testing.T) {
 	require.Nil(t, err, "could not compile ssl request")
 
 	var gotEvent output.InternalEvent
-	ctxArgs := contextargs.NewWithInput("scanme.sh:443")
+	ctxArgs := contextargs.NewWithInput(context.Background(), "scanme.sh:443")
 	err = request.ExecuteWithResults(ctxArgs, nil, nil, func(event *output.InternalWrappedEvent) {
 		gotEvent = event.InternalEvent
 	})

pkg/protocols/utils/variables.go

@@ -24,6 +24,7 @@ func init() {
 		Host:     "Host",
 		Port:     "Port",
 		Path:     "Path",
+		Query:    "Query",
 		File:     "File",
 		Scheme:   "Scheme",
 		Input:    "Input",
@@ -44,6 +45,7 @@ const (
 	Host
 	Port
 	Path
+	Query
 	File
 	Scheme
 	Input
@@ -162,6 +164,12 @@ func generateVariables(inputURL *urlutil.URL, removeTrailingSlash bool) map[stri
 			knownVariables[v] = port
 		case Path:
 			knownVariables[v] = requestPath
+		case Query:
+			if queryParams := urlutil.GetParams(parsed.URL.Query()); len(queryParams) > 0 {
+				knownVariables[v] = "?" + queryParams.Encode()
+			} else {
+				knownVariables[v] = ""
+			}
 		case File:
 			knownVariables[v] = base
 		case Scheme:

pkg/protocols/utils/variables_test.go

@@ -1,6 +1,7 @@
 package utils
 
 import (
+	"context"
 	"reflect"
 	"testing"
 
@@ -49,7 +50,7 @@ func TestHTTPVariables(t *testing.T) {
 	require.Equal(t, values["Hostname"], "foobar.com", "incorrect hostname")
 
 	baseURL = "http://scanme.sh"
-	ctxArgs := contextargs.NewWithInput(baseURL)
+	ctxArgs := contextargs.NewWithInput(context.Background(), baseURL)
 	ctxArgs.MetaInput.CustomIP = "1.2.3.4"
 	values = GenerateVariablesWithContextArgs(ctxArgs, true)
 

pkg/reporting/reporting.go

@@ -276,6 +276,7 @@ func (c *ReportingClient) CreateIssue(event *output.ResultEvent) error {
 			if tracker.ShouldFilter(event) {
 				continue
 			}
+
 			trackerName := tracker.Name()
 			stats, statsOk := c.stats[trackerName]
 

pkg/reporting/trackers/gitea/gitea.go

@@ -140,7 +140,7 @@ func (i *Integration) CloseIssue(event *output.ResultEvent) error {
 // ShouldFilter determines if an issue should be logged to this tracker
 func (i *Integration) ShouldFilter(event *output.ResultEvent) bool {
 	if i.options.AllowList != nil && i.options.AllowList.GetMatch(event) {
-		return true
+		return false
 	}
 
 	if i.options.DenyList != nil && i.options.DenyList.GetMatch(event) {

pkg/reporting/trackers/github/github.go

@@ -176,7 +176,7 @@ func (i *Integration) Name() string {
 // ShouldFilter determines if an issue should be logged to this tracker
 func (i *Integration) ShouldFilter(event *output.ResultEvent) bool {
 	if i.options.AllowList != nil && i.options.AllowList.GetMatch(event) {
-		return true
+		return false
 	}
 
 	if i.options.DenyList != nil && i.options.DenyList.GetMatch(event) {

pkg/reporting/trackers/gitlab/gitlab.go

@@ -165,7 +165,7 @@ func (i *Integration) CloseIssue(event *output.ResultEvent) error {
 // ShouldFilter determines if an issue should be logged to this tracker
 func (i *Integration) ShouldFilter(event *output.ResultEvent) bool {
 	if i.options.AllowList != nil && i.options.AllowList.GetMatch(event) {
-		return true
+		return false
 	}
 
 	if i.options.DenyList != nil && i.options.DenyList.GetMatch(event) {

pkg/reporting/trackers/jira/jira.go

@@ -315,7 +315,7 @@ func (i *Integration) FindExistingIssue(event *output.ResultEvent) (jira.Issue,
 // ShouldFilter determines if an issue should be logged to this tracker
 func (i *Integration) ShouldFilter(event *output.ResultEvent) bool {
 	if i.options.AllowList != nil && i.options.AllowList.GetMatch(event) {
-		return true
+		return false
 	}
 
 	if i.options.DenyList != nil && i.options.DenyList.GetMatch(event) {

pkg/scan/charts/echarts.go

@@ -38,16 +38,18 @@ func (s *ScanEventsCharts) allCharts(c echo.Context) *components.Page {
 	page := components.NewPage()
 	page.PageTitle = "Nuclei Charts"
 	line1 := s.totalRequestsOverTime(c)
-	line1.SetSpacerHeight(SpacerHeight)
+	// line1.SetSpacerHeight(SpacerHeight)
 	kline := s.topSlowTemplates(c)
-	kline.SetSpacerHeight(SpacerHeight)
+	// kline.SetSpacerHeight(SpacerHeight)
 	line2 := s.requestsVSInterval(c)
-	line2.SetSpacerHeight(SpacerHeight)
+	// line2.SetSpacerHeight(SpacerHeight)
 	line3 := s.concurrencyVsTime(c)
-	line3.SetSpacerHeight(SpacerHeight)
+	// line3.SetSpacerHeight(SpacerHeight)
 	page.AddCharts(line1, kline, line2, line3)
-	page.Validate()
 	page.SetLayout(components.PageCenterLayout)
+	page.Theme = "dark"
+	page.Validate()
+
 	return page
 }
 
@@ -59,7 +61,12 @@ func (s *ScanEventsCharts) TotalRequestsOverTime(c echo.Context) error {
 // totalRequestsOverTime generates a line chart showing total requests count over time
 func (s *ScanEventsCharts) totalRequestsOverTime(c echo.Context) *charts.Line {
 	line := charts.NewLine()
-	line.SetCaption("Chart Shows Total Requests Count Over Time (for each/all Protocols)")
+	line.SetGlobalOptions(
+		charts.WithTitleOpts(opts.Title{
+			Title:    "Nuclei: Total Requests vs Time",
+			Subtitle: "Chart Shows Total Requests Count Over Time (for each/all Protocols)",
+		}),
+	)
 
 	var startTime time.Time = time.Now()
 	var endTime time.Time
@@ -120,8 +127,12 @@ func (s *ScanEventsCharts) TopSlowTemplates(c echo.Context) error {
 // topSlowTemplates generates a Kline chart showing the top slow templates by time taken
 func (s *ScanEventsCharts) topSlowTemplates(c echo.Context) *charts.Kline {
 	kline := charts.NewKLine()
-	kline.SetCaption(fmt.Sprintf("Chart Shows Top Slow Templates (by time taken) (Top %v)", TopK))
-
+	kline.SetGlobalOptions(
+		charts.WithTitleOpts(opts.Title{
+			Title:    "Nuclei: Top Slow Templates",
+			Subtitle: fmt.Sprintf("Chart Shows Top Slow Templates (by time taken) (Top %v)", TopK),
+		}),
+	)
 	ids := map[string][]int64{}
 	var startTime time.Time = time.Now()
 	for _, event := range s.data {
@@ -200,7 +211,12 @@ func (s *ScanEventsCharts) RequestsVSInterval(c echo.Context) error {
 // requestsVSInterval generates a line chart showing requests per second over time
 func (s *ScanEventsCharts) requestsVSInterval(c echo.Context) *charts.Line {
 	line := charts.NewLine()
-	line.SetCaption("Chart Shows RPS (Requests Per Second) Over Time")
+	line.SetGlobalOptions(
+		charts.WithTitleOpts(opts.Title{
+			Title:    "Nuclei: Requests Per Second vs Time",
+			Subtitle: "Chart Shows RPS (Requests Per Second) Over Time",
+		}),
+	)
 
 	sort.Slice(s.data, func(i, j int) bool {
 		return s.data[i].Time.Before(s.data[j].Time)
@@ -267,7 +283,12 @@ func (s *ScanEventsCharts) ConcurrencyVsTime(c echo.Context) error {
 // concurrencyVsTime generates a line chart showing concurrency (total workers) over time
 func (s *ScanEventsCharts) concurrencyVsTime(c echo.Context) *charts.Line {
 	line := charts.NewLine()
-	line.SetCaption("Chart Shows Concurrency (Total Workers) Over Time")
+	line.SetGlobalOptions(
+		charts.WithTitleOpts(opts.Title{
+			Title:    "Nuclei: Concurrency vs Time",
+			Subtitle: "Chart Shows Concurrency (Total Workers) Over Time",
+		}),
+	)
 
 	dataset := sliceutil.Clone(s.data)
 

pkg/scan/scan_context.go

@@ -10,7 +10,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
 )
 
-
 type ScanContextOption func(*ScanContext)
 
 func WithEvents() ScanContextOption {
@@ -20,7 +19,8 @@ func WithEvents() ScanContextOption {
 }
 
 type ScanContext struct {
-	context.Context
+	ctx context.Context
+
 	// exported / configurable fields
 	Input *contextargs.Context
 
@@ -43,8 +43,13 @@ type ScanContext struct {
 }
 
 // NewScanContext creates a new scan context using input
-func NewScanContext(input *contextargs.Context) *ScanContext {
-	return &ScanContext{Input: input}
+func NewScanContext(ctx context.Context, input *contextargs.Context) *ScanContext {
+	return &ScanContext{ctx: ctx, Input: input}
+}
+
+// Context returns the context of the scan
+func (s *ScanContext) Context() context.Context {
+	return s.ctx
 }
 
 // GenerateResult returns final results slice from all events

pkg/templates/cluster.go

@@ -303,7 +303,7 @@ func (e *ClusterExecuter) Execute(ctx *scan.ScanContext) (bool, error) {
 
 // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
 func (e *ClusterExecuter) ExecuteWithResults(ctx *scan.ScanContext) ([]*output.ResultEvent, error) {
-	scanCtx := scan.NewScanContext(ctx.Input)
+	scanCtx := scan.NewScanContext(ctx.Context(), ctx.Input)
 	dynamicValues := make(map[string]interface{})
 
 	inputItem := ctx.Input.Clone()

pkg/templates/compile.go

@@ -16,6 +16,7 @@ import (
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v3/pkg/js/compiler"
+	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
 	"github.com/projectdiscovery/nuclei/v3/pkg/operators"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
@@ -341,6 +342,18 @@ func parseTemplate(data []byte, options protocols.ExecutorOptions) (*Template, e
 		return nil, errors.New("no template author field provided")
 	}
 
+	// use default unknown severity
+	if len(template.Workflows) == 0 {
+		if template.Info.SeverityHolder.Severity == severity.Undefined {
+			// set unknown severity with counter and forced warning
+			template.Info.SeverityHolder.Severity = severity.Unknown
+			if options.Options.Validate {
+				// when validating return error
+				return nil, errors.New("no template severity field provided")
+			}
+		}
+	}
+
 	// Setting up variables regarding template metadata
 	options.TemplateID = template.ID
 	options.TemplateInfo = template.Info

pkg/templates/parser.go

@@ -33,6 +33,11 @@ func NewParser() *Parser {
 	return p
 }
 
+// Cache returns the parsed templates cache
+func (p *Parser) Cache() *Cache {
+	return p.parsedTemplatesCache
+}
+
 // LoadTemplate returns true if the template is valid and matches the filtering criteria.
 func (p *Parser) LoadTemplate(templatePath string, t any, extraTags []string, catalog catalog.Catalog) (bool, error) {
 	tagFilter, ok := t.(*TagFilter)

pkg/testutils/testutils.go

@@ -55,6 +55,7 @@ var DefaultOptions = &types.Options{
 	Retries:                    1,
 	RateLimit:                  150,
 	RateLimitDuration:          time.Second,
+	ProbeConcurrency:           50,
 	ProjectPath:                "",
 	Severities:                 severity.Severities{},
 	Targets:                    []string{},
@@ -72,6 +73,7 @@ var DefaultOptions = &types.Options{
 	InteractionsPollDuration:   5,
 	GitHubTemplateRepo:         []string{},
 	GitHubToken:                "",
+	ResponseReadTimeout:        time.Second * 5,
 }
 
 // TemplateInfo contains info for a mock executed template.

pkg/tmplexec/exec.go

@@ -205,9 +205,25 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) {
 
 // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
 func (e *TemplateExecuter) ExecuteWithResults(ctx *scan.ScanContext) ([]*output.ResultEvent, error) {
-	err := e.engine.ExecuteWithResults(ctx)
-	ctx.LogError(err)
-	return ctx.GenerateResult(), err
+	var errx error
+	if e.options.Flow != "" {
+		flowexec, err := flow.NewFlowExecutor(e.requests, ctx, e.options, e.results, e.program)
+		if err != nil {
+			ctx.LogError(err)
+			return nil, fmt.Errorf("could not create flow executor: %s", err)
+		}
+		if err := flowexec.Compile(); err != nil {
+			ctx.LogError(err)
+			return nil, err
+		}
+		errx = flowexec.ExecuteWithResults(ctx)
+	} else {
+		errx = e.engine.ExecuteWithResults(ctx)
+	}
+	if errx != nil {
+		ctx.LogError(errx)
+	}
+	return ctx.GenerateResult(), errx
 }
 
 // getTemplateType returns the template type of the template

pkg/tmplexec/flow/flow_executor.go

@@ -174,6 +174,12 @@ func (f *FlowExecutor) Compile() error {
 
 // ExecuteWithResults executes the flow and returns results
 func (f *FlowExecutor) ExecuteWithResults(ctx *scan.ScanContext) error {
+	select {
+	case <-ctx.Context().Done():
+		return ctx.Context().Err()
+	default:
+	}
+
 	f.ctx.Input = ctx.Input
 	// -----Load all types of variables-----
 	// add all input args to template context
@@ -220,7 +226,11 @@ func (f *FlowExecutor) ExecuteWithResults(ctx *scan.ScanContext) error {
 		}
 	}
 	// register template object
-	if err := runtime.Set("template", f.options.GetTemplateCtx(f.ctx.Input.MetaInput).GetAll()); err != nil {
+	tmplObj := f.options.GetTemplateCtx(f.ctx.Input.MetaInput).GetAll()
+	if tmplObj == nil {
+		tmplObj = map[string]interface{}{}
+	}
+	if err := runtime.Set("template", tmplObj); err != nil {
 		return err
 	}
 

pkg/tmplexec/flow/flow_executor_test.go

@@ -55,8 +55,8 @@ func TestFlowTemplateWithIndex(t *testing.T) {
 	err = Template.Executer.Compile()
 	require.Nil(t, err, "could not compile template")
 
-	input := contextargs.NewWithInput("hackerone.com")
-	ctx := scan.NewScanContext(input)
+	input := contextargs.NewWithInput(context.Background(), "hackerone.com")
+	ctx := scan.NewScanContext(context.Background(), input)
 	gotresults, err := Template.Executer.Execute(ctx)
 	require.Nil(t, err, "could not execute template")
 	require.True(t, gotresults)
@@ -74,8 +74,8 @@ func TestFlowTemplateWithID(t *testing.T) {
 	err = Template.Executer.Compile()
 	require.Nil(t, err, "could not compile template")
 
-	target := contextargs.NewWithInput("hackerone.com")
-	ctx := scan.NewScanContext(target)
+	target := contextargs.NewWithInput(context.Background(), "hackerone.com")
+	ctx := scan.NewScanContext(context.Background(), target)
 	gotresults, err := Template.Executer.Execute(ctx)
 	require.Nil(t, err, "could not execute template")
 	require.True(t, gotresults)
@@ -96,8 +96,8 @@ func TestFlowWithProtoPrefix(t *testing.T) {
 	err = Template.Executer.Compile()
 	require.Nil(t, err, "could not compile template")
 
-	input := contextargs.NewWithInput("hackerone.com")
-	ctx := scan.NewScanContext(input)
+	input := contextargs.NewWithInput(context.Background(), "hackerone.com")
+	ctx := scan.NewScanContext(context.Background(), input)
 	gotresults, err := Template.Executer.Execute(ctx)
 	require.Nil(t, err, "could not execute template")
 	require.True(t, gotresults)
@@ -116,8 +116,8 @@ func TestFlowWithConditionNegative(t *testing.T) {
 	err = Template.Executer.Compile()
 	require.Nil(t, err, "could not compile template")
 
-	input := contextargs.NewWithInput("scanme.sh")
-	ctx := scan.NewScanContext(input)
+	input := contextargs.NewWithInput(context.Background(), "scanme.sh")
+	ctx := scan.NewScanContext(context.Background(), input)
 	// expect no results and verify thant dns request is executed and http is not
 	gotresults, err := Template.Executer.Execute(ctx)
 	require.Nil(t, err, "could not execute template")
@@ -137,8 +137,8 @@ func TestFlowWithConditionPositive(t *testing.T) {
 	err = Template.Executer.Compile()
 	require.Nil(t, err, "could not compile template")
 
-	input := contextargs.NewWithInput("blog.projectdiscovery.io")
-	ctx := scan.NewScanContext(input)
+	input := contextargs.NewWithInput(context.Background(), "blog.projectdiscovery.io")
+	ctx := scan.NewScanContext(context.Background(), input)
 	// positive match . expect results also verify that both dns() and http() were executed
 	gotresults, err := Template.Executer.Execute(ctx)
 	require.Nil(t, err, "could not execute template")
@@ -158,8 +158,8 @@ func TestFlowWithNoMatchers(t *testing.T) {
 	err = Template.Executer.Compile()
 	require.Nil(t, err, "could not compile template")
 
-	input := contextargs.NewWithInput("blog.projectdiscovery.io")
-	ctx := scan.NewScanContext(input)
+	input := contextargs.NewWithInput(context.Background(), "blog.projectdiscovery.io")
+	ctx := scan.NewScanContext(context.Background(), input)
 	// positive match . expect results also verify that both dns() and http() were executed
 	gotresults, err := Template.Executer.Execute(ctx)
 	require.Nil(t, err, "could not execute template")
@@ -174,8 +174,8 @@ func TestFlowWithNoMatchers(t *testing.T) {
 	err = Template.Executer.Compile()
 	require.Nil(t, err, "could not compile template")
 
-	anotherInput := contextargs.NewWithInput("blog.projectdiscovery.io")
-	anotherCtx := scan.NewScanContext(anotherInput)
+	anotherInput := contextargs.NewWithInput(context.Background(), "blog.projectdiscovery.io")
+	anotherCtx := scan.NewScanContext(context.Background(), anotherInput)
 	// positive match . expect results also verify that both dns() and http() were executed
 	gotresults, err = Template.Executer.Execute(anotherCtx)
 	require.Nil(t, err, "could not execute template")

pkg/tmplexec/generic/exec.go

@@ -45,6 +45,12 @@ func (g *Generic) ExecuteWithResults(ctx *scan.ScanContext) error {
 	previous := mapsutil.NewSyncLockMap[string, any]()
 
 	for _, req := range g.requests {
+		select {
+		case <-ctx.Context().Done():
+			return ctx.Context().Err()
+		default:
+		}
+
 		inputItem := ctx.Input.Clone()
 		if g.options.InputHelper != nil && ctx.Input.MetaInput.Input != "" {
 			if inputItem.MetaInput.Input = g.options.InputHelper.Transform(inputItem.MetaInput.Input, req.Type()); inputItem.MetaInput.Input == "" {

pkg/tmplexec/multiproto/multi.go

@@ -44,6 +44,12 @@ func (m *MultiProtocol) Compile() error {
 
 // ExecuteWithResults executes the template and returns results
 func (m *MultiProtocol) ExecuteWithResults(ctx *scan.ScanContext) error {
+	select {
+	case <-ctx.Context().Done():
+		return ctx.Context().Err()
+	default:
+	}
+
 	// put all readonly args into template context
 	m.options.GetTemplateCtx(ctx.Input.MetaInput).Merge(m.readOnlyArgs)
 
@@ -96,6 +102,12 @@ func (m *MultiProtocol) ExecuteWithResults(ctx *scan.ScanContext) error {
 
 	// execute all protocols in the queue
 	for _, req := range m.requests {
+		select {
+		case <-ctx.Context().Done():
+			return ctx.Context().Err()
+		default:
+		}
+
 		values := m.options.GetTemplateCtx(ctx.Input.MetaInput).GetAll()
 		err := req.ExecuteWithResults(ctx.Input, output.InternalEvent(values), nil, multiProtoCallback)
 		// if error skip execution of next protocols

pkg/tmplexec/multiproto/multi_test.go

@@ -54,8 +54,8 @@ func TestMultiProtoWithDynamicExtractor(t *testing.T) {
 	err = Template.Executer.Compile()
 	require.Nil(t, err, "could not compile template")
 
-	input := contextargs.NewWithInput("blog.projectdiscovery.io")
-	ctx := scan.NewScanContext(input)
+	input := contextargs.NewWithInput(context.Background(), "blog.projectdiscovery.io")
+	ctx := scan.NewScanContext(context.Background(), input)
 	gotresults, err := Template.Executer.Execute(ctx)
 	require.Nil(t, err, "could not execute template")
 	require.True(t, gotresults)
@@ -71,8 +71,8 @@ func TestMultiProtoWithProtoPrefix(t *testing.T) {
 	err = Template.Executer.Compile()
 	require.Nil(t, err, "could not compile template")
 
-	input := contextargs.NewWithInput("blog.projectdiscovery.io")
-	ctx := scan.NewScanContext(input)
+	input := contextargs.NewWithInput(context.Background(), "blog.projectdiscovery.io")
+	ctx := scan.NewScanContext(context.Background(), input)
 	gotresults, err := Template.Executer.Execute(ctx)
 	require.Nil(t, err, "could not execute template")
 	require.True(t, gotresults)

pkg/types/types.go

@@ -225,6 +225,8 @@ type Options struct {
 	TemplateDisplay bool
 	// TemplateList lists available templates
 	TemplateList bool
+	// TemplateList lists available tags
+	TagList bool
 	// HangMonitor enables nuclei hang monitoring
 	HangMonitor bool
 	// Stdin specifies whether stdin input was given to the process
@@ -289,6 +291,8 @@ type Options struct {
 	ResponseReadSize int
 	// ResponseSaveSize is the maximum size of response to save
 	ResponseSaveSize int
+	// ResponseReadTimeout is response read timeout in seconds
+	ResponseReadTimeout time.Duration
 	// Health Check
 	HealthCheck bool
 	// Time to wait between each input read operation before closing the stream
@@ -373,6 +377,8 @@ type Options struct {
 	EnableCloudUpload bool
 	// ScanID is the scan ID to use for cloud upload
 	ScanID string
+	// ScanName is the name of the scan to be uploaded
+	ScanName string
 	// JsConcurrency is the number of concurrent js routines to run
 	JsConcurrency int
 	// SecretsFile is file containing secrets for nuclei
@@ -389,6 +395,10 @@ type Options struct {
 	ProbeConcurrency int
 	// Dast only runs DAST templates
 	DAST bool
+	// HttpApiEndpoint is the experimental http api endpoint
+	HttpApiEndpoint string
+	// ListTemplateProfiles lists all available template profiles
+	ListTemplateProfiles bool
 }
 
 // ShouldLoadResume resume file
@@ -427,6 +437,7 @@ func DefaultOptions() *Options {
 		MaxHostError:            30,
 		ResponseReadSize:        10 * 1024 * 1024,
 		ResponseSaveSize:        1024 * 1024,
+		ResponseReadTimeout:     5 * time.Second,
 	}
 }
 

pkg/utils/http_probe.go

@@ -43,6 +43,13 @@ func (i *inputLivenessChecker) ProbeURL(input string) (string, error) {
 	return ProbeURL(input, i.client), nil
 }
 
+func (i *inputLivenessChecker) Close() error {
+	if i.client.Dialer != nil {
+		i.client.Dialer.Close()
+	}
+	return nil
+}
+
 // GetInputLivenessChecker returns a new input liveness checker using provided httpx client
 func GetInputLivenessChecker(client *httpx.HTTPX) types.InputLivenessProbe {
 	x := &inputLivenessChecker{client: client}